From 37d7b9121e93c6a0011fd4a9525d8a0f56dc5702 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 00:21:58 +0800 Subject: [PATCH 001/101] feat(docker): wire Docker-to-Docker links through a kernel bridge Eligible links (both endpoints Docker, same compute, no filters/markers, stopped nodes, no unix-socket NIO containers) are wired as veth pairs enslaved into a per-link Linux kernel bridge (gns3 + 11 hex of the link id, filling IFNAMSIZ) instead of the uBridge UDP relay, so frames flow entirely in the kernel. - NIOBridge NIO type: the controller decides the datapath in UDPLink._prepare and both computes enslave their deterministic gv*/gc* veth pair via uBridge's docker/brctl modules - carrier/suspend = host-end admin state; both endpoints race-safely create/delete the bridge (EEXIST/EBUSY/ENOENT tolerated) - veth adapters can still attach relay (UDP) links on a running node: lazy relay bridge over AF_PACKET on the host end (add_nio_ethernet) - filters/markers/capture are rejected with explicit errors on this datapath (they live in the uBridge relay); vendor unix-socket containers are excluded; Server.enable_kernel_datapath is the kill-switch (default on) - host-end veths survive container death and are removed on stop plus swept by name on the next start (crash/stale recovery) --- gns3server/api/routes/compute/docker_nodes.py | 14 +- gns3server/compute/base_manager.py | 6 + gns3server/compute/docker/docker_vm.py | 227 ++++++++++- gns3server/compute/docker/vendor_docker_vm.py | 7 + gns3server/compute/nios/nio_bridge.py | 67 ++++ gns3server/controller/link.py | 15 + gns3server/controller/udp_link.py | 68 ++++ gns3server/schemas/__init__.py | 3 +- gns3server/schemas/compute/nios.py | 25 +- gns3server/schemas/config.py | 5 + .../docker/test_docker_kernel_datapath.py | 373 ++++++++++++++++++ tests/compute/docker/test_vendor_docker_vm.py | 15 + tests/controller/test_kernel_datapath_link.py | 227 +++++++++++ 13 files changed, 1031 insertions(+), 21 deletions(-) create mode 100644 gns3server/compute/nios/nio_bridge.py create mode 100644 tests/compute/docker/test_docker_kernel_datapath.py create mode 100644 tests/controller/test_kernel_datapath_link.py diff --git a/gns3server/api/routes/compute/docker_nodes.py b/gns3server/api/routes/compute/docker_nodes.py index 37c00d5a3..5e7f1af98 100644 --- a/gns3server/api/routes/compute/docker_nodes.py +++ b/gns3server/api/routes/compute/docker_nodes.py @@ -260,12 +260,13 @@ async def duplicate_docker_node( @router.post( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def create_docker_node_nio( - adapter_number: int, port_number: int, nio_data: schemas.UDPNIO, node: DockerVM = Depends(dep_node) -) -> schemas.UDPNIO: + adapter_number: int, port_number: int, nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], + node: DockerVM = Depends(dep_node) +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Add a NIO (Network Input/Output) to the node. The port number on the Docker node is always 0. @@ -279,12 +280,13 @@ async def create_docker_node_nio( @router.put( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def update_docker_node_nio( - adapter_number: int, port_number: int, nio_data: schemas.UDPNIO, node: DockerVM = Depends(dep_node) -) -> schemas.UDPNIO: + adapter_number: int, port_number: int, nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], + node: DockerVM = Depends(dep_node) +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Update a NIO (Network Input/Output) on the node. The port number on the Docker node is always 0. diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index a455beb97..e7cd790fb 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -41,6 +41,7 @@ from ..utils import force_unix_path from ..utils.images import default_images_directory, images_directories, list_images, md5sum, remove_checksum from .base_node import BaseNode from .error import ImageMissingError, NodeError +from .nios.nio_bridge import NIOBridge from .nios.nio_ethernet import NIOEthernet from .nios.nio_tap import NIOTAP from .nios.nio_udp import NIOUDP @@ -358,6 +359,11 @@ class BaseManager: nio.filters = nio_settings.get("filters", {}) nio.markers = nio_settings.get("markers", {}) nio.suspend = nio_settings.get("suspend", False) + elif nio_settings["type"] == "nio_bridge": + nio = NIOBridge(nio_settings["bridge"]) + if nio_settings.get("filters") or nio_settings.get("markers"): + raise ComputeError("Packet filters and markers are not supported on kernel-datapath links") + nio.suspend = nio_settings.get("suspend", False) elif nio_settings["type"] == "nio_tap": tap_device = nio_settings["tap_device"] # if not is_interface_up(tap_device): diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 97cc661a6..c9dcf6cc5 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -44,6 +44,7 @@ from gns3server.utils.hostname import is_rfc1123_hostname_valid from ..adapters.ethernet_adapter import EthernetAdapter from ..base_node import BaseNode from ..error import ImageMissingError +from ..nios.nio_bridge import NIOBridge from ..nios.nio_udp import NIOUDP from .docker_error import DockerError, DockerHttp404Error, DockerHttp409Error @@ -169,6 +170,10 @@ class DockerVM(BaseNode): self._volumes = [] # Keep a list of created bridge self._bridges = set() + # (adapter_number, port_number) -> veth host-end name for adapters + # wired on the kernel datapath (NIOBridge). The host end lives in the + # root namespace and survives container death, unlike relay TAPs. + self._kernel_veths = {} if adapters is None: self.adapters = 1 @@ -1422,6 +1427,9 @@ class DockerVM(BaseNode): self._console_websocket = None self._cleanup_console_resources() await self._clean_servers() + # veth host ends live in the root namespace and survive container + # death — remove them while the uBridge control channel is still up. + await self._remove_kernel_veths() await self._stop_ubridge() try: @@ -1745,6 +1753,10 @@ class DockerVM(BaseNode): f"Port {port_number} doesn't exist on adapter {adapter_number} of Docker container '{self.name}'" ) + if isinstance(nio, NIOBridge): + await self._add_ubridge_kernel_connection(nio, adapter_number, port_number) + return + for index in range(4096): if f"tap-gns3-e{index}" not in psutil.net_if_addrs(): adapter.host_ifc = f"tap-gns3-e{index!s}" @@ -1758,11 +1770,7 @@ class DockerVM(BaseNode): self._bridges.add(bridge_name) await self._ubridge_send(f"bridge add_nio_tap {bridge_name} {adapter.host_ifc} off") - mac_address = int_to_macaddress(macaddress_to_int(self._mac_address) + adapter_number) - custom_adapter = self._get_custom_adapter_settings(adapter_number) - custom_mac_address = custom_adapter.get("mac_address") - if custom_mac_address: - mac_address = custom_mac_address + mac_address = self._adapter_mac_address(adapter_number) try: await self._ubridge_send(f"docker set_mac_addr {adapter.host_ifc} {mac_address}") @@ -1782,13 +1790,131 @@ class DockerVM(BaseNode): await self._connect_nio(adapter_number, nio, port_number) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) + def _adapter_mac_address(self, adapter_number): + """ + MAC address for an adapter: the node base address plus the adapter + number, unless the adapter has a custom MAC configured. + """ + + mac_address = int_to_macaddress(macaddress_to_int(self._mac_address) + adapter_number) + custom_adapter = self._get_custom_adapter_settings(adapter_number) + custom_mac_address = custom_adapter.get("mac_address") + if custom_mac_address: + mac_address = custom_mac_address + return mac_address + + def _veth_names(self, adapter_number, port_number=0): + """ + Deterministic veth pair names for a kernel-datapath adapter port. + The host end stays in the root namespace (enslaved to the per-link + kernel bridge); the guest end is moved into the container namespace + and renamed. The ``gv``/``gc`` prefixes keep these out of the ``gns3`` + bridge/TAP name space and serve as search keys for stale-interface + cleanup. 8 hex chars of the node id + adapter/port keep the names + unique and within IFNAMSIZ (15). + """ + + suffix = f"{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" + return f"gv{suffix}", f"gc{suffix}" + + async def _add_ubridge_kernel_connection(self, nio, adapter_number, port_number=0): + """ + Wire an adapter port on the kernel datapath (NIOBridge): a veth pair + whose host end will be enslaved to the per-link kernel bridge, and + whose guest end becomes the container interface (the role the TAP + plays on the relay path). No uBridge relay bridge is created — frames + flow entirely in the kernel, which is also why filters, markers and + capture are unavailable on this datapath. + """ + + try: + adapter = self._ethernet_adapters[adapter_number] + except IndexError: + raise DockerError( + "Adapter {adapter_number} doesn't exist on Docker container '{name}'".format( + name=self.name, adapter_number=adapter_number + ) + ) + + host_ifc, guest_ifc = self._veth_names(adapter_number, port_number) + # The host end lives in the root namespace and survives container + # death: best-effort removal of a stale pair from a previous run + # (unclean stop or crash) before recreating it. + for stale_ifc in (host_ifc, guest_ifc): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{stale_ifc}"') + + mac_address = self._adapter_mac_address(adapter_number) + try: + await self._ubridge_send(f'docker create_veth "{host_ifc}" "{guest_ifc}"') + # create_veth leaves the host end UP; carrier must be off until + # a link attaches (mirrors "bridge add_nio_tap ... off"). + await self._ubridge_send(f'link set "{host_ifc}" down') + try: + await self._ubridge_send(f"docker set_mac_addr {guest_ifc} {mac_address}") + except UbridgeError: + log.warning(f"Could not set MAC address {mac_address} on interface {guest_ifc}") + + ifname = self._get_container_ifname(adapter_number) + log.debug(f"Move container {self.name} adapter {guest_ifc} -> {ifname} in ns {self._namespace}") + try: + await self._ubridge_send(f"docker move_to_ns {guest_ifc} {self._namespace} {ifname}") + except UbridgeError as e: + raise UbridgeNamespaceError(e) + except UbridgeNamespaceError: + raise + except Exception: + # Don't leave a half-created pair behind (the next start would + # fail on create_veth EEXIST until the stale sweep runs). + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{host_ifc}"') + raise + + adapter.host_ifc = host_ifc + self._kernel_veths[(adapter_number, port_number)] = host_ifc + log.debug( + "Created kernel-datapath adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( + adapter_number=adapter_number, port_number=port_number, + mac_address=mac_address, namespace=self._namespace, + ) + ) + + if nio: + await self._connect_nio(adapter_number, nio, port_number) + await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) + + async def _remove_kernel_veths(self): + """ + Delete the root-namespace veth host ends. Unlike relay TAPs (which die + with the container network namespace) the host end outlives the + container and must be removed explicitly. Deleting an enslaved veth + detaches it from its bridge automatically. + """ + + if self.ubridge: + for host_ifc in self._kernel_veths.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{host_ifc}"') + self._kernel_veths.clear() + async def _get_namespace(self): result = await self.manager.query("GET", f"containers/{self._cid}/json") return int(result["State"]["Pid"]) async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): - """Replicate a Docker adapter's connection state on its TAP device.""" + """Replicate a Docker adapter's connection state on its host interface. + + Kernel-datapath adapters toggle the veth host end's admin state (the + container end loses carrier when its peer is down); relay adapters + toggle the TAP carrier from the uBridge fd holder. + """ + + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if host_ifc is not None: + state = "up" if connected else "down" + await self._ubridge_send(f'link set "{host_ifc}" {state}') + return bridge_name = self._bridge_name(adapter_number, port_number) state = "on" if connected else "off" @@ -1796,8 +1922,45 @@ class DockerVM(BaseNode): async def _connect_nio(self, adapter_number, nio, port_number=0): + if isinstance(nio, NIOBridge): + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if host_ifc is None: + raise DockerError( + "Adapter {adapter_number} port {port_number} of container '{name}' has no " + "kernel-datapath interface (it was started on the relay datapath); " + "restart the node to attach a kernel link to it".format( + adapter_number=adapter_number, port_number=port_number, name=self.name + ) + ) + # Both endpoints run this concurrently and either may create the + # bridge first: treat an existing bridge as success instead of a + # create race failure. + try: + await self._ubridge_send(f'brctl create "{nio.bridge}"') + except UbridgeError: + # Raises again if the bridge genuinely does not exist. + await self._ubridge_send(f'brctl show "{nio.bridge}"') + await self._ubridge_send(f'link set "{nio.bridge}" up') + await self._ubridge_send(f'brctl addif "{nio.bridge}" "{host_ifc}"') + return + bridge_name = self._bridge_name(adapter_number, port_number) - await self._ubridge_send(f"bridge add_nio_udp {bridge_name} {nio.lport} {nio.rhost} {nio.rport}") + if (adapter_number, port_number) in self._kernel_veths and bridge_name not in self._bridges: + # A veth adapter attaching a UDP NIO (kernel link deleted and a + # relay link created on the running node): relay through AF_PACKET + # on the veth host end instead of a TAP. + await self._ubridge_send(f"bridge create {bridge_name}") + self._bridges.add(bridge_name) + await self._ubridge_send( + 'bridge add_nio_ethernet {bridge_name} "{hostif}"'.format( + bridge_name=bridge_name, hostif=self._kernel_veths[(adapter_number, port_number)] + ) + ) + await self._ubridge_send( + "bridge add_nio_udp {bridge_name} {lport} {rhost} {rport}".format( + bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport + ) + ) if nio.capturing: await self._ubridge_send(f'bridge start_capture {bridge_name} "{nio.pcap_output_file}"') await self._ubridge_send(f"bridge start {bridge_name}") @@ -1840,6 +2003,16 @@ class DockerVM(BaseNode): """ if self.ubridge: + if isinstance(nio, NIOBridge): + if nio.filters or nio.markers: + raise DockerError( + "Packet filters and markers are not supported on kernel-datapath links; " + "delete and recreate the link to use them" + ) + if self.status == "started": + await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) + return + bridge_name = self._bridge_name(adapter_number, port_number) if bridge_name in self._bridges: await self._ubridge_apply_filters(bridge_name, nio.filters) @@ -1865,16 +2038,41 @@ class DockerVM(BaseNode): await self.stop_capture(adapter_number, port_number) if self.ubridge: nio = adapter.get_nio(port_number) - bridge_name = self._bridge_name(adapter_number, port_number) - if self.status == "started": - await self._set_adapter_carrier(adapter_number, False, port_number) - await self._ubridge_send(f"bridge stop {bridge_name}") - await self._ubridge_send(f"bridge remove_nio_udp {bridge_name} {nio.lport} {nio.rhost} {nio.rport}") + if isinstance(nio, NIOBridge): + await self._remove_kernel_nio(nio, adapter_number, port_number) + else: + bridge_name = self._bridge_name(adapter_number, port_number) + if self.status == "started": + await self._set_adapter_carrier(adapter_number, False, port_number) + await self._ubridge_send(f"bridge stop {bridge_name}") + await self._ubridge_send( + "bridge remove_nio_udp {bridge_name} {lport} {rhost} {rport}".format( + bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport + ) + ) adapter.remove_nio(port_number) log.debug(f"Docker VM '{self.name}' [{self.id}]: {adapter.host_ifc} removed from adapter {adapter_number}") + async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): + """ + Detach a kernel-datapath adapter port from its per-link kernel bridge. + Both endpoints run this concurrently: deleting a bridge that still has + the peer's port enslaved fails with EBUSY, so the last endpoint to + remove its port wins the deletion and the loser's failure is expected + and suppressed. + """ + + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if self.status == "started" and host_ifc is not None: + await self._set_adapter_carrier(adapter_number, False, port_number) + if host_ifc is not None: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delete "{nio.bridge}"') + def get_nio(self, adapter_number, port_number=0): """ Gets an adapter NIO binding. @@ -1964,6 +2162,11 @@ class DockerVM(BaseNode): nio = self.get_nio(adapter_number, port_number) if nio.capturing: raise DockerError(f"Packet capture is already activated on adapter {adapter_number}") + if isinstance(nio, NIOBridge): + raise DockerError( + "Packet capture is not supported on kernel-datapath links (there is no uBridge " + "relay in the forwarding path); delete and recreate the link to capture it" + ) nio.start_packet_capture(output_file) if self.status == "started" and self.ubridge: diff --git a/gns3server/compute/docker/vendor_docker_vm.py b/gns3server/compute/docker/vendor_docker_vm.py index 977e3ace4..c1fac493b 100644 --- a/gns3server/compute/docker/vendor_docker_vm.py +++ b/gns3server/compute/docker/vendor_docker_vm.py @@ -36,6 +36,7 @@ import tempfile from gns3server.compute.docker.docker_error import DockerError, DockerHttp304Error, DockerHttp404Error from gns3server.compute.docker.docker_vm import DockerVM +from gns3server.compute.nios.nio_bridge import NIOBridge from gns3server.utils.asyncio import wait_for_file_creation from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer @@ -415,6 +416,12 @@ class VendorDockerVM(DockerVM): if not self._unix_socket_nio: return await super()._add_ubridge_connection(nio, adapter_number, port_number) + if isinstance(nio, NIOBridge): + raise DockerError( + f"Container '{self._name}' bridges its adapters through AF_UNIX sockets; " + "kernel-datapath links are not supported for this container" + ) + try: adapter = self._ethernet_adapters[adapter_number] except IndexError: diff --git a/gns3server/compute/nios/nio_bridge.py b/gns3server/compute/nios/nio_bridge.py new file mode 100644 index 000000000..6692752d1 --- /dev/null +++ b/gns3server/compute/nios/nio_bridge.py @@ -0,0 +1,67 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Interface for kernel-datapath bridge NIOs. + +A NIO of this type instructs the node to enslave its adapter's veth host end +into the named Linux kernel bridge instead of wiring a uBridge UDP relay. +Frames then flow entirely in the kernel (veth -> bridge -> veth). Filters, +markers and packet capture are not available on this datapath (they live in +the uBridge relay); the controller must only emit this NIO type for links +where those features are not in use. +""" + +from .nio import NIO + + +class NIOBridge(NIO): + + """ + Kernel-bridge NIO. + + :param bridge: name of the per-link Linux kernel bridge + """ + + def __init__(self, bridge): + + super().__init__() + self._bridge = bridge + + @property + def bridge(self): + """ + Returns the kernel bridge name. + + :returns: bridge name + """ + + return self._bridge + + def __str__(self): + + return "NIO bridge" + + def asdict(self): + + return { + "type": "nio_bridge", + "bridge": self._bridge, + "suspend": self._suspended, + "filters": self._filters, + "markers": self._markers, + } diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index ff20f7c6b..1091192b8 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -111,6 +111,15 @@ class Link: """ return self._filters + @property + def kernel_datapath(self): + """ + Whether this link is wired on the kernel datapath (veth pairs + enslaved into a per-link Linux bridge — no uBridge relay in the + forwarding path, so filters, markers and capture are unavailable). + """ + return any(d.get("type") == "nio_bridge" for d in (getattr(self, "_link_data", None) or [])) + @property def markers(self): """ @@ -224,6 +233,12 @@ class Link: except FilterValidationError as e: raise ControllerError(f"Invalid packet filter parameters: {e!s}") + if new_filters and self.kernel_datapath: + raise ControllerError( + "Packet filters are not supported on kernel-datapath links (no uBridge relay " + "in the forwarding path); delete and recreate the link to use filters" + ) + if new_filters != self.filters: self._filters = new_filters if self._created: diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 2763c48e7..7a479c1cd 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -19,6 +19,7 @@ import asyncio import logging +from gns3server.config import Config from gns3server.utils.packet_filter_validation import validate_bpf_syntax from .controller_error import ControllerError, ControllerNotFoundError @@ -46,6 +47,19 @@ _MARKER_CAPABLE_TYPES = frozenset( log = logging.getLogger(__name__) +def _is_unix_socket_docker(node): + """ + Best-effort detection of vendor containers bridged through AF_UNIX + socket pairs (GNS3_UNIX_SOCKET_NIO in their environment): they have no + host-side interface to enslave into a kernel bridge. The compute side + independently rejects kernel-datapath NIOs on these containers, so a + missed detection here only surfaces as a clearer-late error. + """ + + environment = (node.properties or {}).get("environment") or "" + return "GNS3_UNIX_SOCKET_NIO" in environment + + class UDPLink(Link): def __init__(self, project, link_id=None): super().__init__(project, link_id=link_id) @@ -110,6 +124,38 @@ class UDPLink(Link): """ return self._markers_for_node(node1), self._markers_for_node(node2) + def _kernel_datapath_eligible(self, node1, node2): + """ + Whether this link can be wired on the kernel datapath (veth pairs + enslaved into a per-link Linux bridge) instead of the uBridge UDP + relay. The kernel path has no userspace relay, so filters, markers + and packet capture — which all live in the relay — disqualify it. + It is host-local (same compute) and requires the adapter interfaces + to be created as veths at container start, hence stopped nodes only. + """ + + if not Config.instance().settings.Server.enable_kernel_datapath: + return False + if node1.node_type != "docker" or node2.node_type != "docker": + return False + if node1.compute.id != node2.compute.id: + return False + if self.get_active_filters(): + return False + if self._markers: + return False + # Project-level marker definitions are inherited by every link right + # after creation (apply_defs_to_new_link) and need the relay datapath. + if getattr(self._project, "_marker_definitions", None): + return False + # Running nodes already have relay TAPs wired into uBridge; the + # kernel path needs the veth variant created at container start. + if node1.status != "stopped" or node2.status != "stopped": + return False + if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): + return False + return True + async def _prepare(self): """ Local-only link setup: resolve peer addresses, reserve UDP ports and @@ -135,6 +181,22 @@ class UDPLink(Link): adapter_number2 = self._nodes[1]["adapter_number"] port_number2 = self._nodes[1]["port_number"] + if self._kernel_datapath_eligible(node1, node2): + # Kernel datapath: each endpoint enslaves its veth host end into + # a per-link Linux bridge. The name derives from the link id so + # both ends compute it independently (no coordinator) and crash + # leftovers sweep deterministically; 11 hex chars fill IFNAMSIZ + # (15), pushing name-collision probability to irrelevance. + bridge_name = "gns3" + self._id.replace("-", "")[:11] + self._link_data = [ + {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": {}, "suspend": self._suspended}, + {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": {}, "suspend": self._suspended}, + ] + return [ + (node1, adapter_number1, port_number1, self._link_data[0]), + (node2, adapter_number2, port_number2, self._link_data[1]), + ] + # Get an IP allowing communication between both host try: (node1_host, node2_host) = await node1.compute.get_ip_on_same_subnet(node2.compute) @@ -472,6 +534,12 @@ class UDPLink(Link): if name in self._markers: raise ControllerError(f"Marker '{name}' already exists on link {self._id}") + if self.kernel_datapath: + raise ControllerError( + "Markers are not supported on kernel-datapath links (no uBridge relay in the " + "forwarding path); delete and recreate the link to use markers" + ) + # Validate the BPF only for private per-link markers. An inherited copy # (``inherited_from`` set) fans out from a definition whose BPF was # already validated once at create/update (and on project load), so diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index cdc1d67ce..196ada3ab 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -101,7 +101,7 @@ from .compute.iou_nodes import IOU, IOUCreate, IOUStart, IOUUpdate from .compute.nat_nodes import NAT, NATCreate, NATUpdate # Compute schemas -from .compute.nios import TAPNIO, UDPNIO, BatchNIOCreate, BatchNIOEntry, EthernetNIO, MarkerRebuild, MarkerToggle +from .compute.nios import TAPNIO, UDPNIO, BridgeNIO, BatchNIOCreate, BatchNIOEntry, EthernetNIO, MarkerRebuild, MarkerToggle from .compute.qemu_nodes import Qemu, QemuCreate, QemuUpdate from .compute.virtualbox_nodes import VirtualBox, VirtualBoxCreate, VirtualBoxUpdate from .compute.vmware_nodes import VMware, VMwareCreate, VMwareUpdate @@ -166,6 +166,7 @@ __all__ = [ "AutoIdlePC", "BatchNIOCreate", "BatchNIOEntry", + "BridgeNIO", "C1700DynamipsTemplate", "C1700DynamipsTemplateUpdate", "C2600DynamipsTemplate", diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 38078812c..10f8ab565 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -16,7 +16,7 @@ from enum import Enum -from typing import List, Optional +from typing import List, Optional, Union from pydantic import BaseModel, Field @@ -65,6 +65,27 @@ class TAPNIO(BaseModel): tap_device: str = Field(..., description="TAP device name e.g. tap0") +class BridgeNIOType(str, Enum): + + bridge = "nio_bridge" + + +class BridgeNIO(BaseModel): + """ + Kernel-datapath bridge Network Input/Output properties. The NIO tells the + node to enslave its veth host end into the named kernel bridge instead of + wiring a uBridge UDP relay. ``filters`` and ``markers`` must stay empty on + this datapath; they are accepted (and rejected server-side) only because + link update payloads always carry the full NIO dictionary. + """ + + type: BridgeNIOType + bridge: str = Field(..., description="Kernel bridge name e.g. gns3a1b2c3d4e5") + suspend: Optional[bool] = Field(None, description="Suspend the NIO") + filters: Optional[dict] = Field(None, description="Packet filters (unsupported, must be empty)") + markers: Optional[dict] = Field(None, description="Traffic-insight markers (unsupported, must be empty)") + + class MarkerToggle(BaseModel): """ Body for the per-marker enable/disable toggle endpoint: flips a running @@ -99,7 +120,7 @@ class BatchNIOEntry(BaseModel): node_id: str = Field(..., description="Node the NIO is attached to") adapter_number: int = Field(0, ge=0, description="Adapter number") port_number: int = Field(0, ge=0, description="Port number") - nio: UDPNIO = Field(..., description="NIO settings") + nio: Union[UDPNIO, BridgeNIO] = Field(..., description="NIO settings") class BatchNIOCreate(BaseModel): diff --git a/gns3server/schemas/config.py b/gns3server/schemas/config.py index f6c978911..fc83f39fd 100644 --- a/gns3server/schemas/config.py +++ b/gns3server/schemas/config.py @@ -183,6 +183,11 @@ class ServerSettings(BaseModel): description='Default symbol theme, e.g. "Classic" or "Affinity-square-blue"', ) allow_raw_images: bool = Field(True, description="Allow raw images to be uploaded to the server") + enable_kernel_datapath: bool = Field( + True, + description="Wire eligible Docker-to-Docker links (same compute, no filters/markers) through " + "kernel veth/bridge interfaces instead of the uBridge UDP relay", + ) auto_discover_images: bool = Field(True, description="Automatically discover images in the images directory") image_sync_interval: int = Field(900, ge=10, description="Seconds between automatic image inventory scans") report_errors: bool = Field(True, description="Automatically send crash reports to the GNS3 team") diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py new file mode 100644 index 000000000..bc86e5fb5 --- /dev/null +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -0,0 +1,373 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Kernel-datapath (NIOBridge) adapter wiring for Docker containers: veth pair +creation, per-link kernel bridge enslavement, carrier semantics, relay +fallback and cleanup. +""" + +import uuid +from unittest.mock import MagicMock, call + +import pytest +import pytest_asyncio + +from tests.utils import AsyncioMagicMock + +from gns3server.compute.docker import Docker +from gns3server.compute.docker.docker_vm import DockerVM +from gns3server.compute.docker.docker_error import DockerError +from gns3server.compute.compute_error import ComputeError +from gns3server.compute.nios.nio_udp import NIOUDP +from gns3server.compute.nios.nio_bridge import NIOBridge +from gns3server.compute.ubridge.ubridge_error import UbridgeError + + +BRIDGE = "gns3a1b2c3d4e5f" + + +@pytest_asyncio.fixture +async def manager(port_manager): + + m = Docker.instance() + m.port_manager = port_manager + return m + + +@pytest_asyncio.fixture(scope="function") +async def vm(compute_project, manager): + + vm = DockerVM("test", str(uuid.uuid4()), compute_project, manager, "ubuntu:latest", aux_type="none") + vm._cid = "e90e34656842" + vm.mac_address = '02:42:3d:b7:93:00' + vm._start_interface_monitor = AsyncioMagicMock() + vm._stop_interface_monitor = AsyncioMagicMock() + return vm + + +# --------------------------------------------------------------------------- +# NIO factory +# --------------------------------------------------------------------------- + +def test_create_nio_bridge(vm): + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + assert isinstance(nio, NIOBridge) + assert nio.bridge == BRIDGE + assert nio.suspend is False + + +def test_create_nio_bridge_rejects_filters(vm): + + with pytest.raises(ComputeError): + vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"loss": [10]}}) + + +def test_create_nio_bridge_rejects_markers(vm): + + with pytest.raises(ComputeError): + vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "markers": {"m": {"bpf": "icmp"}}}) + + +# --------------------------------------------------------------------------- +# Adapter wiring (start path) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_add_ubridge_kernel_connection(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + await vm._add_ubridge_connection(nio, 0) + + host_ifc, guest_ifc = vm._veth_names(0, 0) + calls = [ + # stale-pair sweep (no-ops on a clean host) + call.send(f'docker delete_veth "{host_ifc}"'), + call.send(f'docker delete_veth "{guest_ifc}"'), + call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), + call.send(f'link set "{host_ifc}" down'), + call.send(f'docker set_mac_addr {guest_ifc} 02:42:3d:b7:93:00'), + call.send(f'docker move_to_ns {guest_ifc} 42 eth0'), + # link attach + call.send(f'brctl create "{BRIDGE}"'), + call.send(f'link set "{BRIDGE}" up'), + call.send(f'brctl addif "{BRIDGE}" "{host_ifc}"'), + call.send(f'link set "{host_ifc}" up'), + ] + vm._ubridge_hypervisor.assert_has_calls(calls, any_order=True) + assert vm._kernel_veths[(0, 0)] == host_ifc + # no uBridge relay bridge on the kernel datapath + assert "bridge0" not in vm._bridges + + +@pytest.mark.asyncio +async def test_add_ubridge_connection_none_nio_stays_on_tap_path(vm): + """ + Unconnected adapters keep the relay TAP: the veth variant is only created + for adapters whose NIO is a kernel one at start time. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + + await vm._add_ubridge_connection(None, 0) + + vm._ubridge_hypervisor.assert_has_calls([ + call.send("bridge create bridge0"), + call.send("bridge add_nio_tap bridge0 tap-gns3-e0 off"), + ], any_order=True) + assert vm._kernel_veths == {} + + +@pytest.mark.asyncio +async def test_add_ubridge_kernel_connection_cleans_up_on_failure(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + + async def failing_send(command): + if "create_veth" in command and "delete" not in command: + raise UbridgeError("could not complete netlink transaction") + + vm._ubridge_send = AsyncioMagicMock(side_effect=failing_send) + + with pytest.raises(UbridgeError): + await vm._add_ubridge_connection(vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}), 0) + + host_ifc, _ = vm._veth_names(0, 0) + vm._ubridge_send.assert_any_call(f'docker delete_veth "{host_ifc}"') + assert (0, 0) not in vm._kernel_veths + + +# --------------------------------------------------------------------------- +# Link attach / detach +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_connect_nio_kernel_tolerates_bridge_create_race(vm): + + # Both endpoints create the per-link bridge concurrently; the loser must + # verify (brctl show) instead of failing. + async def send(command): + if 'brctl create' in command: + raise UbridgeError("Could not create bridge: File exists") + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + await vm._connect_nio(0, nio) + + vm._ubridge_send.assert_any_call(f'brctl show "{BRIDGE}"') + vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{host_ifc}"') + + +@pytest.mark.asyncio +async def test_connect_nio_kernel_create_failure_propagates(vm): + + async def send(command): + if 'brctl create' in command or 'brctl show' in command: + raise UbridgeError("Could not create bridge: No such device") + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + vm._kernel_veths[(0, 0)] = vm._veth_names(0, 0)[0] + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + with pytest.raises(UbridgeError): + await vm._connect_nio(0, nio) + + +@pytest.mark.asyncio +async def test_connect_nio_kernel_without_veth_raises(vm): + + vm._ubridge_send = AsyncioMagicMock() + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + with pytest.raises(DockerError, match="restart the node"): + await vm._connect_nio(0, nio) + + +@pytest.mark.asyncio +async def test_remove_kernel_nio(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + vm._ethernet_adapters[0].add_nio(0, vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE})) + + await vm.adapter_remove_nio_binding(0) + + vm._ubridge_send.assert_has_calls([ + call(f'link set "{host_ifc}" down'), + call(f'brctl delif "{BRIDGE}" "{host_ifc}"'), + call(f'brctl delete "{BRIDGE}"'), + ], any_order=True) + assert vm._ethernet_adapters[0].get_nio(0) is None + # the veth itself stays: it is the adapter interface, not the link + assert vm._kernel_veths[(0, 0)] == host_ifc + + +@pytest.mark.asyncio +async def test_remove_kernel_nio_tolerates_peer_winning_bridge_delete(vm): + + # Deleting a bridge that still holds the peer's port fails with EBUSY; + # a bridge already deleted by the peer fails with ENOENT — both expected. + vm._ubridge_hypervisor = MagicMock() + + async def send(command): + if 'brctl delif' in command or 'brctl delete' in command: + raise UbridgeError("busy") + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + vm._kernel_veths[(0, 0)] = vm._veth_names(0, 0)[0] + vm._ethernet_adapters[0].add_nio(0, vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE})) + + await vm.adapter_remove_nio_binding(0) # must not raise + + +# --------------------------------------------------------------------------- +# Carrier / suspend +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_set_adapter_carrier_kernel(vm): + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + await vm._set_adapter_carrier(0, True) + await vm._set_adapter_carrier(0, False) + + vm._ubridge_send.assert_has_calls([ + call(f'link set "{host_ifc}" up'), + call(f'link set "{host_ifc}" down'), + ]) + + +# --------------------------------------------------------------------------- +# UDP fallback on a veth adapter (kernel link deleted, relay link created +# on a running node) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_connect_nio_udp_relays_over_veth(vm): + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) + await vm._connect_nio(0, nio) + + vm._ubridge_send.assert_has_calls([ + call("bridge create bridge0"), + call(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), + call("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), + call("bridge start bridge0"), + ], any_order=True) + assert "bridge0" in vm._bridges + + +@pytest.mark.asyncio +async def test_connect_nio_udp_on_veth_is_idempotent(vm): + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + vm._bridges.add("bridge0") # relay already exists from a previous link + + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) + await vm._connect_nio(0, nio) + + for c in vm._ubridge_send.call_args_list: + command = c.args[0] + assert "bridge create" not in command + assert "add_nio_ethernet" not in command + + +# --------------------------------------------------------------------------- +# Guards and cleanup +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_update_nio_kernel_rejects_filters(vm): + + vm._ubridge_hypervisor = MagicMock() + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"loss": [50]} + + with pytest.raises(DockerError, match="kernel-datapath"): + await vm.adapter_update_nio_binding(0, nio) + + +@pytest.mark.asyncio +async def test_update_nio_kernel_suspend_toggles_carrier(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + nio.suspend = True + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'link set "{host_ifc}" down') + + +@pytest.mark.asyncio +async def test_start_capture_rejected_on_kernel_link(vm): + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + with pytest.raises(DockerError, match="kernel-datapath"): + await vm.start_capture(0, "/tmp/capture.pcap") + + +@pytest.mark.asyncio +async def test_remove_kernel_veths(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + await vm._remove_kernel_veths() + + vm._ubridge_send.assert_called_once_with(f'docker delete_veth "{host_ifc}"') + assert vm._kernel_veths == {} + + +def test_veth_names_deterministic_and_bounded(vm): + + host_ifc, guest_ifc = vm._veth_names(3, 2) + assert host_ifc == f"gv{vm.id.replace('-', '')[:8]}e3p2" + assert guest_ifc == f"gc{vm.id.replace('-', '')[:8]}e3p2" + assert host_ifc != guest_ifc + # multi-port adapters can reach two-digit adapter/port numbers + long_host, long_guest = vm._veth_names(12, 8) + assert len(long_host) <= 15 and len(long_guest) <= 15 diff --git a/tests/compute/docker/test_vendor_docker_vm.py b/tests/compute/docker/test_vendor_docker_vm.py index e594cf5fa..93ef04905 100644 --- a/tests/compute/docker/test_vendor_docker_vm.py +++ b/tests/compute/docker/test_vendor_docker_vm.py @@ -902,3 +902,18 @@ async def test_create_reparse_refreshes_env_knobs(compute_project, manager): assert vm._stop_timeout == 5 assert vm._gns3_init is True # removed entry reset to default + + +# --------------------------------------------------------------------------- +# Kernel-datapath (NIOBridge) rejection +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_unix_socket_nio_rejects_kernel_datapath_link(compute_project, manager): + + vm = _make_vm(compute_project, manager, environment="GNS3_UNIX_SOCKET_NIO=1") + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + nio = manager.create_nio({"type": "nio_bridge", "bridge": "gns3a1b2c3d4e5f"}) + with pytest.raises(DockerError, match="kernel-datapath"): + await vm._add_ubridge_connection(nio, 0) diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py new file mode 100644 index 000000000..f5e9dc938 --- /dev/null +++ b/tests/controller/test_kernel_datapath_link.py @@ -0,0 +1,227 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Controller-side kernel-datapath link selection: eligibility of Docker-to- +Docker links, NIO emission and the filters/markers guards. +""" + +import pytest +from unittest.mock import MagicMock + +from gns3server.config import Config +from gns3server.controller.udp_link import UDPLink +from gns3server.controller.ports.ethernet_port import EthernetPort +from gns3server.controller.node import Node +from gns3server.controller.controller_error import ControllerError + + +def _node(project, compute, name, node_type="docker", status="stopped", environment=None): + node = Node(project, compute, name, node_type=node_type) + node._ports = [EthernetPort("eth0", 0, 0, 0)] + node._status = status + if environment is not None: + node._properties = {"environment": environment} + return node + + +async def _kernel_link(project, **kwargs): + compute = MagicMock() + compute.id = "compute-1" + node1 = _node(project, compute, "docker1", **kwargs) + node2 = _node(project, compute, "docker2", **kwargs) + link = UDPLink(project) + # batch=True: skip the interactive create() (the test drives _prepare() + # itself) — matches the project-open bulk path. + await link.add_node(node1, 0, 0, batch=True, dump=False) + await link.add_node(node2, 0, 0, batch=True, dump=False) + return link, node1, node2 + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible(project): + + link, node1, node2 = await _kernel_link(project) + assert link._kernel_datapath_eligible(node1, node2) is True + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_other_node_types(project): + + compute = MagicMock() + compute.id = "compute-1" + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "qemu1", node_type="qemu") + link = UDPLink(project) + await link.add_node(node1, 0, 0, batch=True, dump=False) + await link.add_node(node2, 0, 0, batch=True, dump=False) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_across_computes(project): + + compute1 = MagicMock() + compute1.id = "compute-1" + compute2 = MagicMock() + compute2.id = "compute-2" + node1 = _node(project, compute1, "docker1") + node2 = _node(project, compute2, "docker2") + link = UDPLink(project) + await link.add_node(node1, 0, 0, batch=True, dump=False) + await link.add_node(node2, 0, 0, batch=True, dump=False) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_with_filters(project): + + link, node1, node2 = await _kernel_link(project) + link._filters = {"delay": [10, 0]} + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_with_markers(project): + + link, node1, node2 = await _kernel_link(project) + link._markers = {"m": {"bpf": "icmp"}} + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_with_project_marker_definitions(project): + + link, node1, node2 = await _kernel_link(project) + project._marker_definitions = {"global-m": {"bpf": "icmp"}} + try: + assert link._kernel_datapath_eligible(node1, node2) is False + finally: + project._marker_definitions = {} + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_running_nodes(project): + + link, node1, node2 = await _kernel_link(project) + node1._status = "started" + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_unix_socket_containers(project): + + link, node1, node2 = await _kernel_link(project) + node2._properties = {"environment": "GNS3_UNIX_SOCKET_NIO=1"} + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_disabled_by_config(project): + + settings = Config.instance().settings.Server + settings.enable_kernel_datapath = False + try: + link, node1, node2 = await _kernel_link(project) + assert link._kernel_datapath_eligible(node1, node2) is False + finally: + settings.enable_kernel_datapath = True + + +@pytest.mark.asyncio +async def test_prepare_kernel_link_emits_bridge_nios(project): + + compute = MagicMock() + compute.id = "compute-1" + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "docker2") + link = UDPLink(project) + await link.add_node(node1, 0, 0, batch=True, dump=False) + await link.add_node(node2, 0, 0, batch=True, dump=False) + + entries = await link._prepare() + + assert len(entries) == 2 + expected_bridge = "gns3" + link.id.replace("-", "")[:11] + assert len(expected_bridge) == 15 + for entry in entries: + assert entry[3]["type"] == "nio_bridge" + assert entry[3]["bridge"] == expected_bridge + # no UDP port reservation on the kernel datapath + compute.post.assert_not_called() + + +@pytest.mark.asyncio +async def test_prepare_relay_link_allocates_udp_ports(project): + + compute1 = MagicMock() + compute1.id = "compute-1" + compute2 = MagicMock() + compute2.id = "compute-2" + + async def subnet_callback(other): + return ("192.168.1.1", "192.168.1.2") + + compute1.get_ip_on_same_subnet.side_effect = subnet_callback + + async def port_callback(path, data={}, **kwargs): + response = MagicMock() + response.json = {"udp_port": 1024} + return response + + compute1.post.side_effect = port_callback + compute2.post.side_effect = port_callback + + node1 = _node(project, compute1, "docker1") + node2 = _node(project, compute2, "docker2") + link = UDPLink(project) + await link.add_node(node1, 0, 0, batch=True, dump=False) + await link.add_node(node2, 0, 0, batch=True, dump=False) + + entries = await link._prepare() + + for entry in entries: + assert entry[3]["type"] == "nio_udp" + + +@pytest.mark.asyncio +async def test_update_filters_rejected_on_kernel_link(project): + + link, _n1, _n2 = await _kernel_link(project) + await link._prepare() + + with pytest.raises(ControllerError, match="kernel-datapath"): + await link.update_filters({"delay": [10, 0]}) + + +@pytest.mark.asyncio +async def test_start_marker_rejected_on_kernel_link(project): + + link, _n1, _n2 = await _kernel_link(project) + await link._prepare() + + with pytest.raises(ControllerError, match="kernel-datapath"): + await link.start_marker("m", "icmp") + + +@pytest.mark.asyncio +async def test_kernel_datapath_property(project): + + link, _n1, _n2 = await _kernel_link(project) + assert link.kernel_datapath is False + await link._prepare() + assert link.kernel_datapath is True From 58b97b4ce5e4ec1af580ce39a049ee7d645c959f Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 00:39:03 +0800 Subject: [PATCH 002/101] fix(controller): don't push the synthetic suspend filter to kernel links update_suspend() piggybacks on the filter system: get_active_filters() returns {"frequency_drop": [-1]} while a link is suspended so relay-based node types drop all traffic at the uBridge filter. Kernel-datapath links implement suspend natively via interface carrier, and pushing the synthetic filter tripped the compute-side guard, making suspend fail with 409 on every kernel link (and leaving controller/kernel state torn). _get_node_filters() now returns empty filters for kernel links. --- gns3server/controller/udp_link.py | 7 +++++ tests/controller/test_kernel_datapath_link.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 7a479c1cd..d78bf6c17 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -88,6 +88,13 @@ class UDPLink(Link): :returns: Tuple of (node1_filters, node2_filters) """ + # Kernel-datapath links implement suspend natively (interface carrier + # driven by the NIO suspend flag). The synthetic frequency_drop filter + # that get_active_filters() injects for suspended links is a + # relay-datapath emulation mechanism — sending it would trip the + # compute-side kernel guard, so kernel links always push empty filters. + if self.kernel_datapath: + return {}, {} filter_node = self._get_filter_node() return ( self.get_active_filters() if filter_node == node1 else {}, diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index f5e9dc938..e3502cdca 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -225,3 +225,32 @@ async def test_kernel_datapath_property(project): assert link.kernel_datapath is False await link._prepare() assert link.kernel_datapath is True + + +@pytest.mark.asyncio +async def test_update_suspend_kernel_link_sends_no_synthetic_filter(project): + """ + Suspend is emulated on the relay datapath via a synthetic frequency_drop + filter (get_active_filters); kernel links implement it natively via + interface carrier and must push empty filters — the compute-side guard + rejects non-empty filters on kernel NIOs. + """ + + link, node1, node2 = await _kernel_link(project) + await link._prepare() + + bodies = [] + + async def capture_put(path, data=None, **kwargs): + bodies.append(data) + + node1.put = capture_put + node2.put = capture_put + + await link.update_suspend(True) + + assert link._suspended is True + assert len(bodies) == 2 + for body in bodies: + assert body["filters"] == {} + assert body["suspend"] is True From f6b869be6338193831779f5cf597339731608626 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 00:46:29 +0800 Subject: [PATCH 003/101] feat(docker): packet capture on kernel-datapath links via AF_PACKET Replace the capture rejection with wiring to uBridge's capture module: an AF_PACKET socket bound to the veth host end writes into the same pcap file the relay path uses (capture start_kernel "" / capture stop_kernel), so the pcap streaming to clients is unchanged. - capture survives node restarts: _connect_nio re-arms it when the NIO is already capturing, mirroring the relay path - uBridge's kernel capture is a singleton per process: a concurrent second capture on the same node surfaces uBridge's EALREADY error - update the datapath docs: filters and markers remain relay-only, capture is now served on both datapaths --- gns3server/compute/docker/docker_vm.py | 29 ++++++--- gns3server/controller/link.py | 3 +- gns3server/controller/udp_link.py | 5 +- gns3server/schemas/compute/nios.py | 3 +- .../docker/test_docker_kernel_datapath.py | 65 ++++++++++++++++++- 5 files changed, 90 insertions(+), 15 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index c9dcf6cc5..a3f98f96f 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1823,8 +1823,9 @@ class DockerVM(BaseNode): whose host end will be enslaved to the per-link kernel bridge, and whose guest end becomes the container interface (the role the TAP plays on the relay path). No uBridge relay bridge is created — frames - flow entirely in the kernel, which is also why filters, markers and - capture are unavailable on this datapath. + flow entirely in the kernel. Filters and markers are unavailable on + this datapath (they live in the uBridge relay); packet capture is + provided by uBridge's AF_PACKET module on the veth host end. """ try: @@ -1942,6 +1943,10 @@ class DockerVM(BaseNode): await self._ubridge_send(f'brctl show "{nio.bridge}"') await self._ubridge_send(f'link set "{nio.bridge}" up') await self._ubridge_send(f'brctl addif "{nio.bridge}" "{host_ifc}"') + if nio.capturing: + # Restore a capture that was active before a node restart + # (mirrors the relay path's start_capture in _connect_nio). + await self._ubridge_send(f'capture start_kernel {host_ifc} "{nio.pcap_output_file}"') return bridge_name = self._bridge_name(adapter_number, port_number) @@ -2132,9 +2137,16 @@ class DockerVM(BaseNode): :param port_number: port number on the adapter (0 for single-port adapters) """ - bridge_name = self._bridge_name(adapter_number, port_number) if not self.ubridge: raise DockerError("Cannot start the packet capture: uBridge is not running") + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if host_ifc is not None: + # Kernel datapath: no relay bridge exists — capture via uBridge's + # AF_PACKET module bound to the veth host end. Single capture per + # uBridge process: a concurrent second one returns EALREADY. + await self._ubridge_send(f'capture start_kernel {host_ifc} "{output_file}"') + return + bridge_name = self._bridge_name(adapter_number, port_number) await self._ubridge_send(f'bridge start_capture {bridge_name} "{output_file}"') async def _stop_ubridge_capture(self, adapter_number, port_number=0): @@ -2145,9 +2157,13 @@ class DockerVM(BaseNode): :param port_number: port number on the adapter (0 for single-port adapters) """ - bridge_name = self._bridge_name(adapter_number, port_number) if not self.ubridge: raise DockerError("Cannot stop the packet capture: uBridge is not running") + if (adapter_number, port_number) in self._kernel_veths: + # Idempotent on the uBridge side (no active capture is OK). + await self._ubridge_send("capture stop_kernel") + return + bridge_name = self._bridge_name(adapter_number, port_number) await self._ubridge_send(f"bridge stop_capture {bridge_name}") async def start_capture(self, adapter_number, output_file, port_number=0): @@ -2162,11 +2178,6 @@ class DockerVM(BaseNode): nio = self.get_nio(adapter_number, port_number) if nio.capturing: raise DockerError(f"Packet capture is already activated on adapter {adapter_number}") - if isinstance(nio, NIOBridge): - raise DockerError( - "Packet capture is not supported on kernel-datapath links (there is no uBridge " - "relay in the forwarding path); delete and recreate the link to capture it" - ) nio.start_packet_capture(output_file) if self.status == "started" and self.ubridge: diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 1091192b8..a919c8037 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -116,7 +116,8 @@ class Link: """ Whether this link is wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge — no uBridge relay in the - forwarding path, so filters, markers and capture are unavailable). + forwarding path, so filters and markers are unavailable; capture is + served by uBridge's AF_PACKET module). """ return any(d.get("type") == "nio_bridge" for d in (getattr(self, "_link_data", None) or [])) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index d78bf6c17..27b2923a1 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -135,8 +135,9 @@ class UDPLink(Link): """ Whether this link can be wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge) instead of the uBridge UDP - relay. The kernel path has no userspace relay, so filters, markers - and packet capture — which all live in the relay — disqualify it. + relay. The kernel path has no userspace relay, so filters and + markers — which live in the relay — disqualify it (capture is + served separately by uBridge's AF_PACKET module). It is host-local (same compute) and requires the adapter interfaces to be created as veths at container start, hence stopped nodes only. """ diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 10f8ab565..4ae5a6475 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -75,7 +75,8 @@ class BridgeNIO(BaseModel): Kernel-datapath bridge Network Input/Output properties. The NIO tells the node to enslave its veth host end into the named kernel bridge instead of wiring a uBridge UDP relay. ``filters`` and ``markers`` must stay empty on - this datapath; they are accepted (and rejected server-side) only because + this datapath (rejected server-side); packet capture is supported via + uBridge's AF_PACKET module. Both are accepted as fields only because link update payloads always carry the full NIO dictionary. """ diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index bc86e5fb5..e9437861c 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -339,15 +339,76 @@ async def test_update_nio_kernel_suspend_toggles_carrier(vm): @pytest.mark.asyncio -async def test_start_capture_rejected_on_kernel_link(vm): +async def test_start_capture_kernel_uses_af_packet(vm): + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) vm._ethernet_adapters[0].add_nio(0, nio) - with pytest.raises(DockerError, match="kernel-datapath"): + await vm.start_capture(0, "/tmp/capture.pcap") + + assert nio.capturing is True + vm._ubridge_send.assert_any_call(f'capture start_kernel {host_ifc} "/tmp/capture.pcap"') + + +@pytest.mark.asyncio +async def test_stop_capture_kernel(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + await vm.start_capture(0, "/tmp/capture.pcap") + + await vm.stop_capture(0) + + assert nio.capturing is False + vm._ubridge_send.assert_any_call("capture stop_kernel") + + +@pytest.mark.asyncio +async def test_start_capture_second_concurrent_fails(vm): + """ + uBridge's kernel capture is a singleton per process: a second concurrent + capture returns EALREADY and must surface to the caller. + """ + + async def send(command): + if "capture start_kernel" in command: + raise UbridgeError("Could not start kernel capture: Operation already in progress") + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + vm.status = "started" + vm._kernel_veths[(0, 0)] = vm._veth_names(0, 0)[0] + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + with pytest.raises(UbridgeError, match="already in progress"): await vm.start_capture(0, "/tmp/capture.pcap") +@pytest.mark.asyncio +async def test_connect_nio_kernel_restarts_capture(vm): + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + nio.start_packet_capture("/tmp/capture.pcap") + await vm._connect_nio(0, nio) + + vm._ubridge_send.assert_any_call(f'capture start_kernel {host_ifc} "/tmp/capture.pcap"') + + @pytest.mark.asyncio async def test_remove_kernel_veths(vm): From 61020d3bbc01043ea1e24c53feb3eb6f8df8e9d6 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 10:04:00 +0800 Subject: [PATCH 004/101] feat(docker): traffic-insight markers on kernel-datapath links via AF_PACKET Markers no longer disqualify the kernel datapath. They ride the NIO like on the relay path and attach to the veth host end through uBridge's marker add_kernel/delete_kernel/enable_kernel commands (AF_PACKET taps, same BPF/direction/signal semantics as the relay mark filter). - base_node: marker anchors are now polymorphic (relay bridge name or kernel veth host interface); the reconcile loop is datapath-agnostic and DockerVM overrides the add/delete/enable primitives to translate the commands. Marker-name validation extracted for both add paths. - docker_vm: kernel NIO connect/update apply markers on the veth anchor; link deletion tears markers down explicitly (the veth survives the link, unlike a relay bridge whose death drops its filters). - controller: eligibility and start_marker no longer exclude markers or project-level marker definitions; the kernel NIO carries the capture node's marker specs. - NIO creation accepts markers on kernel NIOs (filters still rejected). --- gns3server/compute/base_manager.py | 7 +- gns3server/compute/base_node.py | 78 ++++++++--- gns3server/compute/docker/docker_vm.py | 117 +++++++++++++++- gns3server/compute/nios/nio_bridge.py | 8 +- gns3server/controller/udp_link.py | 26 ++-- gns3server/schemas/compute/nios.py | 9 +- .../docker/test_docker_kernel_datapath.py | 131 +++++++++++++++++- tests/controller/test_kernel_datapath_link.py | 71 ++++++++-- 8 files changed, 378 insertions(+), 69 deletions(-) diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index e7cd790fb..7c4887727 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -361,8 +361,11 @@ class BaseManager: nio.suspend = nio_settings.get("suspend", False) elif nio_settings["type"] == "nio_bridge": nio = NIOBridge(nio_settings["bridge"]) - if nio_settings.get("filters") or nio_settings.get("markers"): - raise ComputeError("Packet filters and markers are not supported on kernel-datapath links") + if nio_settings.get("filters"): + raise ComputeError("Packet filters are not supported on kernel-datapath links") + # Markers ride the NIO and attach to the veth host interface via + # uBridge's AF_PACKET marker module (marker add_kernel). + nio.markers = nio_settings.get("markers", {}) nio.suspend = nio_settings.get("suspend", False) elif nio_settings["type"] == "nio_tap": tap_device = nio_settings["tap_device"] diff --git a/gns3server/compute/base_node.py b/gns3server/compute/base_node.py index 033ef96d9..74f6a028c 100644 --- a/gns3server/compute/base_node.py +++ b/gns3server/compute/base_node.py @@ -1172,6 +1172,24 @@ class BaseNode: dlt = dlt[4:] return None if dlt == "EN10MB" else dlt + @staticmethod + def _validate_marker_name(name): + """ + Defense-in-depth marker-name check shared by every add path (relay + ``mark`` filter and kernel ``marker add_kernel``). The name travels + from the controller REST layer (MarkerCreate schema) but is validated + here too against hand-edited topology files. Note: "global-*" names are + legitimate — they come from project-level marker definitions + (inherit_marker). The prefix is only forbidden at the user-facing + schema layer, not at the uBridge boundary. The user-facing name is + capped at 32 by the schema; inherited copies carry a ``global-`` + prefix (≤ 39), so allow up to 48 here. + """ + + _MARKER_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") + if not _MARKER_NAME_RE.match(name) or len(name) > 48: + raise UbridgeError(f"Invalid marker name: {name!r}") + async def _ubridge_add_marker_filter( self, bridge_name, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None ): @@ -1184,7 +1202,11 @@ class BaseNode: and it is added/removed on its own (not via reset_packet_filters) so the pcap is not closed/reopened on unrelated filter changes. - :param bridge_name: uBridge bridge carrying the link's traffic + ``bridge_name`` is the anchor the marker attaches to: the uBridge relay + bridge here, or the kernel veth host interface on the kernel datapath + (DockerVM overrides this to emit ``marker add_kernel``). + + :param bridge_name: anchor carrying the link's traffic :param name: stable, gns3server-chosen filter name (pcap identity + echoed in signals) :param bpf: libpcap BPF expression :param pcap_path: absolute path ubridge appends matched packets to @@ -1192,18 +1214,8 @@ class BaseNode: """ # mark [tag ] [pcap ] — tag/pcap keyword pairs, any order. - # name travels from the controller REST layer (MarkerCreate schema) but is - # validated here too as defense-in-depth against hand-edited topology files. - # Note: "global-*" names are legitimate here — they come from project-level - # marker definitions (inherit_marker). The prefix is only forbidden at the - # user-facing schema layer, not at the uBridge boundary. - _MARKER_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") - # Defense-in-depth vs hand-edited topology: the user-facing name is capped - # at 32 by the schema; inherited copies carry a ``global-`` prefix (≤ 39), - # so allow up to 48 here. - if not _MARKER_NAME_RE.match(name) or len(name) > 48: - raise UbridgeError(f"Invalid marker name: {name!r}") - cmd = f'bridge add_packet_filter {bridge_name} {name} mark "{bpf}"' + self._validate_marker_name(name) + cmd = 'bridge add_packet_filter {bridge} {name} mark "{bpf}"'.format(bridge=bridge_name, name=name, bpf=bpf) if tag is not None: cmd += f" tag {tag}" # Per-link attribution (contract §3.2): when one ubridge bridge serves @@ -1290,6 +1302,12 @@ class BaseNode: Reconcile the traffic-insight markers carried by *nio* onto bridge *bridge_name* with what is already installed there. + ``bridge_name`` is the marker anchor: the uBridge relay bridge on the + relay datapath, or the kernel veth host interface on the kernel + datapath (DockerVM passes the veth name; its add/delete/enable + primitive overrides translate the commands to ``marker *_kernel``). + The reconcile itself is datapath-agnostic. + uBridge's ``reset_packet_filters`` preserves mark filters (contract), so a plain re-add would duplicate them; instead this diffs the desired ``nio.markers`` against the installed ``_marker_specs``: @@ -1303,8 +1321,9 @@ class BaseNode: * desired and new → add Called from ``add_ubridge_udp_connection`` (fresh bridge, empty maps → - installs all) and ``update_ubridge_udp_connection`` / the batch NIO - update path (incremental reconcile). + installs all), ``update_ubridge_udp_connection`` / the batch NIO + update path (incremental reconcile), and DockerVM's kernel-datapath + connection paths (veth anchor). """ from gns3server.compute.marker.marker_manager import MarkerManager @@ -1381,12 +1400,13 @@ class BaseNode: raise # A disabled marker is installed but turned off (a paused tap), not # dropped — so the UI can flip it back on instantly with - # enable_packet_filter, no NIO rebuild (ubridge contract §3.2). + # enable_packet_filter / marker enable_kernel, no NIO rebuild + # (ubridge contract §3.2). if not enabled: try: - await self._ubridge_send(f"bridge enable_packet_filter {bridge_name} {name} off") + await self._ubridge_enable_marker_filter(bridge_name, name, "off") except UbridgeError as e: - # Old ubridge without enable_packet_filter: leave it installed + # Old ubridge without the enable command: leave it installed # (on) rather than fail the whole link/marker apply. log.warning(f"Could not turn marker '{name}' off on {bridge_name}: {e}") manager.register(str(self.project.id), self._id, name, link_id, tag) @@ -1396,12 +1416,26 @@ class BaseNode: self._marker_filter_bridges[name, link_id] = bridge_name self._marker_specs[name, link_id] = spec + async def _ubridge_enable_marker_filter(self, anchor, name, state): + """ + Flip one installed marker's on/off state with a single uBridge command + (``bridge enable_packet_filter … on|off``). ``anchor`` is the uBridge + relay bridge, or the kernel veth host interface on the kernel datapath + (DockerVM overrides this to emit ``marker enable_kernel``). + + :param anchor: relay bridge name or kernel veth host interface + :param name: marker filter name + :param state: "on" or "off" + """ + + await self._ubridge_send(f"bridge enable_packet_filter {anchor} {name} {state}") + async def _ubridge_set_marker_filter_state(self, name, enabled): """ Toggle an installed marker filter on/off with a single uBridge command - (``bridge enable_packet_filter … on|off``) — no NIO reset/reapply, so the - pcap identity and emitted counter are preserved (ubridge contract §3.2). - The bridge is resolved from the (name, link_id)→bridge map populated at + — no NIO reset/reapply, so the pcap identity and emitted counter are + preserved (ubridge contract §3.2). + The anchor is resolved from the (name, link_id)→bridge map populated at apply time; entries are iterated so a node that hosts the same marker name on several links (e.g. IOU with one IOL-BRIDGE per node) toggles every copy. IOU overrides this for its ``iol_bridge`` command shape. @@ -1413,7 +1447,7 @@ class BaseNode: state = "on" if enabled else "off" for (n, lid), bridge_name in list(self._marker_filter_bridges.items()): if n == name: - await self._ubridge_send(f"bridge enable_packet_filter {bridge_name} {name} {state}") + await self._ubridge_enable_marker_filter(bridge_name, name, state) async def _ubridge_marker_pause(self): """ diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index a3f98f96f..f70e37edd 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1823,9 +1823,9 @@ class DockerVM(BaseNode): whose host end will be enslaved to the per-link kernel bridge, and whose guest end becomes the container interface (the role the TAP plays on the relay path). No uBridge relay bridge is created — frames - flow entirely in the kernel. Filters and markers are unavailable on - this datapath (they live in the uBridge relay); packet capture is - provided by uBridge's AF_PACKET module on the veth host end. + flow entirely in the kernel. Filters are unavailable on this datapath + (they live in the uBridge relay); packet capture and markers are + provided by uBridge's AF_PACKET modules on the veth host end. """ try: @@ -1947,6 +1947,9 @@ class DockerVM(BaseNode): # Restore a capture that was active before a node restart # (mirrors the relay path's start_capture in _connect_nio). await self._ubridge_send(f'capture start_kernel {host_ifc} "{nio.pcap_output_file}"') + # Markers carried by the NIO attach to the veth host end (AF_PACKET + # taps) — the anchor is the interface, not a relay bridge. + await self._ubridge_apply_markers(host_ifc, nio) return bridge_name = self._bridge_name(adapter_number, port_number) @@ -2009,12 +2012,17 @@ class DockerVM(BaseNode): if self.ubridge: if isinstance(nio, NIOBridge): - if nio.filters or nio.markers: + if nio.filters: raise DockerError( - "Packet filters and markers are not supported on kernel-datapath links; " + "Packet filters are not supported on kernel-datapath links; " "delete and recreate the link to use them" ) if self.status == "started": + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if host_ifc is not None: + # Incremental marker reconcile on the veth anchor — + # the relay-datapath equivalent of the branch below. + await self._ubridge_apply_markers(host_ifc, nio) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) return @@ -2060,6 +2068,100 @@ class DockerVM(BaseNode): log.debug(f"Docker VM '{self.name}' [{self.id}]: {adapter.host_ifc} removed from adapter {adapter_number}") + def _kernel_marker_anchor(self, anchor): + """ + Whether *anchor* names one of this container's kernel-datapath veth + host ends. Marker anchors are polymorphic: a relay bridge name on the + relay datapath, the veth host interface on the kernel datapath. A veth + name can never collide with a uBridge bridge name (relay bridges are + ``bridge{adapter}``, veths are ``gv…e…p…``). + """ + + return anchor in self._kernel_veths.values() + + async def _ubridge_add_marker_filter( + self, bridge_name, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None + ): + """ + Kernel-datapath variant of the `mark` filter attach: ``marker + add_kernel`` binds an AF_PACKET socket to the veth host end. Keyword + pairs and semantics (match → MARK signal + pcap append) are identical + to the relay ``bridge add_packet_filter … mark`` command. + """ + + if self._kernel_marker_anchor(bridge_name): + self._validate_marker_name(name) + cmd = 'marker add_kernel {name} {ifc} "{bpf}"'.format(name=name, ifc=bridge_name, bpf=bpf) + if tag is not None: + cmd += f" tag {tag}" + if link_id: + cmd += f" link {link_id}" + if direction is not None: + cmd += f" dir {direction}" + linktype = self._marker_linktype(data_link_type) + if linktype is not None: + cmd += f" linktype {linktype}" + cmd += ' pcap "{path}"'.format(path=pcap_path) + await self._ubridge_send(cmd) + return + await super()._ubridge_add_marker_filter( + bridge_name, name, bpf, pcap_path, tag=tag, link_id=link_id, direction=direction, data_link_type=data_link_type + ) + + async def _ubridge_delete_marker_filter(self, bridge_name, name): + """ + Kernel-datapath variant of the marker removal: ``marker delete_kernel`` + (idempotent in uBridge — a no-op when the marker is already gone). + """ + + if self._kernel_marker_anchor(bridge_name): + if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): + return + try: + await self._ubridge_send(f"marker delete_kernel {bridge_name} {name}") + except UbridgeError as e: + log.warning("Could not remove kernel marker '%s' from %s: %s", name, bridge_name, e) + return + await super()._ubridge_delete_marker_filter(bridge_name, name) + + async def _ubridge_enable_marker_filter(self, anchor, name, state): + """ + Kernel-datapath variant of the marker on/off toggle: ``marker + enable_kernel`` — installed but silent when off, pcap preserved. + """ + + if self._kernel_marker_anchor(anchor): + await self._ubridge_send(f"marker enable_kernel {anchor} {name} {state}") + return + await super()._ubridge_enable_marker_filter(anchor, name, state) + + async def _remove_kernel_markers(self, host_ifc): + """ + Tear down every kernel marker anchored to *host_ifc*: the veth survives + link deletion (it is the adapter interface, not the link), so the + AF_PACKET sockets must be closed explicitly — otherwise a deleted + link's markers would keep sniffing and signaling. Mirrors the relay + datapath where deleting the uBridge bridge silently drops its filters. + """ + + from gns3server.compute.marker.marker_manager import MarkerManager + + manager = MarkerManager.instance() + markers_dir = self.project.markers_working_directory() + for (name, link_id), anchor in list(self._marker_filter_bridges.items()): + if anchor != host_ifc: + continue + self._marker_filter_bridges.pop((name, link_id)) + self._marker_specs.pop((name, link_id), None) + await self._ubridge_delete_marker_filter(host_ifc, name) + try: + os.remove(os.path.join(markers_dir, f"{self._id}_{link_id}_{name}.pcap")) + except FileNotFoundError: + pass + except OSError as e: + log.warning("Could not remove marker pcap for '%s' on link %s: %s", name, link_id, e) + manager.unregister(self._id, name) + async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): """ Detach a kernel-datapath adapter port from its per-link kernel bridge. @@ -2070,9 +2172,10 @@ class DockerVM(BaseNode): """ host_ifc = self._kernel_veths.get((adapter_number, port_number)) - if self.status == "started" and host_ifc is not None: - await self._set_adapter_carrier(adapter_number, False, port_number) if host_ifc is not None: + await self._remove_kernel_markers(host_ifc) + if self.status == "started": + await self._set_adapter_carrier(adapter_number, False, port_number) with contextlib.suppress(UbridgeError): await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') with contextlib.suppress(UbridgeError): diff --git a/gns3server/compute/nios/nio_bridge.py b/gns3server/compute/nios/nio_bridge.py index 6692752d1..785ac61e3 100644 --- a/gns3server/compute/nios/nio_bridge.py +++ b/gns3server/compute/nios/nio_bridge.py @@ -20,10 +20,10 @@ Interface for kernel-datapath bridge NIOs. A NIO of this type instructs the node to enslave its adapter's veth host end into the named Linux kernel bridge instead of wiring a uBridge UDP relay. -Frames then flow entirely in the kernel (veth -> bridge -> veth). Filters, -markers and packet capture are not available on this datapath (they live in -the uBridge relay); the controller must only emit this NIO type for links -where those features are not in use. +Frames then flow entirely in the kernel (veth -> bridge -> veth). Packet +filters are not available on this datapath (they live in the uBridge relay); +markers and packet capture are served by uBridge's AF_PACKET modules on the +veth host end (marker add_kernel / capture start_kernel). """ from .nio import NIO diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 27b2923a1..ea759a5a8 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -135,9 +135,9 @@ class UDPLink(Link): """ Whether this link can be wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge) instead of the uBridge UDP - relay. The kernel path has no userspace relay, so filters and - markers — which live in the relay — disqualify it (capture is - served separately by uBridge's AF_PACKET module). + relay. The kernel path has no userspace relay, so filters — which + live in the relay — disqualify it. Markers and capture are served + by uBridge's AF_PACKET modules on the veth host end. It is host-local (same compute) and requires the adapter interfaces to be created as veths at container start, hence stopped nodes only. """ @@ -150,12 +150,6 @@ class UDPLink(Link): return False if self.get_active_filters(): return False - if self._markers: - return False - # Project-level marker definitions are inherited by every link right - # after creation (apply_defs_to_new_link) and need the relay datapath. - if getattr(self._project, "_marker_definitions", None): - return False # Running nodes already have relay TAPs wired into uBridge; the # kernel path needs the veth variant created at container start. if node1.status != "stopped" or node2.status != "stopped": @@ -195,10 +189,14 @@ class UDPLink(Link): # both ends compute it independently (no coordinator) and crash # leftovers sweep deterministically; 11 hex chars fill IFNAMSIZ # (15), pushing name-collision probability to irrelevance. + # Markers ride the NIO like on the relay datapath, routed by + # capture node; they attach to the veth host end via uBridge's + # AF_PACKET marker module instead of a relay `mark` filter. bridge_name = "gns3" + self._id.replace("-", "")[:11] + node1_markers, node2_markers = self._get_node_markers(node1, node2) self._link_data = [ - {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": {}, "suspend": self._suspended}, - {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": {}, "suspend": self._suspended}, + {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": node1_markers, "suspend": self._suspended}, + {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": node2_markers, "suspend": self._suspended}, ] return [ (node1, adapter_number1, port_number1, self._link_data[0]), @@ -542,12 +540,6 @@ class UDPLink(Link): if name in self._markers: raise ControllerError(f"Marker '{name}' already exists on link {self._id}") - if self.kernel_datapath: - raise ControllerError( - "Markers are not supported on kernel-datapath links (no uBridge relay in the " - "forwarding path); delete and recreate the link to use markers" - ) - # Validate the BPF only for private per-link markers. An inherited copy # (``inherited_from`` set) fans out from a definition whose BPF was # already validated once at create/update (and on project load), so diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 4ae5a6475..6ee90c2a3 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -74,17 +74,16 @@ class BridgeNIO(BaseModel): """ Kernel-datapath bridge Network Input/Output properties. The NIO tells the node to enslave its veth host end into the named kernel bridge instead of - wiring a uBridge UDP relay. ``filters`` and ``markers`` must stay empty on - this datapath (rejected server-side); packet capture is supported via - uBridge's AF_PACKET module. Both are accepted as fields only because - link update payloads always carry the full NIO dictionary. + wiring a uBridge UDP relay. ``filters`` must stay empty on this datapath + (rejected server-side); markers and packet capture are served by uBridge's + AF_PACKET modules on the veth host end. """ type: BridgeNIOType bridge: str = Field(..., description="Kernel bridge name e.g. gns3a1b2c3d4e5") suspend: Optional[bool] = Field(None, description="Suspend the NIO") filters: Optional[dict] = Field(None, description="Packet filters (unsupported, must be empty)") - markers: Optional[dict] = Field(None, description="Traffic-insight markers (unsupported, must be empty)") + markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the veth host end)") class MarkerToggle(BaseModel): diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index e9437861c..e200590da 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -21,6 +21,7 @@ creation, per-link kernel bridge enslavement, carrier semantics, relay fallback and cleanup. """ +import os import uuid from unittest.mock import MagicMock, call @@ -78,10 +79,15 @@ def test_create_nio_bridge_rejects_filters(vm): vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"loss": [10]}}) -def test_create_nio_bridge_rejects_markers(vm): +def test_create_nio_bridge_accepts_markers(vm): + """ + Markers ride the kernel NIO (attached to the veth host end via uBridge's + AF_PACKET marker module); only filters are rejected. + """ - with pytest.raises(ComputeError): - vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "markers": {"m": {"bpf": "icmp"}}}) + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "markers": {"m": {"bpf": "icmp"}}}) + assert isinstance(nio, NIOBridge) + assert nio.markers == {"m": {"bpf": "icmp"}} # --------------------------------------------------------------------------- @@ -409,6 +415,125 @@ async def test_connect_nio_kernel_restarts_capture(vm): vm._ubridge_send.assert_any_call(f'capture start_kernel {host_ifc} "/tmp/capture.pcap"') +# --------------------------------------------------------------------------- +# Markers (AF_PACKET taps on the veth host end) +# --------------------------------------------------------------------------- + +def _marker_spec(bpf="icmp", tag=1, enabled=True, direction=None, link_id="l1"): + return {"bpf": bpf, "tag": tag, "enabled": enabled, "direction": direction, + "data_link_type": "DLT_EN10MB", "link_id": link_id} + + +@pytest.mark.asyncio +async def test_connect_nio_kernel_applies_markers(vm): + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "markers": {"icmp-m": _marker_spec()}}) + await vm._connect_nio(0, nio) + + pcap = os.path.join(vm.project.markers_working_directory(), f"{vm.id}_l1_icmp-m.pcap") + vm._ubridge_send.assert_any_call(f'marker add_kernel icmp-m {host_ifc} "icmp" tag 1 link l1 pcap "{pcap}"') + assert vm._marker_filter_bridges[("icmp-m", "l1")] == host_ifc + + +@pytest.mark.asyncio +async def test_connect_nio_kernel_marker_disabled_at_apply(vm): + """ + A disabled marker is installed but silenced (marker enable_kernel off) so + the UI can flip it back on instantly — same contract as the relay path. + """ + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + + nio = vm.manager.create_nio( + {"type": "nio_bridge", "bridge": BRIDGE, "markers": {"icmp-m": _marker_spec(enabled=False)}} + ) + await vm._connect_nio(0, nio) + + vm._ubridge_send.assert_any_call(f"marker enable_kernel {host_ifc} icmp-m off") + + +@pytest.mark.asyncio +async def test_update_nio_kernel_applies_markers(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + nio.markers = {"tcp-m": _marker_spec(bpf="tcp", tag=2, link_id="l2")} + await vm.adapter_update_nio_binding(0, nio) + + pcap = os.path.join(vm.project.markers_working_directory(), f"{vm.id}_l2_tcp-m.pcap") + vm._ubridge_send.assert_any_call(f'marker add_kernel tcp-m {host_ifc} "tcp" tag 2 link l2 pcap "{pcap}"') + + +@pytest.mark.asyncio +async def test_marker_toggle_kernel_uses_enable_kernel(vm): + + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + vm._marker_filter_bridges[("icmp-m", "l1")] = host_ifc + + await vm._ubridge_set_marker_filter_state("icmp-m", False) + vm._ubridge_send.assert_any_call(f"marker enable_kernel {host_ifc} icmp-m off") + await vm._ubridge_set_marker_filter_state("icmp-m", True) + vm._ubridge_send.assert_any_call(f"marker enable_kernel {host_ifc} icmp-m on") + + +@pytest.mark.asyncio +async def test_remove_kernel_nio_deletes_markers(vm): + """ + The veth survives link deletion (it is the adapter interface), so kernel + markers must be torn down explicitly — otherwise a deleted link's markers + would keep sniffing and signaling. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + vm._marker_filter_bridges[("icmp-m", "l1")] = host_ifc + vm._marker_specs[("icmp-m", "l1")] = _marker_spec() + vm._ethernet_adapters[0].add_nio(0, vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE})) + + await vm.adapter_remove_nio_binding(0) + + vm._ubridge_send.assert_any_call(f"marker delete_kernel {host_ifc} icmp-m") + assert ("icmp-m", "l1") not in vm._marker_filter_bridges + assert ("icmp-m", "l1") not in vm._marker_specs + + +@pytest.mark.asyncio +async def test_rebuild_marker_filter_kernel(vm): + """ + The fine-grained rebuild path (bpf/tag/direction change) goes through the + overridden primitives: delete_kernel + add_kernel with the new expression. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + vm._marker_filter_bridges[("icmp-m", "l1")] = host_ifc + + await vm.rebuild_marker_filter("icmp-m", "l1", "tcp", tag=9) + + pcap = os.path.join(vm.project.markers_working_directory(), f"{vm.id}_l1_icmp-m.pcap") + vm._ubridge_send.assert_any_call(f"marker delete_kernel {host_ifc} icmp-m") + vm._ubridge_send.assert_any_call(f'marker add_kernel icmp-m {host_ifc} "tcp" tag 9 link l1 pcap "{pcap}"') + + @pytest.mark.asyncio async def test_remove_kernel_veths(vm): diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index e3502cdca..755ad3bfa 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -21,7 +21,7 @@ Docker links, NIO emission and the filters/markers guards. """ import pytest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from gns3server.config import Config from gns3server.controller.udp_link import UDPLink @@ -96,20 +96,28 @@ async def test_kernel_datapath_not_eligible_with_filters(project): @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_with_markers(project): +async def test_kernel_datapath_eligible_with_markers(project): + """ + Markers no longer disqualify the kernel datapath: they attach to the veth + host end via uBridge's AF_PACKET marker module (marker add_kernel). + """ link, node1, node2 = await _kernel_link(project) link._markers = {"m": {"bpf": "icmp"}} - assert link._kernel_datapath_eligible(node1, node2) is False + assert link._kernel_datapath_eligible(node1, node2) is True @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_with_project_marker_definitions(project): +async def test_kernel_datapath_eligible_with_project_marker_definitions(project): + """ + Project-level marker definitions are inherited by every new link and ride + the kernel datapath like private markers. + """ link, node1, node2 = await _kernel_link(project) project._marker_definitions = {"global-m": {"bpf": "icmp"}} try: - assert link._kernel_datapath_eligible(node1, node2) is False + assert link._kernel_datapath_eligible(node1, node2) is True finally: project._marker_definitions = {} @@ -209,13 +217,58 @@ async def test_update_filters_rejected_on_kernel_link(project): @pytest.mark.asyncio -async def test_start_marker_rejected_on_kernel_link(project): +async def test_prepare_kernel_link_carries_markers(project): + """ + Markers on a kernel link ride the capture node's NIO (routed by + capture_node_id) exactly like on the relay datapath. + """ - link, _n1, _n2 = await _kernel_link(project) + link, node1, node2 = await _kernel_link(project) + link._markers = {"icmp": {"bpf": "icmp", "tag": 1, "capture_node_id": node1.id, "direction": None}} + + entries = await link._prepare() + + by_node = {entry[0].id: entry[3] for entry in entries} + assert by_node[node1.id]["type"] == "nio_bridge" + assert by_node[node1.id]["markers"]["icmp"]["bpf"] == "icmp" + assert by_node[node1.id]["markers"]["icmp"]["tag"] == 1 + assert by_node[node2.id]["markers"] == {} + + +@pytest.mark.asyncio +async def test_start_marker_on_kernel_link_pushes_nio(project): + """ + start_marker works on kernel links: the marker is stored and pushed via + the NIO update like on the relay datapath (no kernel guard anymore). + """ + + link, node1, node2 = await _kernel_link(project) await link._prepare() + link._created = True - with pytest.raises(ControllerError, match="kernel-datapath"): - await link.start_marker("m", "icmp") + bodies = [] + + async def put1(path, data=None, **kwargs): + bodies.append((node1, data)) + + async def put2(path, data=None, **kwargs): + bodies.append((node2, data)) + + node1.put = put1 + node2.put = put2 + + with patch("gns3server.controller.udp_link.validate_bpf_syntax", return_value={"valid": True, "error": None}): + await link.start_marker("icmp", "icmp", tag=3) + + entry = link._markers["icmp"] + assert entry["bpf"] == "icmp" + assert entry["capture_node_id"] == node1.id # auto-picked: first marker-capable node + n1_data = next(data for node, data in bodies if node is node1) + n2_data = next(data for node, data in bodies if node is node2) + assert n1_data["type"] == "nio_bridge" + assert n1_data["filters"] == {} + assert n1_data["markers"]["icmp"]["tag"] == 3 + assert n2_data["markers"] == {} @pytest.mark.asyncio From 672efda5923c1f3745770643bac6e10fc8428d02 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 14:37:41 +0800 Subject: [PATCH 005/101] feat(docker): unify adapter interfaces on veth, allow kernel links at runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Docker adapter is now born as a veth pair at container start (unified interface) regardless of NIO type: guest end in the container namespace, host end in the root namespace. Kernel links enslave the host end into the per-link Linux bridge; relay links attach it to the uBridge relay via AF_PACKET (bridge add_nio_ethernet) — the TAP path is gone. This makes the datapath a runtime decision: - links can be created on running containers (kernel or relay) without touching container interfaces — the controller stopped-node eligibility check is removed - project reopen re-prepares relay links and upgrades eligible ones to the kernel datapath - packet capture start/stop now key on the NIO type (not veth presence) so relay-over-veth captures use the relay capture, kernel ones the AF_PACKET module - vendor unix-socket containers keep their wiring: no veth is required on the relay path --- gns3server/compute/docker/docker_vm.py | 119 ++++++++---------- gns3server/controller/udp_link.py | 12 +- .../docker/test_docker_kernel_datapath.py | 44 ++++++- tests/compute/docker/test_docker_vm.py | 38 +++--- tests/compute/docker/test_vendor_docker_vm.py | 4 +- tests/controller/test_kernel_datapath_link.py | 9 +- 6 files changed, 121 insertions(+), 105 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index f70e37edd..b4c88f62d 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -170,9 +170,11 @@ class DockerVM(BaseNode): self._volumes = [] # Keep a list of created bridge self._bridges = set() - # (adapter_number, port_number) -> veth host-end name for adapters - # wired on the kernel datapath (NIOBridge). The host end lives in the - # root namespace and survives container death, unlike relay TAPs. + # (adapter_number, port_number) -> veth host-end name. Every adapter + # is born as a veth pair (unified interface): guest end in the + # container, host end in the root namespace where it survives + # container death. Kernel links enslave it into a Linux bridge, + # relay links attach it to the uBridge relay via AF_PACKET. self._kernel_veths = {} if adapters is None: @@ -1735,12 +1737,23 @@ class DockerVM(BaseNode): async def _add_ubridge_connection(self, nio, adapter_number, port_number=0): """ - Creates a connection in uBridge. + Create the adapter's interfaces at container start. - :param nio: NIO instance or None if it's a dummy interface (if an interface is missing in ubridge you can't see it via ifconfig in the container) + Every adapter is born as a veth pair — the unified interface — + regardless of NIO type: the guest end moves into the container + namespace as eth{N}, the host end stays in the root namespace. A + kernel-datapath NIO (NIOBridge) enslaves the host end into the + per-link Linux bridge; a relay NIO (UDP) attaches it to the uBridge + relay via AF_PACKET. The datapath is thus a runtime decision: any + link type can be attached to a running container without touching + its interfaces. + + :param nio: NIO instance, or None for an unconnected adapter — the + veth still exists (carrier off) so the interface shows up inside + the container :param adapter_number: adapter number :param port_number: port number on the adapter (standard Docker - adapters have a single port, so this is always 0 on the TAP path) + adapters have a single port, so this is always 0) """ try: @@ -1753,38 +1766,7 @@ class DockerVM(BaseNode): f"Port {port_number} doesn't exist on adapter {adapter_number} of Docker container '{self.name}'" ) - if isinstance(nio, NIOBridge): - await self._add_ubridge_kernel_connection(nio, adapter_number, port_number) - return - - for index in range(4096): - if f"tap-gns3-e{index}" not in psutil.net_if_addrs(): - adapter.host_ifc = f"tap-gns3-e{index!s}" - break - if adapter.host_ifc is None: - raise DockerError( - f"Adapter {adapter_number} couldn't allocate interface on Docker container '{self.name}'. Too many Docker interfaces already exists" - ) - bridge_name = self._bridge_name(adapter_number, port_number) - await self._ubridge_send(f"bridge create {bridge_name}") - self._bridges.add(bridge_name) - await self._ubridge_send(f"bridge add_nio_tap {bridge_name} {adapter.host_ifc} off") - - mac_address = self._adapter_mac_address(adapter_number) - - try: - await self._ubridge_send(f"docker set_mac_addr {adapter.host_ifc} {mac_address}") - except UbridgeError: - log.warning(f"Could not set MAC address {mac_address} on interface {adapter.host_ifc}") - - ifname = self._get_container_ifname(adapter_number) - log.debug(f"Move container {self.name} adapter {adapter.host_ifc} -> {ifname} in ns {self._namespace}") - try: - await self._ubridge_send(f"docker move_to_ns {adapter.host_ifc} {self._namespace} {ifname}") - except UbridgeError as e: - raise UbridgeNamespaceError(e) - else: - log.debug(f"Created adapter {adapter_number} with MAC address {mac_address} in namespace {self._namespace}") + await self._create_veth(adapter_number, port_number) if nio: await self._connect_nio(adapter_number, nio, port_number) @@ -1817,15 +1799,14 @@ class DockerVM(BaseNode): suffix = f"{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" return f"gv{suffix}", f"gc{suffix}" - async def _add_ubridge_kernel_connection(self, nio, adapter_number, port_number=0): + async def _create_veth(self, adapter_number, port_number=0): """ - Wire an adapter port on the kernel datapath (NIOBridge): a veth pair - whose host end will be enslaved to the per-link kernel bridge, and - whose guest end becomes the container interface (the role the TAP - plays on the relay path). No uBridge relay bridge is created — frames - flow entirely in the kernel. Filters are unavailable on this datapath - (they live in the uBridge relay); packet capture and markers are - provided by uBridge's AF_PACKET modules on the veth host end. + Create the adapter's unified veth pair: the guest end moves into the + container namespace as eth{N} (the role the TAP used to play on the + relay path), the host end stays in the root namespace with carrier + off until a link attaches. Both datapaths anchor here — kernel links + enslave the host end into a Linux bridge; relay links attach it to + the uBridge relay via AF_PACKET. """ try: @@ -1849,7 +1830,7 @@ class DockerVM(BaseNode): try: await self._ubridge_send(f'docker create_veth "{host_ifc}" "{guest_ifc}"') # create_veth leaves the host end UP; carrier must be off until - # a link attaches (mirrors "bridge add_nio_tap ... off"). + # a link attaches (mirrors the old "bridge add_nio_tap ... off"). await self._ubridge_send(f'link set "{host_ifc}" down') try: await self._ubridge_send(f"docker set_mac_addr {guest_ifc} {mac_address}") @@ -1874,16 +1855,12 @@ class DockerVM(BaseNode): adapter.host_ifc = host_ifc self._kernel_veths[(adapter_number, port_number)] = host_ifc log.debug( - "Created kernel-datapath adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( + "Created veth adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( adapter_number=adapter_number, port_number=port_number, mac_address=mac_address, namespace=self._namespace, ) ) - if nio: - await self._connect_nio(adapter_number, nio, port_number) - await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) - async def _remove_kernel_veths(self): """ Delete the root-namespace veth host ends. Unlike relay TAPs (which die @@ -1923,12 +1900,16 @@ class DockerVM(BaseNode): async def _connect_nio(self, adapter_number, nio, port_number=0): + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if isinstance(nio, NIOBridge): - host_ifc = self._kernel_veths.get((adapter_number, port_number)) if host_ifc is None: + # Only reachable for containers started before the + # unified-veth change (or unix-socket vendor containers, + # rejected earlier): the kernel path needs the veth host end. raise DockerError( - "Adapter {adapter_number} port {port_number} of container '{name}' has no " - "kernel-datapath interface (it was started on the relay datapath); " + "Adapter {adapter_number} port {port_number} of container '{name}' has no veth " + "interface (it was started before the unified-veth datapath); " "restart the node to attach a kernel link to it".format( adapter_number=adapter_number, port_number=port_number, name=self.name ) @@ -1952,18 +1933,17 @@ class DockerVM(BaseNode): await self._ubridge_apply_markers(host_ifc, nio) return + # Relay NIO. On a veth-backed adapter (the unified Docker interface) + # the uBridge bridge reads/writes the veth host end via AF_PACKET + # (bridge add_nio_ethernet) instead of owning a TAP fd. Vendor + # containers with unix-socket wiring have no veth: their bridge and + # its unix NIO already exist (added by their _add_ubridge_connection + # override), so only the UDP half is wired here. bridge_name = self._bridge_name(adapter_number, port_number) - if (adapter_number, port_number) in self._kernel_veths and bridge_name not in self._bridges: - # A veth adapter attaching a UDP NIO (kernel link deleted and a - # relay link created on the running node): relay through AF_PACKET - # on the veth host end instead of a TAP. + if host_ifc is not None and bridge_name not in self._bridges: await self._ubridge_send(f"bridge create {bridge_name}") self._bridges.add(bridge_name) - await self._ubridge_send( - 'bridge add_nio_ethernet {bridge_name} "{hostif}"'.format( - bridge_name=bridge_name, hostif=self._kernel_veths[(adapter_number, port_number)] - ) - ) + await self._ubridge_send(f'bridge add_nio_ethernet {bridge_name} "{host_ifc}"') await self._ubridge_send( "bridge add_nio_udp {bridge_name} {lport} {rhost} {rport}".format( bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport @@ -2242,13 +2222,17 @@ class DockerVM(BaseNode): if not self.ubridge: raise DockerError("Cannot start the packet capture: uBridge is not running") - host_ifc = self._kernel_veths.get((adapter_number, port_number)) - if host_ifc is not None: + nio = self._ethernet_adapters[adapter_number].get_nio(port_number) + if isinstance(nio, NIOBridge): # Kernel datapath: no relay bridge exists — capture via uBridge's # AF_PACKET module bound to the veth host end. Single capture per # uBridge process: a concurrent second one returns EALREADY. + host_ifc = self._kernel_veths.get((adapter_number, port_number)) await self._ubridge_send(f'capture start_kernel {host_ifc} "{output_file}"') return + # Relay datapath (even though the adapter interface is a veth, the + # traffic flows through the uBridge bridge — key on the NIO type, + # not on the veth presence): capture at the relay. bridge_name = self._bridge_name(adapter_number, port_number) await self._ubridge_send(f'bridge start_capture {bridge_name} "{output_file}"') @@ -2262,7 +2246,8 @@ class DockerVM(BaseNode): if not self.ubridge: raise DockerError("Cannot stop the packet capture: uBridge is not running") - if (adapter_number, port_number) in self._kernel_veths: + nio = self._ethernet_adapters[adapter_number].get_nio(port_number) + if isinstance(nio, NIOBridge): # Idempotent on the uBridge side (no active capture is OK). await self._ubridge_send("capture stop_kernel") return diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index ea759a5a8..4d02cf9fd 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -138,8 +138,12 @@ class UDPLink(Link): relay. The kernel path has no userspace relay, so filters — which live in the relay — disqualify it. Markers and capture are served by uBridge's AF_PACKET modules on the veth host end. - It is host-local (same compute) and requires the adapter interfaces - to be created as veths at container start, hence stopped nodes only. + + Docker adapters are born as veth pairs (unified interface), so the + datapath is a runtime decision — links attach to running containers + too: the compute side enslaves the veth host end (brctl) or attaches + the relay to it (add_nio_ethernet) without touching the container's + interfaces. """ if not Config.instance().settings.Server.enable_kernel_datapath: @@ -150,10 +154,6 @@ class UDPLink(Link): return False if self.get_active_filters(): return False - # Running nodes already have relay TAPs wired into uBridge; the - # kernel path needs the veth variant created at container start. - if node1.status != "stopped" or node2.status != "stopped": - return False if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): return False return True diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index e200590da..f09ba6591 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -125,10 +125,11 @@ async def test_add_ubridge_kernel_connection(vm): @pytest.mark.asyncio -async def test_add_ubridge_connection_none_nio_stays_on_tap_path(vm): +async def test_add_ubridge_connection_none_nio_creates_veth(vm): """ - Unconnected adapters keep the relay TAP: the veth variant is only created - for adapters whose NIO is a kernel one at start time. + Unconnected adapters are born as veths too (the unified interface): the + interface exists inside the container with carrier off, and any link + type can be attached later — at runtime — without rebirthing it. """ vm._ubridge_hypervisor = MagicMock() @@ -136,11 +137,42 @@ async def test_add_ubridge_connection_none_nio_stays_on_tap_path(vm): await vm._add_ubridge_connection(None, 0) + host_ifc, guest_ifc = vm._veth_names(0, 0) vm._ubridge_hypervisor.assert_has_calls([ - call.send("bridge create bridge0"), - call.send("bridge add_nio_tap bridge0 tap-gns3-e0 off"), + call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), + call.send(f'link set "{host_ifc}" down'), + call.send(f'docker move_to_ns {guest_ifc} 42 eth0'), ], any_order=True) - assert vm._kernel_veths == {} + assert vm._kernel_veths[(0, 0)] == host_ifc + # no relay bridge and no TAP for an unconnected adapter + assert "bridge0" not in vm._bridges + for c in vm._ubridge_hypervisor.method_calls: + assert "add_nio_tap" not in str(c) + + +@pytest.mark.asyncio +async def test_add_ubridge_connection_udp_relays_over_veth(vm): + """ + A relay NIO at start: the adapter is born as a veth and the uBridge + relay attaches to its host end via AF_PACKET (add_nio_ethernet) — + no TAP anywhere. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) + await vm._add_ubridge_connection(nio, 0) + + host_ifc, _ = vm._veth_names(0, 0) + vm._ubridge_hypervisor.assert_has_calls([ + call.send(f'docker create_veth "{host_ifc}" "{vm._veth_names(0, 0)[1]}"'), + call.send("bridge create bridge0"), + call.send(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), + call.send("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), + call.send("bridge start bridge0"), + ], any_order=True) + assert "bridge0" in vm._bridges @pytest.mark.asyncio diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index c98998a81..51a1e4176 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -1541,14 +1541,17 @@ async def test_add_ubridge_connection(vm): vm._namespace = 42 await vm._add_ubridge_connection(nio, 0) + host_ifc, guest_ifc = vm._veth_names(0, 0) calls = [ + call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), + call.send(f'link set "{host_ifc}" down'), + call.send(f"docker move_to_ns {guest_ifc} 42 eth0"), call.send("bridge create bridge0"), - call.send("bridge add_nio_tap bridge0 tap-gns3-e0 off"), - call.send("docker move_to_ns tap-gns3-e0 42 eth0"), + call.send(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), call.send("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), call.send('bridge start_capture bridge0 "/tmp/capture.pcap"'), call.send("bridge start bridge0"), - call.send("bridge set_nio_tap_carrier bridge0 on"), + call.send(f'link set "{host_ifc}" up'), ] assert "bridge0" in vm._bridges # We need to check any_order otherwise mock is confused by asyncio @@ -1571,11 +1574,13 @@ async def test_add_ubridge_connections_with_base_mac_address(vm): nio = vm.manager.create_nio(nio_params) await vm._add_ubridge_connection(nio, 1) + _, guest0 = vm._veth_names(0, 0) + _, guest1 = vm._veth_names(1, 0) calls = [ call.send("bridge create bridge0"), call.send("bridge create bridge1"), - call.send("docker set_mac_addr tap-gns3-e0 02:42:42:42:42:00"), - call.send("docker set_mac_addr tap-gns3-e0 02:42:42:42:42:01"), + call.send(f"docker set_mac_addr {guest0} 02:42:42:42:42:00"), + call.send(f"docker set_mac_addr {guest1} 02:42:42:42:42:01"), ] # We need to check any_order otherwise mock is confused by asyncio @@ -1591,12 +1596,14 @@ async def test_add_ubridge_connection_none_nio(vm): await vm._add_ubridge_connection(nio, 0) + host_ifc, guest_ifc = vm._veth_names(0, 0) calls = [ - call.send("bridge create bridge0"), - call.send("bridge add_nio_tap bridge0 tap-gns3-e0 off"), - call.send("docker move_to_ns tap-gns3-e0 42 eth0"), + call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), + call.send(f'link set "{host_ifc}" down'), + call.send(f"docker move_to_ns {guest_ifc} 42 eth0"), ] - assert "bridge0" in vm._bridges + # unconnected adapter: interface exists (carrier off), no relay bridge + assert "bridge0" not in vm._bridges # We need to check any_order ortherwise mock is confused by asyncio vm._ubridge_hypervisor.assert_has_calls(calls, any_order=True) @@ -1626,19 +1633,6 @@ async def test_add_ubridge_connection_invalid_adapter_number(vm): await vm._add_ubridge_connection(nio, 12) -@pytest.mark.asyncio -async def test_add_ubridge_connection_no_free_interface(vm): - - nio = {"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"} - nio = vm.manager.create_nio(nio) - with pytest.raises(DockerError): - # We create fake ethernet interfaces for docker - interfaces = [f"tap-gns3-e{index}" for index in range(4096)] - - with patch("psutil.net_if_addrs", return_value=interfaces): - await vm._add_ubridge_connection(nio, 0) - - @pytest.mark.asyncio async def test_adapter_add_nio_binding_1(vm): diff --git a/tests/compute/docker/test_vendor_docker_vm.py b/tests/compute/docker/test_vendor_docker_vm.py index 93ef04905..f508bcf29 100644 --- a/tests/compute/docker/test_vendor_docker_vm.py +++ b/tests/compute/docker/test_vendor_docker_vm.py @@ -314,7 +314,7 @@ async def test_move_to_ns_uses_renamed_interface(compute_project, manager): # adapter 0 should be renamed to mgmt0 move_calls = [c for c in vm._ubridge_hypervisor.method_calls if "move_to_ns" in str(c)] assert move_calls, "move_to_ns was not sent" - assert call.send("docker move_to_ns tap-gns3-e0 42 mgmt0") in move_calls + assert call.send(f"docker move_to_ns {vm._veth_names(0, 0)[1]} 42 mgmt0") in move_calls @pytest.mark.asyncio @@ -326,7 +326,7 @@ async def test_move_to_ns_falls_back_to_eth(compute_project, manager): nio = manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) await vm._add_ubridge_connection(nio, 1) move_calls = [c for c in vm._ubridge_hypervisor.method_calls if "move_to_ns" in str(c)] - assert call.send("docker move_to_ns tap-gns3-e0 42 eth1") in move_calls + assert call.send(f"docker move_to_ns {vm._veth_names(1, 0)[1]} 42 eth1") in move_calls # --------------------------------------------------------------------------- diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 755ad3bfa..b47532de6 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -123,11 +123,16 @@ async def test_kernel_datapath_eligible_with_project_marker_definitions(project) @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_for_running_nodes(project): +async def test_kernel_datapath_eligible_for_running_nodes(project): + """ + Docker adapters are born as veths (unified interface), so links attach + to running containers too — no stopped-node requirement. + """ link, node1, node2 = await _kernel_link(project) node1._status = "started" - assert link._kernel_datapath_eligible(node1, node2) is False + node2._status = "started" + assert link._kernel_datapath_eligible(node1, node2) is True @pytest.mark.asyncio From 94b5495683969a3c426893811a5dec677e80ef7d Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 15:09:03 +0800 Subject: [PATCH 006/101] docs: Docker kernel datapath implementation document Cover the four-stage stack (kernel datapath, AF_PACKET capture, markers, veth-everywhere): adapter interface types per node class, datapath selection, veth lifecycle, link operations, suspend/capture/markers semantics, uBridge command surface and the tc netem filters roadmap. Cross-link from the traffic-insight marker doc for the kernel variant. --- docs/features/docker-kernel-datapath.md | 198 ++++++++++++++++++++++++ docs/features/marker-traffic-insight.md | 8 + 2 files changed, 206 insertions(+) create mode 100644 docs/features/docker-kernel-datapath.md diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md new file mode 100644 index 000000000..f3aa181e0 --- /dev/null +++ b/docs/features/docker-kernel-datapath.md @@ -0,0 +1,198 @@ + + +> This documentation is AI-generated with reference to actual code and verified +> against real-kernel testing. AI can make mistakes — please verify against +> the source code when in doubt. + +# Docker Kernel Datapath (veth + per-link Linux bridge) + +## Overview + +Docker node links historically flowed through a uBridge userspace relay: each +adapter was a TAP whose file descriptor uBridge pumped between the container +and UDP tunnels. This implementation replaces that model with **kernel-native +forwarding**: every adapter is born as a **veth pair**, and each link is a +**dedicated Linux kernel bridge** enslaving the two endpoints' veth host ends. +Frames never leave the kernel. + +| | Old (relay) | New (kernel datapath) | +|---|---|---| +| Adapter interface | TAP moved into the container netns | veth pair (guest end in container, host end in root ns) | +| Link | uBridge bridge, TAP fd ↔ UDP, userspace copy | per-link kernel bridge `gns3{link_id[:11]}`, zero-copy | +| Relay latency | ~0.3 ms RTT | ~0.05 ms RTT (measured) | +| Link attach at runtime | yes (relay attaches the pre-existing TAP) | yes (brctl addif / add_nio_ethernet — interface untouched) | +| Packet filters | uBridge userspace filters | not supported yet (tc netem roadmap) → relay fallback | +| Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | + +The work landed in four stages on branch stack `feat/docker-kernel-datapath` → +`feat/docker-kernel-capture` → `feat/docker-kernel-markers` → +`feat/docker-veth-everywhere`: + +1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based + suspend, crash-safe reconciliation. +2. **Capture** — `capture start_kernel` (AF_PACKET) on the veth host end. +3. **Markers** — `marker add_kernel` (AF_PACKET + BPF) with the same signals, + pcaps and fine-grained REST operations as the relay `mark` filter. +4. **veth-everywhere** — the TAP path deleted; every adapter is a veth and the + datapath is a *runtime* decision (kernel or relay) that never touches a + running container's interfaces. + +## Adapter interface types + +Which interface a Docker node's adapters use is decided by the node class +(`Docker._select_node_class`, from `console_type` + `GNS3_*` environment +markers): + +| Node class | Selected by | Adapter interface | Kernel links | +|---|---|---|---| +| `DockerVM` | default | veth pair | yes | +| `VendorDockerVM` (non-unix-socket) | `console_type=docker_exec`, `GNS3_SKIP_INIT`, … (XRd, SR Linux, …) | veth pair (same path via `super()`) | yes | +| `IOLDockerVM` | `GNS3_IOL_RUNNER=1` | AF_UNIX datagram socket pairs (`sNN`/`cNN`) — the container netns is unused | rejected (no host-side interface) | +| `VendorDockerVM` (unix-socket) | `GNS3_UNIX_SOCKET_NIO=1` | same socket contract, generic capability for vendor NOS images | rejected | + +Unix-socket containers are detected both in the controller +(`_is_unix_socket_docker`, link eligibility) and on the compute side (NIO +rejection) — a missed detection surfaces as a clearer-late error. + +## Architecture + +Kernel link between two containers on the same compute: + +``` + container A netns root namespace container B netns + ┌────────────┐ veth pair ┌───────────────────┐ veth pair ┌────────────┐ + │ eth0 ◄─────┼───────────────┼─► gv{id}e0p0 ──────┼──► gns3{lid} │──────────► │ eth0 + └────────────┘ gc{id}e0p0 │ (bridge port) │ (bridge port)│ └────────────┘ + │ per-link kernel │ gv{id}e0p0 + │ bridge = "cable" │ + └───────────────────┘ +``` + +Relay link on the same unified veth (fallback for filtered links or when +`enable_kernel_datapath` is off — e.g. to reach another compute): + +``` + ┌────────────┐ ┌────────────────────────────┐ ┌────────────┐ + │ eth0 │ veth │ uBridge relay bridge │ UDP │ peer node │ + │ (gc end) │◄────────────►│ add_nio_ethernet (AF_ │◄────────────►│ (any type, │ + └────────────┘ gv host end │ PACKET) ↔ add_nio_udp │ tunnel │ any host) │ + └────────────────────────────┘ └────────────┘ +``` + +Both datapaths anchor on the *same* veth host end; switching between them +(link delete + recreate) never touches the container-side interface. + +## Datapath selection + +`UDPLink._kernel_datapath_eligible` decides in the controller at NIO prepare +time; the compute side only reacts to the NIO type: + +* both endpoints Docker (any class except unix-socket containers), same + compute +* no active packet filters (filters live in the relay; kernelization via tc + netem is planned) +* `Server.enable_kernel_datapath` enabled (default) + +There is **no stopped-node requirement**. Since every adapter is a veth, links +attach to running containers: `brctl addif` (kernel) or +`bridge add_nio_ethernet` (relay) are runtime-safe operations on the host end. +On project (re)open every link re-runs `_prepare`, so relay links whose +endpoints became eligible are upgraded to the kernel datapath automatically. + +## Adapter lifecycle + +* **Birth (container start)** — `_create_veth`: stale sweep of any leftover + pair (crash residue), `docker create_veth`, host end admin-down, MAC from + the adapter base, `docker move_to_ns` of the guest end (renamed `eth{N}`, or + the `GNS3_INTERFACE_NAMES` mapping). Names are deterministic: + `gv|gc{node_id[:8]}e{adapter}p{port}` (≤ 15 chars). Unconnected adapters are + born too (carrier off) so the interface is visible inside the container. +* **Life (running)** — the veth is never created/deleted/moved again. Link + create/delete/switch, suspend, capture and markers only change *what is + attached to the host end*. +* **Death (container stop)** — `_remove_kernel_veths` deletes the host ends + explicitly: unlike a TAP (which died with the container netns), a veth host + end outlives the container. Deleting either end destroys the pair. + +## Link operations + +| Operation | Kernel-datapath action | +|---|---| +| Create / attach | `brctl create` (EEXIST tolerated via `brctl show` verify) + `link set up` + `brctl addif` both ends (concurrent, race-tolerant) + carrier up | +| Delete / detach | marker teardown → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The veth survives — unlike a relay bridge, whose death dropped its filters, an orphaned AF_PACKET marker would keep sniffing, hence the explicit teardown | +| Reset (`POST /links/{id}/reset`) | delete + create (re-evaluates eligibility) | +| Suspend (`PUT /links/{id}` `{"suspend": true}`) | veth host end admin-state down both ends — 100 % loss, no synthetic filter needed; resume restores | + +## Capture + +Kernel links capture via uBridge's AF_PACKET module bound to the veth host +end: `capture start_kernel "" [dlt]` / `capture stop_kernel`. +Single capture per uBridge process (second concurrent → EALREADY). Start/stop +key on the **NIO type**, not on veth presence — a relay NIO riding a veth +captures at its relay bridge. + +## Markers + +Markers ride the NIO like on the relay datapath, routed by `capture_node_id`; +the anchor is the veth host end instead of a relay bridge +(`_ubridge_apply_markers(anchor, nio)` is datapath-agnostic; DockerVM +overrides the add/delete/enable primitives to translate the commands): + +``` +marker add_kernel "" [tag ] [link ] [dir ] [linktype ] [pcap ""] +marker delete_kernel # idempotent +marker enable_kernel # off = installed but silent +``` + +* Per-deployment convention: **only the capture node's end** runs + `add_kernel` (symmetric with capture) — signals form clean send/return + pairs (`PACKET_OUTGOING` on the host end = container receiving = `rx`). +* Same BPF compilation, MARK UDP signals, pcap files and fine-grained REST + toggle/rebuild/delete endpoints as the relay `mark` filter. +* Node restart reinstalls markers from the NIO; link deletion tears them down. + +## Packet filters + +Not supported on the kernel datapath yet: a link with active filters is wired +on the relay (where uBridge's userspace filters apply), and adding filters to +an existing kernel link returns 409. Kernelization via `tc netem` on the veth +is the planned next stage — it also removes the relay's `delay` filter +nanosleep bottleneck under high packet rates. + +## uBridge command surface + +No uBridge changes were needed beyond the marker module's `*_kernel` commands +(the AF_PACKET capture module was pre-existing): + +``` +docker create_veth / delete_veth / move_to_ns / set_mac_addr +brctl create / delete / addif / delif / show +link set up|down +capture start_kernel / stop_kernel +marker add_kernel / delete_kernel / enable_kernel +bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capture +``` + +## Configuration + +```ini +[Server] +# Wire eligible Docker-to-Docker links through kernel veth/bridge interfaces +enable_kernel_datapath = True +``` + +Intended to grow into the global datapath switch as QEMU / IOU / Dynamips +migrate to kernel bridges (QEMU: tap enslaved per-link bridge, zero relay). + +## Verification + +End-to-end on a five-container FRR topology (mixed runtime-drawn and reloaded +links): 8/8 links on the kernel datapath, ping RTT ≈ 0.05 ms, FDB learning, +suspend = 100 % loss / resume restores, project reopen upgrades relay links, +runtime link creation on running containers, marker pcaps exact (tx/rx +pairs), capture freeze on stop, server restart reconciliation. Unit tests: +`tests/compute/docker/test_docker_kernel_datapath.py`, +`tests/controller/test_kernel_datapath_link.py`. diff --git a/docs/features/marker-traffic-insight.md b/docs/features/marker-traffic-insight.md index ab71cbf65..4beb725c2 100644 --- a/docs/features/marker-traffic-insight.md +++ b/docs/features/marker-traffic-insight.md @@ -46,6 +46,14 @@ unchanged. Each compute process runs one UDP listener serving every uBridge on t `node` and `link` fields in each signal together identify the source link (see [Per-link attribution](#per-link-attribution)). +On **kernel-datapath** links (see +[docker-kernel-datapath.md](docker-kernel-datapath.md)) there is no relay +bridge: the marker attaches directly to the veth host end via uBridge's +AF_PACKET marker module (`marker add_kernel`), with identical BPF matching, +signals, pcaps and REST operations. Deployment convention: only the capture +node's end carries the marker, so each exchange produces one clean +`tx`/`rx` signal pair. + ## Business Process ```mermaid From 0a8d430d503b9f0771fd3f0616a7e47744765ba5 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 20:07:21 +0800 Subject: [PATCH 007/101] feat(docker): packet filters on kernel-datapath links via tc netem Impairment filters with a netem equivalent (delay, packet_loss, corrupt) now run on the kernel datapath as one tc netem qdisc per veth host end, pushed through uBridge's netlink tc module (tc netem set / tc reset). This removes the relay delay filter's nanosleep bottleneck (upstream Both link endpoints carry the filters: each end's qdisc egress covers the traffic entering that container, so every direction is impaired exactly once -- the same net effect as the relay, where both directions cross the single filtered bridge. - controller: eligibility keys on relay-only types (frequency_drop, bpf) instead of any filter; kernel links push the real filters (the synthetic suspend frequency_drop no longer flips a suspended link to relay on reopen/reset); update_filters 409s only relay-only types; available_filters hides them on kernel links - compute: create_nio whitelists netem-capable filter types; the NIO update path keeps its own guard (PUT mutates the NIO in place); _ubridge_apply_netem translates the filter dict into one atomic netem-set (full re-apply IS the reconcile); node restart restores the qdisc, link deletion detaches it (the veth survives the link) - e2e (27/27): delay 100 -> 200.2 ms RTT, atomic replace 50 -> 100.2 ms, clear -> baseline, packet_loss 30 -> 48 % round-trip (expected 51 %), suspend/resume and node restart keep the qdisc, link delete leaves no residual impairment --- docs/features/docker-kernel-datapath.md | 69 +++++- gns3server/compute/base_manager.py | 14 +- gns3server/compute/docker/docker_vm.py | 75 +++++- gns3server/compute/nios/nio_bridge.py | 5 +- gns3server/controller/link.py | 29 ++- gns3server/controller/udp_link.py | 66 ++++-- gns3server/schemas/compute/nios.py | 11 +- gns3server/utils/packet_filter_validation.py | 10 +- .../docker/test_docker_kernel_datapath.py | 213 ++++++++++++++---- tests/controller/test_kernel_datapath_link.py | 60 ++++- tests/controller/test_udp_link.py | 2 +- 11 files changed, 459 insertions(+), 95 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index f3aa181e0..21c95f352 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -24,12 +24,12 @@ Frames never leave the kernel. | Link | uBridge bridge, TAP fd ↔ UDP, userspace copy | per-link kernel bridge `gns3{link_id[:11]}`, zero-copy | | Relay latency | ~0.3 ms RTT | ~0.05 ms RTT (measured) | | Link attach at runtime | yes (relay attaches the pre-existing TAP) | yes (brctl addif / add_nio_ethernet — interface untouched) | -| Packet filters | uBridge userspace filters | not supported yet (tc netem roadmap) → relay fallback | +| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt); frequency_drop & bpf stay relay-only | | Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | -The work landed in four stages on branch stack `feat/docker-kernel-datapath` → +The work landed in five stages on branch stack `feat/docker-kernel-datapath` → `feat/docker-kernel-capture` → `feat/docker-kernel-markers` → -`feat/docker-veth-everywhere`: +`feat/docker-veth-everywhere` → `feat/docker-kernel-filters`: 1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based suspend, crash-safe reconciliation. @@ -39,6 +39,10 @@ The work landed in four stages on branch stack `feat/docker-kernel-datapath` → 4. **veth-everywhere** — the TAP path deleted; every adapter is a veth and the datapath is a *runtime* decision (kernel or relay) that never touches a running container's interfaces. +5. **Filters** — impairment filters (delay, packet_loss, corrupt) become one + tc netem qdisc per veth host end, pushed through uBridge's netlink `tc` + module. This also removes the relay delay filter's nanosleep bottleneck + (upstream #2827) from the kernel path. ## Adapter interface types @@ -121,10 +125,10 @@ endpoints became eligible are upgraded to the kernel datapath automatically. | Operation | Kernel-datapath action | |---|---| -| Create / attach | `brctl create` (EEXIST tolerated via `brctl show` verify) + `link set up` + `brctl addif` both ends (concurrent, race-tolerant) + carrier up | -| Delete / detach | marker teardown → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The veth survives — unlike a relay bridge, whose death dropped its filters, an orphaned AF_PACKET marker would keep sniffing, hence the explicit teardown | +| Create / attach | `brctl create` (EEXIST tolerated via `brctl show` verify) + `link set up` + `brctl addif` both ends (concurrent, race-tolerant) + carrier up + markers + `tc netem set` (filters) | +| Delete / detach | marker teardown → `tc reset` (netem teardown — the veth survives the link) → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The veth survives — unlike a relay bridge, whose death dropped its filters, an orphaned AF_PACKET marker would keep sniffing, hence the explicit teardown | | Reset (`POST /links/{id}/reset`) | delete + create (re-evaluates eligibility) | -| Suspend (`PUT /links/{id}` `{"suspend": true}`) | veth host end admin-state down both ends — 100 % loss, no synthetic filter needed; resume restores | +| Suspend (`PUT /links/{id}` `{"suspend": true}`) | veth host end admin-state down both ends — 100 % loss, no synthetic filter needed; resume restores (netem qdisc survives the carrier flap) | ## Capture @@ -156,11 +160,38 @@ marker enable_kernel # off = installed but silent ## Packet filters -Not supported on the kernel datapath yet: a link with active filters is wired -on the relay (where uBridge's userspace filters apply), and adding filters to -an existing kernel link returns 409. Kernelization via `tc netem` on the veth -is the planned next stage — it also removes the relay's `delay` filter -nanosleep bottleneck under high packet rates. +Impairment filters run **in the kernel** as tc netem qdiscs on the veth host +ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): + +| GNS3 filter | netem mapping | Notes | +|---|---|---| +| `delay [ms, jitter]` | `delay X jitter Y` | jitter 0 omitted | +| `packet_loss [%]` | `loss P` | per direction (see below) | +| `corrupt [%]` | `corrupt P` | | +| `frequency_drop` | — no netem equivalent | 409 on kernel links; relay fallback | +| `bpf` | — needs a tc classifier | 409 on kernel links; relay fallback | + +Semantics: + +* **Both endpoints** receive the filter dict and attach one qdisc to *their* + veth host end. The qdisc's egress covers traffic entering that container, + so every direction of the link is impaired exactly once — the same net + effect as the relay, where both directions cross the single filtered + bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` + measures ≈51 % round-trip (1 − 0.7²). +* **Reconcile = full re-apply.** `netem set` is an atomic replace (NLM_F_REPLACE), + so every NIO update just rebuilds the qdisc from the current filters; no + per-parameter diffing. An empty filter set detaches it (`tc reset`, + ENOENT-tolerated). +* **Node restart** restores the qdisc from the NIO (like capture and markers); + **link deletion** detaches it explicitly — the veth survives the link and an + orphaned qdisc would keep impairing the next one. +* **Suspend** keeps the qdisc (carrier-driven loss); the first packet after + resume takes one extra delay interval (known netem idle-baseline behaviour) + and steady state is exact. +* A link carrying `frequency_drop` or `bpf` is wired on the **relay** (they + only exist in uBridge's userspace filters); `available_filters` hides them + on kernel links and setting them returns 409 with a clear message. ## uBridge command surface @@ -173,9 +204,14 @@ brctl create / delete / addif / delif / show link set up|down capture start_kernel / stop_kernel marker add_kernel / delete_kernel / enable_kernel +tc netem set [delay ] [jitter ] [loss <%>] [dup <%>] [corrupt <%>] / tc reset bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capture ``` +Note: `tc reset` currently replies 207-ENOENT when no qdisc is attached — the +server tolerates it; making the command idempotent in uBridge is a desirable +polish. + ## Configuration ```ini @@ -193,6 +229,15 @@ End-to-end on a five-container FRR topology (mixed runtime-drawn and reloaded links): 8/8 links on the kernel datapath, ping RTT ≈ 0.05 ms, FDB learning, suspend = 100 % loss / resume restores, project reopen upgrades relay links, runtime link creation on running containers, marker pcaps exact (tx/rx -pairs), capture freeze on stop, server restart reconciliation. Unit tests: +pairs), capture freeze on stop, server restart reconciliation. + +Filters (two-container kernel link, 27/27 checks): baseline 0.06 ms → +`delay 100` = 200.2 ms RTT (2× per direction) → `delay 50` = 100.2 ms +(atomic replace) → clear = baseline (qdisc detached); `packet_loss 30` = +48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop`/`bpf` = +409; suspend with delay active = 100 % loss, resume keeps the qdisc +(200.1 ms); node restart restores it (200.1 ms); link delete detaches it +(`/usr/sbin/tc qdisc show` — no netem left); re-created link has no residual +impairment. Unit tests: `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index 7c4887727..30a6fc1c3 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -361,8 +361,18 @@ class BaseManager: nio.suspend = nio_settings.get("suspend", False) elif nio_settings["type"] == "nio_bridge": nio = NIOBridge(nio_settings["bridge"]) - if nio_settings.get("filters"): - raise ComputeError("Packet filters are not supported on kernel-datapath links") + # Impairment filters with a tc netem equivalent run on the veth + # host end; the relay-only types are rejected here (the controller + # already keeps them off kernel links — this is the second guard). + from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS + + unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(nio_settings.get("filters") or {}) + if unsupported: + raise ComputeError( + "Packet filter(s) {} cannot run on a kernel-datapath link " + "(no uBridge relay in the forwarding path)".format(", ".join(sorted(unsupported))) + ) + nio.filters = nio_settings.get("filters", {}) # Markers ride the NIO and attach to the veth host interface via # uBridge's AF_PACKET marker module (marker add_kernel). nio.markers = nio_settings.get("markers", {}) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index b4c88f62d..921bcfc0e 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -40,6 +40,7 @@ from gns3server.utils.asyncio.raw_command_server import AsyncioRawCommandServer from gns3server.utils.asyncio.ssh_server import AsyncioSSHServer from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer from gns3server.utils.hostname import is_rfc1123_hostname_valid +from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS from ..adapters.ethernet_adapter import EthernetAdapter from ..base_node import BaseNode @@ -1856,8 +1857,10 @@ class DockerVM(BaseNode): self._kernel_veths[(adapter_number, port_number)] = host_ifc log.debug( "Created veth adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( - adapter_number=adapter_number, port_number=port_number, - mac_address=mac_address, namespace=self._namespace, + adapter_number=adapter_number, + port_number=port_number, + mac_address=mac_address, + namespace=self._namespace, ) ) @@ -1931,6 +1934,9 @@ class DockerVM(BaseNode): # Markers carried by the NIO attach to the veth host end (AF_PACKET # taps) — the anchor is the interface, not a relay bridge. await self._ubridge_apply_markers(host_ifc, nio) + # Impairment filters become one tc netem qdisc on the veth host + # end (restored here on node restart, like the capture above). + await self._ubridge_apply_netem(host_ifc, nio.filters) return # Relay NIO. On a veth-backed adapter (the unified Docker interface) @@ -1992,10 +1998,14 @@ class DockerVM(BaseNode): if self.ubridge: if isinstance(nio, NIOBridge): - if nio.filters: + # The controller keeps relay-only filters off kernel links and + # create_nio rejects them on the POST path; this PUT path + # mutates the NIO in place, so guard here too. + unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(nio.filters or {}) + if unsupported: raise DockerError( - "Packet filters are not supported on kernel-datapath links; " - "delete and recreate the link to use them" + "Packet filter(s) {} cannot run on a kernel-datapath link " + "(no uBridge relay in the forwarding path)".format(", ".join(sorted(unsupported))) ) if self.status == "started": host_ifc = self._kernel_veths.get((adapter_number, port_number)) @@ -2003,6 +2013,9 @@ class DockerVM(BaseNode): # Incremental marker reconcile on the veth anchor — # the relay-datapath equivalent of the branch below. await self._ubridge_apply_markers(host_ifc, nio) + # netem set is an atomic replace, so a full re-apply + # IS the incremental reconcile for impairment filters. + await self._ubridge_apply_netem(host_ifc, nio.filters) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) return @@ -2085,7 +2098,14 @@ class DockerVM(BaseNode): await self._ubridge_send(cmd) return await super()._ubridge_add_marker_filter( - bridge_name, name, bpf, pcap_path, tag=tag, link_id=link_id, direction=direction, data_link_type=data_link_type + bridge_name, + name, + bpf, + pcap_path, + tag=tag, + link_id=link_id, + direction=direction, + data_link_type=data_link_type, ) async def _ubridge_delete_marker_filter(self, bridge_name, name): @@ -2115,6 +2135,43 @@ class DockerVM(BaseNode): return await super()._ubridge_enable_marker_filter(anchor, name, state) + async def _ubridge_apply_netem(self, host_ifc, filters): + """ + Translate the NIO's impairment filters into one tc netem qdisc on the + veth host end. The supported types (delay, packet_loss, corrupt) merge + into a single qdisc — its egress covers the traffic entering this + container, and with both link endpoints applying theirs, every + direction of the link is impaired exactly once (the same net effect + as the relay, where both directions cross the single filtered bridge). + ``netem set`` atomically replaces whatever qdisc was attached, so a + full re-apply on every NIO update *is* the incremental reconcile. An + empty filter set detaches the qdisc (``tc reset``). + + :param host_ifc: veth host-end interface name + :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) + """ + + parts = [] + delay = filters.get("delay") + if delay: + parts.append(f"delay {int(delay[0])}") + if len(delay) > 1 and int(delay[1]): + parts.append(f"jitter {int(delay[1])}") + for filter_type, netem_kw in (("packet_loss", "loss"), ("corrupt", "corrupt")): + values = filters.get(filter_type) + if values and int(values[0]): + parts.append(f"{netem_kw} {int(values[0])}") + if parts: + await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) + return + try: + await self._ubridge_send(f'tc reset "{host_ifc}"') + except UbridgeError as e: + # "No such file or directory" = no qdisc was attached: already + # clean. Everything else is a real failure and must surface. + if "No such file" not in str(e): + raise + async def _remove_kernel_markers(self, host_ifc): """ Tear down every kernel marker anchored to *host_ifc*: the veth survives @@ -2154,6 +2211,12 @@ class DockerVM(BaseNode): host_ifc = self._kernel_veths.get((adapter_number, port_number)) if host_ifc is not None: await self._remove_kernel_markers(host_ifc) + # The veth survives link deletion, so an orphaned netem qdisc + # would keep impairing whatever attaches to the adapter next + # (kernel link or relay). Best-effort: ENOENT just means no + # qdisc was attached. + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tc reset "{host_ifc}"') if self.status == "started": await self._set_adapter_carrier(adapter_number, False, port_number) with contextlib.suppress(UbridgeError): diff --git a/gns3server/compute/nios/nio_bridge.py b/gns3server/compute/nios/nio_bridge.py index 785ac61e3..d65e38471 100644 --- a/gns3server/compute/nios/nio_bridge.py +++ b/gns3server/compute/nios/nio_bridge.py @@ -20,8 +20,9 @@ Interface for kernel-datapath bridge NIOs. A NIO of this type instructs the node to enslave its adapter's veth host end into the named Linux kernel bridge instead of wiring a uBridge UDP relay. -Frames then flow entirely in the kernel (veth -> bridge -> veth). Packet -filters are not available on this datapath (they live in the uBridge relay); +Frames then flow entirely in the kernel (veth -> bridge -> veth). Impairment +filters ride the NIO and run as one tc netem qdisc per veth host end (the +types without a netem equivalent — frequency_drop, bpf — stay relay-only); markers and packet capture are served by uBridge's AF_PACKET modules on the veth host end (marker add_kernel / capture start_kernel). """ diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index a919c8037..fdfaa1162 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -116,8 +116,9 @@ class Link: """ Whether this link is wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge — no uBridge relay in the - forwarding path, so filters and markers are unavailable; capture is - served by uBridge's AF_PACKET module). + forwarding path). Impairment filters run as tc netem on the veth + host end; markers and capture are served by uBridge's AF_PACKET + module. """ return any(d.get("type") == "nio_bridge" for d in (getattr(self, "_link_data", None) or [])) @@ -234,11 +235,18 @@ class Link: except FilterValidationError as e: raise ControllerError(f"Invalid packet filter parameters: {e!s}") - if new_filters and self.kernel_datapath: - raise ControllerError( - "Packet filters are not supported on kernel-datapath links (no uBridge relay " - "in the forwarding path); delete and recreate the link to use filters" - ) + if self.kernel_datapath: + from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS + + unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(new_filters or {}) + if unsupported: + raise ControllerError( + "Packet filter(s) {} cannot run on a kernel-datapath link (no uBridge relay " + "in the forwarding path; delay, packet loss and corrupt are served by tc " + "netem on the veth); delete and recreate the link to use them".format( + ", ".join(sorted(unsupported)) + ) + ) if new_filters != self.filters: self._filters = new_filters @@ -637,6 +645,13 @@ class Link: """ filter_node = self._get_filter_node() if filter_node: + if self.kernel_datapath: + # Kernel-datapath links serve filters via tc netem on the + # veth host end; types without a netem equivalent (which + # only run in the uBridge relay) are hidden from the picker. + from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS + + return [f for f in FILTERS if f["type"] not in KERNEL_UNSUPPORTED_FILTERS] return FILTERS return [] diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 4d02cf9fd..494bfc976 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -20,7 +20,10 @@ import asyncio import logging from gns3server.config import Config -from gns3server.utils.packet_filter_validation import validate_bpf_syntax +from gns3server.utils.packet_filter_validation import ( + KERNEL_UNSUPPORTED_FILTERS, + validate_bpf_syntax, +) from .controller_error import ControllerError, ControllerNotFoundError from .link import _UNSET, Link @@ -82,19 +85,32 @@ class UDPLink(Link): """Whether NIO creation is waiting for missing node images.""" return self._deferred - def _get_node_filters(self, node1, node2): + def _get_node_filters(self, node1, node2, kernel=None): """ Determine which node gets the active filters applied. + :param kernel: explicit datapath decision for the caller's context. + ``_prepare`` must pass it: it clears ``_link_data`` (the source + of the ``kernel_datapath`` property) before rebuilding the NIO + specs, so deriving the datapath there would read the previous + run's. Callers operating on established link data (``update``) + derive it from the property. + :returns: Tuple of (node1_filters, node2_filters) """ # Kernel-datapath links implement suspend natively (interface carrier # driven by the NIO suspend flag). The synthetic frequency_drop filter # that get_active_filters() injects for suspended links is a - # relay-datapath emulation mechanism — sending it would trip the - # compute-side kernel guard, so kernel links always push empty filters. - if self.kernel_datapath: - return {}, {} + # relay-datapath emulation mechanism, so kernel links push the *real* + # filters straight from storage. Both endpoints receive them: each + # end's compute attaches one tc netem qdisc to its veth host end + # (egress = traffic entering that container), so every direction of + # the link is impaired exactly once — the same net effect as the + # relay, where both directions cross the single filtered bridge. + if kernel is None: + kernel = self.kernel_datapath + if kernel: + return self._filters, self._filters filter_node = self._get_filter_node() return ( self.get_active_filters() if filter_node == node1 else {}, @@ -135,9 +151,11 @@ class UDPLink(Link): """ Whether this link can be wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge) instead of the uBridge UDP - relay. The kernel path has no userspace relay, so filters — which - live in the relay — disqualify it. Markers and capture are served - by uBridge's AF_PACKET modules on the veth host end. + relay. Impairment filters with a tc netem equivalent (delay, + packet_loss, corrupt) are served on the veth host end; the others + (frequency_drop, bpf) only exist in the uBridge relay and disqualify + the kernel path. Capture and markers are served by uBridge's + AF_PACKET modules on the veth host end. Docker adapters are born as veth pairs (unified interface), so the datapath is a runtime decision — links attach to running containers @@ -152,7 +170,11 @@ class UDPLink(Link): return False if node1.compute.id != node2.compute.id: return False - if self.get_active_filters(): + # Look at the *stored* filters, not get_active_filters(): a suspended + # link reports the synthetic frequency_drop emulation, which is + # relay-only mechanics and must not flip a kernel link to the relay + # on reopen/reset. + if KERNEL_UNSUPPORTED_FILTERS.intersection(self._filters or {}): return False if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): return False @@ -192,11 +214,27 @@ class UDPLink(Link): # Markers ride the NIO like on the relay datapath, routed by # capture node; they attach to the veth host end via uBridge's # AF_PACKET marker module instead of a relay `mark` filter. + # Filters (delay/packet_loss/corrupt — eligibility guarantees no + # other type is present) become one tc netem qdisc per veth host + # end, pushed to both endpoints. bridge_name = "gns3" + self._id.replace("-", "")[:11] + node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True) node1_markers, node2_markers = self._get_node_markers(node1, node2) self._link_data = [ - {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": node1_markers, "suspend": self._suspended}, - {"type": "nio_bridge", "bridge": bridge_name, "filters": {}, "markers": node2_markers, "suspend": self._suspended}, + { + "type": "nio_bridge", + "bridge": bridge_name, + "filters": node1_filters, + "markers": node1_markers, + "suspend": self._suspended, + }, + { + "type": "nio_bridge", + "bridge": bridge_name, + "filters": node2_filters, + "markers": node2_markers, + "suspend": self._suspended, + }, ] return [ (node1, adapter_number1, port_number1, self._link_data[0]), @@ -223,7 +261,7 @@ class UDPLink(Link): _allocate_port(node1.compute), _allocate_port(node2.compute) ) - node1_filters, node2_filters = self._get_node_filters(node1, node2) + node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=False) node1_markers, node2_markers = self._get_node_markers(node1, node2) # Build the tunnel specs for both sides. Index 0 is always node1 so @@ -331,7 +369,7 @@ class UDPLink(Link): self._link_data[1]["suspend"] = self._suspended if node2.node_type != "ethernet_hub": await node2.put( - f"/adapters/{adapter_number2}/ports/{port_number2}/nio", data=self._link_data[1], timeout=221 + f"/adapters/{adapter_number2}/ports/{port_number2}/nio", data=self._link_data[1], timeout=120 ) async def delete(self): diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 6ee90c2a3..53d9dc585 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -74,15 +74,18 @@ class BridgeNIO(BaseModel): """ Kernel-datapath bridge Network Input/Output properties. The NIO tells the node to enslave its veth host end into the named kernel bridge instead of - wiring a uBridge UDP relay. ``filters`` must stay empty on this datapath - (rejected server-side); markers and packet capture are served by uBridge's - AF_PACKET modules on the veth host end. + wiring a uBridge UDP relay. Impairment filters with a tc netem equivalent + (delay, packet_loss, corrupt) run on the veth host end; the relay-only + kinds (frequency_drop, bpf) are rejected. Markers and packet capture are + served by uBridge's AF_PACKET modules on the veth host end. """ type: BridgeNIOType bridge: str = Field(..., description="Kernel bridge name e.g. gns3a1b2c3d4e5") suspend: Optional[bool] = Field(None, description="Suspend the NIO") - filters: Optional[dict] = Field(None, description="Packet filters (unsupported, must be empty)") + filters: Optional[dict] = Field( + None, description="Packet filters served by tc netem on the veth host end (delay/packet_loss/corrupt)" + ) markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the veth host end)") diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index 39f994016..57bcdba60 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -4,7 +4,7 @@ Packet filter parameter validation utilities. import logging import subprocess -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional log = logging.getLogger(__name__) @@ -15,7 +15,13 @@ class FilterValidationError(Exception): pass -def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Any]: +# Packet filters with no kernel-datapath equivalent (no tc netem mapping): +# they only run in the uBridge userspace relay, so a link carrying one is +# wired on the relay, and adding them to a kernel-datapath link is rejected. +KERNEL_UNSUPPORTED_FILTERS = frozenset({"frequency_drop", "bpf"}) + + +def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Optional[str]]: """ Validate BPF filter expression syntax using tcpdump. diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index f09ba6591..53f57d9a4 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -55,7 +55,7 @@ async def vm(compute_project, manager): vm = DockerVM("test", str(uuid.uuid4()), compute_project, manager, "ubuntu:latest", aux_type="none") vm._cid = "e90e34656842" - vm.mac_address = '02:42:3d:b7:93:00' + vm.mac_address = "02:42:3d:b7:93:00" vm._start_interface_monitor = AsyncioMagicMock() vm._stop_interface_monitor = AsyncioMagicMock() return vm @@ -65,6 +65,7 @@ async def vm(compute_project, manager): # NIO factory # --------------------------------------------------------------------------- + def test_create_nio_bridge(vm): nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) @@ -73,10 +74,25 @@ def test_create_nio_bridge(vm): assert nio.suspend is False -def test_create_nio_bridge_rejects_filters(vm): +def test_create_nio_bridge_accepts_netem_filters(vm): + """ + Impairment filters with a tc netem equivalent ride the kernel NIO (they + become one netem qdisc on the veth host end); relay-only types are + rejected. + """ - with pytest.raises(ComputeError): - vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"loss": [10]}}) + nio = vm.manager.create_nio( + {"type": "nio_bridge", "bridge": BRIDGE, "filters": {"delay": [50, 5], "packet_loss": [10]}} + ) + assert isinstance(nio, NIOBridge) + assert nio.filters == {"delay": [50, 5], "packet_loss": [10]} + + +@pytest.mark.parametrize("filters", [{"frequency_drop": [10]}, {"bpf": ["icmp"]}]) +def test_create_nio_bridge_rejects_relay_only_filters(vm, filters): + + with pytest.raises(ComputeError, match="kernel-datapath"): + vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": filters}) def test_create_nio_bridge_accepts_markers(vm): @@ -94,6 +110,7 @@ def test_create_nio_bridge_accepts_markers(vm): # Adapter wiring (start path) # --------------------------------------------------------------------------- + @pytest.mark.asyncio async def test_add_ubridge_kernel_connection(vm): @@ -110,8 +127,8 @@ async def test_add_ubridge_kernel_connection(vm): call.send(f'docker delete_veth "{guest_ifc}"'), call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), call.send(f'link set "{host_ifc}" down'), - call.send(f'docker set_mac_addr {guest_ifc} 02:42:3d:b7:93:00'), - call.send(f'docker move_to_ns {guest_ifc} 42 eth0'), + call.send(f"docker set_mac_addr {guest_ifc} 02:42:3d:b7:93:00"), + call.send(f"docker move_to_ns {guest_ifc} 42 eth0"), # link attach call.send(f'brctl create "{BRIDGE}"'), call.send(f'link set "{BRIDGE}" up'), @@ -124,6 +141,26 @@ async def test_add_ubridge_kernel_connection(vm): assert "bridge0" not in vm._bridges +@pytest.mark.asyncio +async def test_add_ubridge_kernel_connection_applies_netem(vm): + """ + Filters carried by the NIO become a tc netem qdisc at start (node + restart restore), exactly like capture and markers. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + nio = vm.manager.create_nio( + {"type": "nio_bridge", "bridge": BRIDGE, "filters": {"delay": [50, 5], "packet_loss": [10]}} + ) + + await vm._add_ubridge_connection(nio, 0) + + host_ifc, _ = vm._veth_names(0, 0) + netem_calls = [c for c in vm._ubridge_hypervisor.method_calls if "netem set" in str(c)] + assert call.send(f'tc netem set "{host_ifc}" delay 50 jitter 5 loss 10') in netem_calls + + @pytest.mark.asyncio async def test_add_ubridge_connection_none_nio_creates_veth(vm): """ @@ -138,11 +175,14 @@ async def test_add_ubridge_connection_none_nio_creates_veth(vm): await vm._add_ubridge_connection(None, 0) host_ifc, guest_ifc = vm._veth_names(0, 0) - vm._ubridge_hypervisor.assert_has_calls([ - call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), - call.send(f'link set "{host_ifc}" down'), - call.send(f'docker move_to_ns {guest_ifc} 42 eth0'), - ], any_order=True) + vm._ubridge_hypervisor.assert_has_calls( + [ + call.send(f'docker create_veth "{host_ifc}" "{guest_ifc}"'), + call.send(f'link set "{host_ifc}" down'), + call.send(f"docker move_to_ns {guest_ifc} 42 eth0"), + ], + any_order=True, + ) assert vm._kernel_veths[(0, 0)] == host_ifc # no relay bridge and no TAP for an unconnected adapter assert "bridge0" not in vm._bridges @@ -165,13 +205,16 @@ async def test_add_ubridge_connection_udp_relays_over_veth(vm): await vm._add_ubridge_connection(nio, 0) host_ifc, _ = vm._veth_names(0, 0) - vm._ubridge_hypervisor.assert_has_calls([ - call.send(f'docker create_veth "{host_ifc}" "{vm._veth_names(0, 0)[1]}"'), - call.send("bridge create bridge0"), - call.send(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), - call.send("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), - call.send("bridge start bridge0"), - ], any_order=True) + vm._ubridge_hypervisor.assert_has_calls( + [ + call.send(f'docker create_veth "{host_ifc}" "{vm._veth_names(0, 0)[1]}"'), + call.send("bridge create bridge0"), + call.send(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), + call.send("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), + call.send("bridge start bridge0"), + ], + any_order=True, + ) assert "bridge0" in vm._bridges @@ -199,13 +242,14 @@ async def test_add_ubridge_kernel_connection_cleans_up_on_failure(vm): # Link attach / detach # --------------------------------------------------------------------------- + @pytest.mark.asyncio async def test_connect_nio_kernel_tolerates_bridge_create_race(vm): # Both endpoints create the per-link bridge concurrently; the loser must # verify (brctl show) instead of failing. async def send(command): - if 'brctl create' in command: + if "brctl create" in command: raise UbridgeError("Could not create bridge: File exists") vm._ubridge_send = AsyncioMagicMock(side_effect=send) @@ -223,7 +267,7 @@ async def test_connect_nio_kernel_tolerates_bridge_create_race(vm): async def test_connect_nio_kernel_create_failure_propagates(vm): async def send(command): - if 'brctl create' in command or 'brctl show' in command: + if "brctl create" in command or "brctl show" in command: raise UbridgeError("Could not create bridge: No such device") vm._ubridge_send = AsyncioMagicMock(side_effect=send) @@ -255,11 +299,15 @@ async def test_remove_kernel_nio(vm): await vm.adapter_remove_nio_binding(0) - vm._ubridge_send.assert_has_calls([ - call(f'link set "{host_ifc}" down'), - call(f'brctl delif "{BRIDGE}" "{host_ifc}"'), - call(f'brctl delete "{BRIDGE}"'), - ], any_order=True) + vm._ubridge_send.assert_has_calls( + [ + call(f'tc reset "{host_ifc}"'), + call(f'link set "{host_ifc}" down'), + call(f'brctl delif "{BRIDGE}" "{host_ifc}"'), + call(f'brctl delete "{BRIDGE}"'), + ], + any_order=True, + ) assert vm._ethernet_adapters[0].get_nio(0) is None # the veth itself stays: it is the adapter interface, not the link assert vm._kernel_veths[(0, 0)] == host_ifc @@ -273,7 +321,7 @@ async def test_remove_kernel_nio_tolerates_peer_winning_bridge_delete(vm): vm._ubridge_hypervisor = MagicMock() async def send(command): - if 'brctl delif' in command or 'brctl delete' in command: + if "brctl delif" in command or "brctl delete" in command: raise UbridgeError("busy") vm._ubridge_send = AsyncioMagicMock(side_effect=send) @@ -287,6 +335,7 @@ async def test_remove_kernel_nio_tolerates_peer_winning_bridge_delete(vm): # Carrier / suspend # --------------------------------------------------------------------------- + @pytest.mark.asyncio async def test_set_adapter_carrier_kernel(vm): @@ -297,10 +346,12 @@ async def test_set_adapter_carrier_kernel(vm): await vm._set_adapter_carrier(0, True) await vm._set_adapter_carrier(0, False) - vm._ubridge_send.assert_has_calls([ - call(f'link set "{host_ifc}" up'), - call(f'link set "{host_ifc}" down'), - ]) + vm._ubridge_send.assert_has_calls( + [ + call(f'link set "{host_ifc}" up'), + call(f'link set "{host_ifc}" down'), + ] + ) # --------------------------------------------------------------------------- @@ -308,6 +359,7 @@ async def test_set_adapter_carrier_kernel(vm): # on a running node) # --------------------------------------------------------------------------- + @pytest.mark.asyncio async def test_connect_nio_udp_relays_over_veth(vm): @@ -318,12 +370,15 @@ async def test_connect_nio_udp_relays_over_veth(vm): nio = vm.manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) await vm._connect_nio(0, nio) - vm._ubridge_send.assert_has_calls([ - call("bridge create bridge0"), - call(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), - call("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), - call("bridge start bridge0"), - ], any_order=True) + vm._ubridge_send.assert_has_calls( + [ + call("bridge create bridge0"), + call(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), + call("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), + call("bridge start bridge0"), + ], + any_order=True, + ) assert "bridge0" in vm._bridges @@ -348,18 +403,89 @@ async def test_connect_nio_udp_on_veth_is_idempotent(vm): # Guards and cleanup # --------------------------------------------------------------------------- + @pytest.mark.asyncio -async def test_update_nio_kernel_rejects_filters(vm): +async def test_update_nio_kernel_rejects_relay_only_filters(vm): + """ + The PUT route mutates the NIO in place (no create_nio), so the update + path carries its own guard against relay-only filter types. + """ vm._ubridge_hypervisor = MagicMock() nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) vm._ethernet_adapters[0].add_nio(0, nio) - nio.filters = {"loss": [50]} + nio.filters = {"frequency_drop": [10]} with pytest.raises(DockerError, match="kernel-datapath"): await vm.adapter_update_nio_binding(0, nio) +@pytest.mark.asyncio +async def test_update_nio_kernel_applies_netem(vm): + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"delay": [50, 5], "packet_loss": [10], "corrupt": [2]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc netem set "{host_ifc}" delay 50 jitter 5 loss 10 corrupt 2') + + +@pytest.mark.asyncio +async def test_update_nio_kernel_clears_netem(vm): + """ + Filters dropped to empty must detach the qdisc (the veth survives link + changes); ENOENT (no qdisc was attached) is tolerated. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc reset "{host_ifc}"') + + +@pytest.mark.asyncio +async def test_apply_netem_reset_tolerates_missing_qdisc(vm): + + vm._ubridge_hypervisor = MagicMock() + from gns3server.compute.ubridge.ubridge_error import UbridgeError + + async def refuse(command): + raise UbridgeError("207-Could not reset qdisc on gv0: No such file or directory") + + vm._ubridge_send = refuse + host_ifc, _ = vm._veth_names(0, 0) + # ENOENT means "already clean" — must not raise + await vm._ubridge_apply_netem(host_ifc, {}) + + +@pytest.mark.asyncio +async def test_apply_netem_reset_reraises_real_errors(vm): + + vm._ubridge_hypervisor = MagicMock() + from gns3server.compute.ubridge.ubridge_error import UbridgeError + + async def refuse(command): + raise UbridgeError("206-Could not reset qdisc on gv0: Operation not permitted") + + vm._ubridge_send = refuse + host_ifc, _ = vm._veth_names(0, 0) + with pytest.raises(UbridgeError): + await vm._ubridge_apply_netem(host_ifc, {}) + + @pytest.mark.asyncio async def test_update_nio_kernel_suspend_toggles_carrier(vm): @@ -451,9 +577,16 @@ async def test_connect_nio_kernel_restarts_capture(vm): # Markers (AF_PACKET taps on the veth host end) # --------------------------------------------------------------------------- + def _marker_spec(bpf="icmp", tag=1, enabled=True, direction=None, link_id="l1"): - return {"bpf": bpf, "tag": tag, "enabled": enabled, "direction": direction, - "data_link_type": "DLT_EN10MB", "link_id": link_id} + return { + "bpf": bpf, + "tag": tag, + "enabled": enabled, + "direction": direction, + "data_link_type": "DLT_EN10MB", + "link_id": link_id, + } @pytest.mark.asyncio diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index b47532de6..b9301c049 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -88,12 +88,46 @@ async def test_kernel_datapath_not_eligible_across_computes(project): @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_with_filters(project): +async def test_kernel_datapath_eligible_with_netem_filters(project): + """ + Impairment filters with a tc netem equivalent (delay, packet_loss, + corrupt) no longer disqualify the kernel datapath — they become one + netem qdisc per veth host end. + """ link, node1, node2 = await _kernel_link(project) - link._filters = {"delay": [10, 0]} + link._filters = {"delay": [10, 0], "packet_loss": [5], "corrupt": [1]} + assert link._kernel_datapath_eligible(node1, node2) is True + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_with_relay_only_filters(project): + """ + frequency_drop and bpf only exist in the uBridge userspace relay — a + link carrying one stays on the relay. + """ + + link, node1, node2 = await _kernel_link(project) + link._filters = {"delay": [10, 0], "frequency_drop": [7]} assert link._kernel_datapath_eligible(node1, node2) is False + link._filters = {"bpf": ["icmp"]} + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_when_suspended(project): + """ + A suspended link reports the synthetic frequency_drop emulation through + get_active_filters() — that is relay-only mechanics and must not flip a + kernel link to the relay on reopen/reset (suspend is carrier-driven). + """ + + link, node1, node2 = await _kernel_link(project) + link._suspended = True + assert link.get_active_filters() == {"frequency_drop": [-1]} + assert link._kernel_datapath_eligible(node1, node2) is True + @pytest.mark.asyncio async def test_kernel_datapath_eligible_with_markers(project): @@ -191,7 +225,7 @@ async def test_prepare_relay_link_allocates_udp_ports(project): compute1.get_ip_on_same_subnet.side_effect = subnet_callback - async def port_callback(path, data={}, **kwargs): + async def port_callback(path, data=None, **kwargs): response = MagicMock() response.json = {"udp_port": 1024} return response @@ -212,13 +246,29 @@ async def test_prepare_relay_link_allocates_udp_ports(project): @pytest.mark.asyncio -async def test_update_filters_rejected_on_kernel_link(project): +async def test_update_netem_filters_accepted_on_kernel_link(project): + + link, node1, node2 = await _kernel_link(project) + await link._prepare() + + await link.update_filters({"delay": [10, 0], "packet_loss": [5]}) + assert link.filters == {"delay": [10, 0], "packet_loss": [5]} + # both endpoints carry the filters (one netem qdisc per veth host end) + by_node = {entry[0].id: entry[3] for entry in (await link._prepare())} + assert by_node[node1.id]["filters"] == {"delay": [10, 0], "packet_loss": [5]} + assert by_node[node2.id]["filters"] == {"delay": [10, 0], "packet_loss": [5]} + + +@pytest.mark.asyncio +async def test_update_relay_only_filters_rejected_on_kernel_link(project): link, _n1, _n2 = await _kernel_link(project) await link._prepare() with pytest.raises(ControllerError, match="kernel-datapath"): - await link.update_filters({"delay": [10, 0]}) + await link.update_filters({"frequency_drop": [10]}) + with pytest.raises(ControllerError, match="kernel-datapath"): + await link.update_filters({"bpf": ["icmp"]}) @pytest.mark.asyncio diff --git a/tests/controller/test_udp_link.py b/tests/controller/test_udp_link.py index 032bff079..448c1b198 100644 --- a/tests/controller/test_udp_link.py +++ b/tests/controller/test_udp_link.py @@ -565,7 +565,7 @@ async def test_update_ethernet_switch_nio(project): "markers": {}, "filters": {}, }, - timeout=221, + timeout=120, ) From bd1c2056510dea5652002a004e6e40a5ea8221b7 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 28 Sep 2026 22:17:51 +0800 Subject: [PATCH 008/101] docs: frozen uBridge spec for kernel impairment (netem extensions, eBPF/cBPF classifiers) --- docs/design/ubridge-kernel-impairment-spec.md | 277 ++++++++++++++++++ docs/features/docker-kernel-datapath.md | 9 +- 2 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 docs/design/ubridge-kernel-impairment-spec.md diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md new file mode 100644 index 000000000..9999456b1 --- /dev/null +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -0,0 +1,277 @@ + + +> Frozen requirements spec for the **uBridge** project. The gns3-server side is +> NOT implemented yet — it lands once uBridge delivers this surface. Everything +> marked *shipped* already exists in uBridge 1.2.3+ and is verified against +> gns3-server branch `feat/docker-kernel-filters`. + +# uBridge kernel impairment: tc netem extensions + eBPF classifiers + +## 0. Background & scope + +On the kernel datapath (Docker veth pairs enslaved into per-link Linux +bridges) frames never pass through uBridge's userspace relay, so link +impairment must live in the kernel: a **netem qdisc** on each veth host end +(serving delay / jitter / loss / dup / corrupt — *shipped*) plus **tc +classifiers** on the clsact egress hook for everything netem cannot express. + +This spec freezes: + +| Part | Content | New capability needed | +|---|---|---| +| A | netem keyword extensions: `rate`, `reorder`, loss models, jitter distributions, correlation, `seed`, `limit` | none (netlink only) | +| B | eBPF stateful classifier: `frequency_drop` (nth-packet), byte quota, time window, flow select | **CAP_BPF** | +| C | classic-BPF match-drop: the GNS3 `bpf` filter | none (netlink only) | +| D | lifecycle, composition & ordering contract, `tc reset` semantics fix | — | +| E | error strings, `tc capabilities` probe | — | +| F | test requirements | — | + +Non-goals (explicitly out of scope for now): packet mangling (`pedit`, +`csum`), mirroring (`mirred`), HTB hierarchies, RST/connection-kill +injection, ingress-side (XDP) impairment. + +All commands operate on **egress of the veth host end** (traffic entering +the container), consistent with the existing netem placement. + +--- + +## A. `tc netem set` keyword extensions *(P6a)* + +Extend the shipped grammar — existing keywords and their encoding are +frozen and unchanged: + +``` +tc netem set + [delay ] [jitter ] [distribution uniform|normal|pareto|paretonormal] [seed ] + [loss [correl ]] + [loss gemodel

[ [<1-h>]]] # mutually exclusive with plain loss + [dup [correl ]] [corrupt ] + [reorder [correl ] [gap ]] # requires delay + [rate ] # e.g. 10mbit, 512kbit + [limit ] +``` + +### A.1 Parameter validation + +| Keyword | Range / format | Errors | +|---|---|---| +| `rate` | `` = integer + unit `bit\|kbit\|mbit\|gbit\|bps\|kbps\|mbps` (decimal), max 100gbit | 204 `invalid rate ''` | +| `reorder` | pct 0–100, correl 0–100, gap 1–1000 | 204; 204 `reorder requires delay` if no `delay` present (mirror the kernel's EINVAL instead of passing it through — the server keys on the string) | +| `loss gemodel` | p 0–100, r 0–100, h as `1-h` where h is 0–100 (default r=0, 1-h=0) | 204 | +| `distribution` | one of the four names | 204 `unknown distribution ''` | +| `seed` | u32 decimal | 204 | +| `limit` | 1–1000000 packets (replaces the current fixed 1000 default) | 204 | +| `correl` suffix on loss/dup | 0–100, only directly after its keyword | 204 | + +Max token count for the command table rises accordingly (recommend 32). + +### A.2 Netlink encoding notes + +* Correlation: loss/dup/reorder use their existing nested structs with the + `correlation` field set (`tc_netem_loss`? no — see `struct tc_netem_qopt` + + `TCA_NETEM_CORR` carrying `tc_netem_corr` {delay, loss, dup}). Fill the + loss correlation there; dup likewise. +* gemodel: `TCA_NETEM_LOSS` nested attr, kind `TCA_NETEM_LOSS_GEMODEL` + (`struct tc_netem_gemodel` {p, r, h}), probabilities in the same % × 2³² + encoding as `qopt.loss`. When gemodel is set, `qopt.loss` stays 0. +* reorder: `TCA_NETEM_REORDER` (`struct tc_netem_reorder` + {probability, correlation}). +* rate: prefer `TCA_NETEM_RATE64` (u64 bits/s) and also fill legacy + `TCA_NETEM_RATE` (`tc_netem_rate`) with the truncated value for older + kernels; `slot_overhead = 0`. +* distribution: `TCA_NETEM_JITTER64` + `TCA_NETEM_DIST` must carry the + distribution table (s16 deltas). **Embed the three standard tables** + (normal / pareto / paretonormal — same values iproute2 ships as + `*.dist`, generated by netem's `maketable`) as const arrays in uBridge; + select by name. Without DIST the kernel silently uses uniform jitter. +* seed: `TCA_NETEM_SEED` (u64 allowed) — makes random draws reproducible + for tests. When omitted keep the current behaviour (kernel picks). + +### A.3 Reply contract (unchanged shape) + +``` +100-netem set on +204-invalid value '' +204-reorder requires delay +207-Could not set netem on : +``` + +--- + +## B. eBPF stateful classifier *(frequency_drop + P6b modes)* + +**One** SCHED_CLS eBPF program, loaded once per interface, attached to the +clsact egress hook. All modes are configured through one ARRAY map entry — +no program reload on parameter changes. + +### B.1 Program + +* Source `src/tc_impair.bpf.c` committed for reference; the build commits + the compiled object (`tc_impair.bpf.o`, freestanding: no CO-RE, no + BTF-typed ptr, plain `struct __sk_buff` + direct packet access). + No runtime clang/libbpf dependency. +* Maps: + * `CFG` — `BPF_MAP_TYPE_ARRAY`, 1 entry: + ``` + struct cfg { + __u32 nth; /* 0 = off, else drop every Nth packet */ + __u64 quota_bytes; /* 0 = off */ + __u32 quota_pct; /* random drop % after quota reached */ + __u64 win_start_ns; /* recurring window start (monotonic) */ + __u64 win_len_ns; /* 0 = off */ + __u32 win_pct; /* random drop % inside the window */ + __u32 flow_mask; /* bitmask: 1=src 2=dst 4=sport 8=dport 16=proto */ + __u32 flow_target; /* required hash remainder, 0 = off */ + }; + ``` + * `CNT` — `BPF_MAP_TYPE_ARRAY`, 1 entry: `{__u64 packets; __u64 bytes; __u64 nth_state;}` + Counters are updated with `__sync_fetch_and_add` on the map/global value + so the every-Nth count is exact across CPUs (requires kernel ≥ 5.1 — + acceptable; probe at load, see E). +* Evaluation order is fixed: **nth → quota → window → flow**; the first + rule that decides a drop returns `TC_ACT_SHOT`, otherwise `TC_ACT_OK`. + Random percentages use the same 2³²-probability comparison as netem + (`prandom` seeded via map so tests are reproducible). +* Verifier-friendly: no loops, bounded accesses (check `data + eth+h` before + reading fields; when flow_mask touches ports, verify L4 header length). + +### B.2 Command surface + +``` +tc nth_drop +tc quota_drop | off +tc window_drop | off +tc flow_drop | off +``` + +* First enable on an interface: create clsact (EEXIST tolerated), load the + program, attach one filter at **prio 1** (see D). Subsequent commands only + update `CFG`/reset the relevant `CNT` fields. +* `off` for every mode → the filter is removed (and the prog fd closed). + Each `off` resets that mode's counters. +* `flow_drop` mask is the decimal bitmask from B.1 (server composes it); + hash = Jenkins/equal-fold over the selected header fields modulo + `flow_target` == remainder 0 → drop. Server documents: per-flow drop only + (a filter cannot delay; delay stays netem's job). +* Window semantics: `win_start_ns`/`win_len_ns` define a recurring window on + the monotonic clock — inside the window drop with `win_pct`; outside pass. + The server computes the first start; the program advances by `len` when + `now >= start + len` (single writer — the CFG update — so this is safe). + +### B.3 Capability requirement + +`BPF_PROG_LOAD(SCHED_CLS)` needs **CAP_BPF** (or CAP_SYS_ADMIN) on kernels +≥ 5.8. Installation must therefore set: + +``` +setcap cap_bpf,cap_net_admin,cap_net_raw=ep +``` + +On EPERM: reply `210-uBridge lacks CAP_BPF (setcap cap_bpf,cap_net_admin,cap_net_raw=ep) and the kernel requires it for stateful filters` +— the gns3-server side falls back to the relay datapath on this string (E). + +--- + +## C. classic-BPF match drop (the GNS3 `bpf` filter) + +No eBPF, no CAP_BPF: `cls_bpf` accepts **classic bytecode** via +`TCA_BPF_BYTECODE` (`sock_fprog`), and the kernel internally migrates it to +eBPF. uBridge already links libpcap. + +``` +tc bpf_drop add "" # prio: 10..99, server-assigned +tc bpf_drop flush +``` + +* `add`: `pcap_compile(DLT_EN10MB, snaplen 65535)` the expression; on + failure reply + `209-Cannot compile filter '': ` — gns3-server keys on + this exact prefix (same shape as the relay's error so one regex serves + both datapaths). Netlink: RTM_NEWTFILTER on clsact egress, protocol + ETH_P_ALL, `TCA_KIND "bpf"`, `TCA_BPF_BYTECODE`, action nested + `TCA_BPF_ACT` → `gact` with `action = TC_ACT_SHOT` (pcap match ≠ 0 → + filter matches → drop; non-match falls through, no action → pass). +* Multiple expressions (GNS3 sends one line per filter): server adds one + filter per line at successive prios — any match drops (OR). +* `flush`: delete every bpf_drop filter uBridge added on this interface + (track prios in-process), tolerate ENOENT per filter. Does NOT touch the + eBPF prio-1 filter or netem. +* Update-by-replace: server always `flush` + re-`add` (mirrors `netem set`). + +--- + +## D. Lifecycle, composition & ordering (frozen contract) + +* **clsact coexists with the netem root qdisc** — never replace the root. +* Execution order on egress: classifier filters by ascending prio — + eBPF impair filter at **prio 1**, `bpf_drop` at **prio 10..99** — then the + root netem qdisc. Dropped packets therefore never reach netem **nor the + AF_PACKET tap points** (capture/markers will not observe cls_bpf-dropped + frames; documented server-side). +* **`tc reset ` becomes the full restore** (behaviour change + fix): + 1. remove all bpf_drop filters and the eBPF filter (clsact egress side we + own), 2. delete clsact, 3. delete the root qdisc. On an untouched + interface it must now reply **100-OK** (idempotent — currently returns + 207-ENOENT; gns3-server tolerates both, but the new contract is OK). +* All state is kernel state tied to the interface: deleting the veth + (container stop) clears everything; no ubridge-side bookkeeping survives + a restart (a re-attached link rebuilds from the NIO, as today). + +--- + +## E. Errors & capability probe + +Error strings are contract (gns3-server matches them): + +| Reply | Meaning | +|---|---| +| `100-netem set on ` / `100-qdisc reset on ` | success | +| `203-Bad number of parameters (%d with min/max=%d/%d)` | argc mismatch | +| `204-invalid value ''` | value validation | +| `204-reorder requires delay` | keyword dependency | +| `207-Could not (set netem\|reset qdisc) on : ` | netlink/kernel failure | +| `209-Cannot compile filter '': ` | bpf_drop pcap_compile failure | +| `210-uBridge lacks CAP_BPF ...` | stateful filters unavailable | + +New introspection command (also serves the gns3-server degradation TODO — +old uBridge without these commands simply stays on the relay datapath): + +``` +tc capabilities +100-netem=delay,jitter,loss,dup,corrupt,rate,reorder,gemodel,dist,seed,limit;ebpf=1;cbpf=1 +``` + +`ebpf=0` when the load probe fails at startup (kernel < 5.1 or no CAP_BPF); +`cbpf=0` when RTM_NEWTFILTER/cls_bpf is unavailable. gns3-server hides the +corresponding filter types per capability. + +--- + +## F. Test requirements + +* **Unit**: per keyword → expected netlink attrs (assert message bytes); + every 204 table row; `bpf_drop` compile-fail path; capabilities string. +* **Functional** (root, veth pair + netns + nsenter ping): + delay+reorder observable (mdev), gemodel loss within ±5% of target, + rate within ±10% (measured byte throughput), nth_drop **exact** pattern + (ICMP seq survives 1,2,…,N-1, Nth dropped), quota/window/flow with + crafted senders, `tc reset` idempotent on a clean interface, P5 surface + regression (delay/jitter/loss/dup/corrupt, REPLACE semantics). +* Determinism: netem `seed` + the CFG-seeded prandom make runs repeatable; + CI asserts identical drop patterns for identical seeds. + +--- + +## gns3-server alignment (informational — not uBridge scope) + +* P6a types map 1:1 onto new GNS3 filter entries (rate, reorder, gemodel…), + translated in `DockerVM._ubridge_apply_netem`. +* `frequency_drop` / `bpf` move out of `KERNEL_UNSUPPORTED_FILTERS` once + `tc capabilities` reports `ebpf=1` / `cbpf=1` (per-compute probe, cached); + without the capability the link stays relay-wired exactly as today. +* Every veth end owns one qdisc + its filters: per-direction impairment is + an architectural freebie to expose later (API `direction` field), aligned + with marker `dir` semantics. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 21c95f352..a8a2e6da6 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -168,8 +168,8 @@ ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): | `delay [ms, jitter]` | `delay X jitter Y` | jitter 0 omitted | | `packet_loss [%]` | `loss P` | per direction (see below) | | `corrupt [%]` | `corrupt P` | | -| `frequency_drop` | — no netem equivalent | 409 on kernel links; relay fallback | -| `bpf` | — needs a tc classifier | 409 on kernel links; relay fallback | +| `frequency_drop` | — no netem equivalent | 409 on kernel links; relay fallback (eBPF classifier spec'd: [ubridge-kernel-impairment-spec](../design/ubridge-kernel-impairment-spec.md)) | +| `bpf` | — needs a tc classifier | 409 on kernel links; relay fallback (cBPF match-drop spec'd, same doc) | Semantics: @@ -192,6 +192,11 @@ Semantics: * A link carrying `frequency_drop` or `bpf` is wired on the **relay** (they only exist in uBridge's userspace filters); `available_filters` hides them on kernel links and setting them returns 409 with a clear message. + Kernel-side implementations are frozen in + [docs/design/ubridge-kernel-impairment-spec.md](../design/ubridge-kernel-impairment-spec.md) + (eBPF stateful classifier for `frequency_drop` + quota/window/flow modes, + classic-BPF match-drop for `bpf`, plus netem keyword extensions: rate, + reorder, gemodel loss, jitter distributions). ## uBridge command surface From 2b45f29b2c58fb41411083c795084636ffbd96a3 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 29 Sep 2026 10:31:52 +0800 Subject: [PATCH 009/101] feat(docker): bpf packet filter on kernel links via cls_bpf match-drop The GNS3 bpf filter (one expression per line, any line matching drops the packet) now runs on the kernel datapath: uBridge's tc bpf_drop compiles each line with libpcap (same compiler as the relay filter) into classic BPF, attaches it to the veth host end's clsact egress as a cls_bpf classifier with a gact TC_ACT_SHOT action, and the kernel migrates the cBPF to eBPF internally -- no CAP_BPF needed. - reconcile is flush + one add per line at priorities 10, 11, ... (OR semantics, same as the relay's per-line filters); tc reset (link delete / clear) is the full restore in uBridge and clears them - capability-gated: the compute probes "tc capabilities" once per uBridge process (cached, invalidated on stop); without cbpf=1 the filter apply fails with an actionable upgrade error, and old uBridge builds without the tc module are never touched when no bpf filter is present - a line that compiled for the controller's tcpdump but not for the compute's uBridge libpcap is skipped with a warning, mirroring the relay path - KERNEL_UNSUPPORTED_FILTERS shrinks to frequency_drop (its eBPF stateful classifier is still pending, spec part B) e2e (22/22): bpf "icmp" = 100% loss; size-discriminating expression ("greater 150") passes 98-byte and drops 300-byte pings (real byte-level match); two-line OR; coexistence with netem (delay 50 -> small pings 100.2 ms while big dropped); clear removes clsact; suspend/resume keeps the drops; link delete + recreate leaves no residual filters. --- docs/design/ubridge-kernel-impairment-spec.md | 13 +- docs/features/docker-kernel-datapath.md | 66 +++++---- gns3server/compute/docker/docker_vm.py | 96 ++++++++++++- gns3server/schemas/compute/nios.py | 13 +- gns3server/utils/packet_filter_validation.py | 10 +- .../docker/test_docker_kernel_datapath.py | 127 +++++++++++++++++- tests/controller/test_kernel_datapath_link.py | 28 +++- 7 files changed, 303 insertions(+), 50 deletions(-) diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md index 9999456b1..52b36399a 100644 --- a/docs/design/ubridge-kernel-impairment-spec.md +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -3,10 +3,15 @@ SPDX-License-Identifier: CC-BY-SA-4.0 See LICENSE file for licensing information. --> -> Frozen requirements spec for the **uBridge** project. The gns3-server side is -> NOT implemented yet — it lands once uBridge delivers this surface. Everything -> marked *shipped* already exists in uBridge 1.2.3+ and is verified against -> gns3-server branch `feat/docker-kernel-filters`. +> Frozen requirements spec for the **uBridge** project. Delivery status: +> parts **A** (netem keyword extensions, uBridge `feature/tc-netem-ext`) and +> **C + D** (cBPF match-drop, full-restore idempotent `tc reset`, +> `tc capabilities` with `cbpf` probe — uBridge `feature/tc-bpf-drop`) are +> **delivered and integrated** on gns3-server branch +> `feat/docker-kernel-bpf-drop`. Part **B** (eBPF stateful classifier) is +> still pending. The originally shipped netem surface (delay/jitter/loss/ +> dup/corrupt) exists in uBridge 1.2.3+ and is verified against gns3-server +> branch `feat/docker-kernel-filters`. # uBridge kernel impairment: tc netem extensions + eBPF classifiers diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index a8a2e6da6..38764531b 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -24,12 +24,13 @@ Frames never leave the kernel. | Link | uBridge bridge, TAP fd ↔ UDP, userspace copy | per-link kernel bridge `gns3{link_id[:11]}`, zero-copy | | Relay latency | ~0.3 ms RTT | ~0.05 ms RTT (measured) | | Link attach at runtime | yes (relay attaches the pre-existing TAP) | yes (brctl addif / add_nio_ethernet — interface untouched) | -| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt); frequency_drop & bpf stay relay-only | +| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt); bpf as cls_bpf match-drop classifiers; frequency_drop stays relay-only (eBPF classifier pending) | | Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | -The work landed in five stages on branch stack `feat/docker-kernel-datapath` → +The work landed in six stages on branch stack `feat/docker-kernel-datapath` → `feat/docker-kernel-capture` → `feat/docker-kernel-markers` → -`feat/docker-veth-everywhere` → `feat/docker-kernel-filters`: +`feat/docker-veth-everywhere` → `feat/docker-kernel-filters` → +`feat/docker-kernel-bpf-drop`: 1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based suspend, crash-safe reconciliation. @@ -43,6 +44,11 @@ The work landed in five stages on branch stack `feat/docker-kernel-datapath` → tc netem qdisc per veth host end, pushed through uBridge's netlink `tc` module. This also removes the relay delay filter's nanosleep bottleneck (upstream #2827) from the kernel path. +6. **bpf match-drop** — the `bpf` filter (any line matching drops) runs as + cls_bpf classifiers on the veth host end's clsact egress (`tc bpf_drop`: + uBridge pcap-compiles the line to classic BPF, the kernel migrates it to + eBPF internally — no CAP_BPF needed). Requires a uBridge reporting + `cbpf=1` via `tc capabilities`; older builds get a clear upgrade error. ## Adapter interface types @@ -163,13 +169,13 @@ marker enable_kernel # off = installed but silent Impairment filters run **in the kernel** as tc netem qdiscs on the veth host ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): -| GNS3 filter | netem mapping | Notes | +| GNS3 filter | kernel implementation | Notes | |---|---|---| -| `delay [ms, jitter]` | `delay X jitter Y` | jitter 0 omitted | -| `packet_loss [%]` | `loss P` | per direction (see below) | -| `corrupt [%]` | `corrupt P` | | -| `frequency_drop` | — no netem equivalent | 409 on kernel links; relay fallback (eBPF classifier spec'd: [ubridge-kernel-impairment-spec](../design/ubridge-kernel-impairment-spec.md)) | -| `bpf` | — needs a tc classifier | 409 on kernel links; relay fallback (cBPF match-drop spec'd, same doc) | +| `delay [ms, jitter]` | netem `delay X jitter Y` | jitter 0 omitted | +| `packet_loss [%]` | netem `loss P` | per direction (see below) | +| `corrupt [%]` | netem `corrupt P` | | +| `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | +| `frequency_drop` | — not yet (eBPF stateful classifier spec'd: [ubridge-kernel-impairment-spec](../design/ubridge-kernel-impairment-spec.md), part B pending) | 409 on kernel links; relay fallback | Semantics: @@ -189,14 +195,21 @@ Semantics: * **Suspend** keeps the qdisc (carrier-driven loss); the first packet after resume takes one extra delay interval (known netem idle-baseline behaviour) and steady state is exact. -* A link carrying `frequency_drop` or `bpf` is wired on the **relay** (they - only exist in uBridge's userspace filters); `available_filters` hides them - on kernel links and setting them returns 409 with a clear message. - Kernel-side implementations are frozen in +* A link carrying `frequency_drop` is wired on the **relay** (its eBPF + stateful classifier is not delivered yet); `available_filters` hides it on + kernel links and setting it returns 409 with a clear message. The remaining + kernel-side roadmap is frozen in [docs/design/ubridge-kernel-impairment-spec.md](../design/ubridge-kernel-impairment-spec.md) (eBPF stateful classifier for `frequency_drop` + quota/window/flow modes, - classic-BPF match-drop for `bpf`, plus netem keyword extensions: rate, - reorder, gemodel loss, jitter distributions). + plus netem keyword extensions: rate, reorder, gemodel loss, jitter + distributions — parts C and D of that spec are delivered). +* **bpf reconcile = flush + re-add** (`tc bpf_drop flush`, then one add per + line at priorities 10, 11, …). `tc reset` (link delete / filter clear) is + the full restore in uBridge: filters → clsact → root qdisc, idempotent. +* **Observability caveat:** clsact drops happen before the AF_PACKET tap + points — markers/captures on the same veth do not observe cls_bpf-dropped + frames (on the relay, filter-vs-mark ordering determined visibility + instead). ## uBridge command surface @@ -209,14 +222,13 @@ brctl create / delete / addif / delif / show link set up|down capture start_kernel / stop_kernel marker add_kernel / delete_kernel / enable_kernel -tc netem set [delay ] [jitter ] [loss <%>] [dup <%>] [corrupt <%>] / tc reset +tc netem set [delay ] [jitter ] [loss <%>] [dup <%>] [corrupt <%>] +tc bpf_drop add "" / tc bpf_drop flush +tc reset # full restore: filters -> clsact -> root qdisc, idempotent +tc capabilities # "netem=;ebpf=0|1;cbpf=0|1", probed once per process bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capture ``` -Note: `tc reset` currently replies 207-ENOENT when no qdisc is attached — the -server tolerates it; making the command idempotent in uBridge is a desirable -polish. - ## Configuration ```ini @@ -239,10 +251,18 @@ pairs), capture freeze on stop, server restart reconciliation. Filters (two-container kernel link, 27/27 checks): baseline 0.06 ms → `delay 100` = 200.2 ms RTT (2× per direction) → `delay 50` = 100.2 ms (atomic replace) → clear = baseline (qdisc detached); `packet_loss 30` = -48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop`/`bpf` = -409; suspend with delay active = 100 % loss, resume keeps the qdisc +48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop` = 409; +suspend with delay active = 100 % loss, resume keeps the qdisc (200.1 ms); node restart restores it (200.1 ms); link delete detaches it (`/usr/sbin/tc qdisc show` — no netem left); re-created link has no residual -impairment. Unit tests: +impairment. + +bpf match-drop (same setup, 22/22 checks): `bpf "icmp"` = 100 % loss on both +ends' clsact; size-discriminating expression (`greater 150`) = small pings +pass / 300-byte pings dropped (real byte-level BPF match); two-line OR; +coexistence with netem (delay 50 → small pings 100.2 ms while big dropped); +clear removes clsact entirely; suspend/resume keeps the drops; link delete + +recreate leaves no residual filters; `frequency_drop` still 409; +`available_filters` shows `bpf` and hides `frequency_drop`. Unit tests: `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 921bcfc0e..63ca5a089 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -177,6 +177,11 @@ class DockerVM(BaseNode): # container death. Kernel links enslave it into a Linux bridge, # relay links attach it to the uBridge relay via AF_PACKET. self._kernel_veths = {} + # uBridge tc-module capability report, probed lazily once per uBridge + # process (None = not probed yet, {} = no tc support, else the parsed + # "tc capabilities" reply). Invalidated on stop: the next start runs a + # fresh uBridge process which may differ (upgrade/downgrade). + self._ubridge_tc_caps = None if adapters is None: self.adapters = 1 @@ -1430,6 +1435,9 @@ class DockerVM(BaseNode): self._console_websocket = None self._cleanup_console_resources() await self._clean_servers() + # The next start spawns a fresh uBridge process — its capabilities + # must be re-probed (see _ubridge_tc_capabilities). + self._ubridge_tc_caps = None # veth host ends live in the root namespace and survive container # death — remove them while the uBridge control channel is still up. await self._remove_kernel_veths() @@ -1935,8 +1943,10 @@ class DockerVM(BaseNode): # taps) — the anchor is the interface, not a relay bridge. await self._ubridge_apply_markers(host_ifc, nio) # Impairment filters become one tc netem qdisc on the veth host - # end (restored here on node restart, like the capture above). + # end (restored here on node restart, like the capture above); + # bpf expressions become cls_bpf match-drop classifiers. await self._ubridge_apply_netem(host_ifc, nio.filters) + await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) return # Relay NIO. On a veth-backed adapter (the unified Docker interface) @@ -1998,9 +2008,11 @@ class DockerVM(BaseNode): if self.ubridge: if isinstance(nio, NIOBridge): - # The controller keeps relay-only filters off kernel links and - # create_nio rejects them on the POST path; this PUT path - # mutates the NIO in place, so guard here too. + # The controller keeps frequency_drop off kernel links (no + # kernel equivalent until the eBPF classifier lands) and + # create_nio rejects it on the POST path; this PUT path + # mutates the NIO in place, so guard here too. bpf is fine — + # it becomes cls_bpf match-drop classifiers (cbpf-probed). unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(nio.filters or {}) if unsupported: raise DockerError( @@ -2014,8 +2026,10 @@ class DockerVM(BaseNode): # the relay-datapath equivalent of the branch below. await self._ubridge_apply_markers(host_ifc, nio) # netem set is an atomic replace, so a full re-apply - # IS the incremental reconcile for impairment filters. + # IS the incremental reconcile for impairment filters; + # bpf drops reconcile as flush + re-add. await self._ubridge_apply_netem(host_ifc, nio.filters) + await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) return @@ -2172,6 +2186,78 @@ class DockerVM(BaseNode): if "No such file" not in str(e): raise + async def _ubridge_tc_capabilities(self): + """ + uBridge's tc-module feature report (``tc capabilities``), probed once + per uBridge process and cached — the reply shape is + ``netem=;ebpf=0|1;cbpf=0|1``. uBridge builds without the tc + module answer "Unknown module" and old builds without the command + answer "Unknown command": both mean no kernel filter support, cached + as an empty dict. + """ + + if self._ubridge_tc_caps is None: + caps = {} + if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): + try: + reply = await self._ubridge_hypervisor.send("tc capabilities") + except UbridgeError: + reply = [] + for entry in (reply[0] if reply else "").split(";"): + key, _, value = entry.partition("=") + if key: + caps[key.strip()] = value.strip() + self._ubridge_tc_caps = caps + return self._ubridge_tc_caps + + async def _ubridge_apply_bpf_drops(self, host_ifc, filters): + """ + Translate the NIO's ``bpf`` filter (one expression per line, any line + matching drops the packet) into cls_bpf classifiers on the veth host + end's clsact egress: ``tc bpf_drop`` compiles each line with libpcap + (same compiler as the relay's bpf filter) and drops matches with a + gact TC_ACT_SHOT. Reconcile is always flush + re-add — same-prio + resend replaces the filter node in uBridge, and the flush drops + lines the user removed. + + :param host_ifc: veth host-end interface name + :param filters: NIO filters dictionary + """ + + bpf = filters.get("bpf") + lines = [line.strip() for line in (bpf[0].split("\n") if bpf else []) if line.strip()] + if not lines and self._ubridge_tc_caps is None: + # Nothing to add and nothing was ever installed (no tc module + # probed yet) — skip touching an old uBridge entirely. + return + caps = await self._ubridge_tc_capabilities() + if not lines: + if caps: + await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') + return + if caps.get("cbpf") != "1": + raise DockerError( + "Packet filter 'bpf' on a kernel-datapath link needs a uBridge with cBPF support " + "(tc capabilities reports no cbpf); upgrade uBridge on this compute or keep the " + "link on the relay datapath" + ) + await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') + for offset, line in enumerate(lines): + try: + await self._ubridge_send( + 'tc bpf_drop add "{ifc}" {prio} "{expr}"'.format(ifc=host_ifc, prio=10 + offset, expr=line) + ) + except UbridgeError as e: + if "Cannot compile filter" not in str(e): + raise + # Mirror the relay path: a line that no longer compiles on + # this uBridge's libpcap is skipped with a warning instead + # of breaking the link (the controller already validated + # the syntax with tcpdump at create/update time). + message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {line}" + log.warning(message) + self.project.emit("log.warning", {"message": message}) + async def _remove_kernel_markers(self, host_ifc): """ Tear down every kernel marker anchored to *host_ifc*: the veth survives diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 53d9dc585..13e403af4 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -66,7 +66,6 @@ class TAPNIO(BaseModel): class BridgeNIOType(str, Enum): - bridge = "nio_bridge" @@ -74,17 +73,19 @@ class BridgeNIO(BaseModel): """ Kernel-datapath bridge Network Input/Output properties. The NIO tells the node to enslave its veth host end into the named kernel bridge instead of - wiring a uBridge UDP relay. Impairment filters with a tc netem equivalent - (delay, packet_loss, corrupt) run on the veth host end; the relay-only - kinds (frequency_drop, bpf) are rejected. Markers and packet capture are - served by uBridge's AF_PACKET modules on the veth host end. + wiring a uBridge UDP relay. Impairment filters run on the veth host end: + delay/packet_loss/corrupt as one tc netem qdisc, bpf as cls_bpf match-drop + classifiers (needs a uBridge reporting cbpf). frequency_drop still has no + kernel equivalent and is rejected. Markers and packet capture are served + by uBridge's AF_PACKET modules on the veth host end. """ type: BridgeNIOType bridge: str = Field(..., description="Kernel bridge name e.g. gns3a1b2c3d4e5") suspend: Optional[bool] = Field(None, description="Suspend the NIO") filters: Optional[dict] = Field( - None, description="Packet filters served by tc netem on the veth host end (delay/packet_loss/corrupt)" + None, + description="Packet filters served on the veth host end (delay/packet_loss/corrupt via netem, bpf via cls_bpf)", ) markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the veth host end)") diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index 57bcdba60..6a0337552 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -15,10 +15,12 @@ class FilterValidationError(Exception): pass -# Packet filters with no kernel-datapath equivalent (no tc netem mapping): -# they only run in the uBridge userspace relay, so a link carrying one is -# wired on the relay, and adding them to a kernel-datapath link is rejected. -KERNEL_UNSUPPORTED_FILTERS = frozenset({"frequency_drop", "bpf"}) +# Packet filters with no kernel-datapath equivalent yet: frequency_drop needs +# the eBPF stateful classifier (frozen in +# docs/design/ubridge-kernel-impairment-spec.md, part B — not delivered), +# so it still only runs in the uBridge userspace relay. bpf is served by the +# cBPF match-drop classifier (tc bpf_drop) when uBridge reports cbpf=1. +KERNEL_UNSUPPORTED_FILTERS = frozenset({"frequency_drop"}) def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Optional[str]]: diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 53f57d9a4..a467a21c9 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -28,6 +28,8 @@ from unittest.mock import MagicMock, call import pytest import pytest_asyncio +from unittest.mock import patch + from tests.utils import AsyncioMagicMock from gns3server.compute.docker import Docker @@ -88,11 +90,22 @@ def test_create_nio_bridge_accepts_netem_filters(vm): assert nio.filters == {"delay": [50, 5], "packet_loss": [10]} -@pytest.mark.parametrize("filters", [{"frequency_drop": [10]}, {"bpf": ["icmp"]}]) -def test_create_nio_bridge_rejects_relay_only_filters(vm, filters): +def test_create_nio_bridge_accepts_bpf_filters(vm): + """ + bpf expressions run on the kernel datapath as cls_bpf match-drop + classifiers (uBridge tc bpf_drop); only frequency_drop has no kernel + equivalent yet. + """ + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"bpf": ["icmp"]}}) + assert isinstance(nio, NIOBridge) + assert nio.filters == {"bpf": ["icmp"]} + + +def test_create_nio_bridge_rejects_relay_only_filters(vm): with pytest.raises(ComputeError, match="kernel-datapath"): - vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": filters}) + vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"frequency_drop": [10]}}) def test_create_nio_bridge_accepts_markers(vm): @@ -420,6 +433,114 @@ async def test_update_nio_kernel_rejects_relay_only_filters(vm): await vm.adapter_update_nio_binding(0, nio) +def _caps_hypervisor(cbpf="1"): + """A hypervisor mock whose direct send() answers `tc capabilities`.""" + hyp = MagicMock() + hyp.is_running.return_value = True + hyp.send = AsyncioMagicMock(return_value=[f"netem=delay;ebpf=0;cbpf={cbpf}"]) + return hyp + + +@pytest.mark.asyncio +async def test_update_nio_kernel_applies_bpf_drops(vm): + """ + bpf lines reconcile as flush + one bpf_drop add per line at successive + priorities (any line matching drops — OR semantics like the relay). + """ + + vm._ubridge_hypervisor = _caps_hypervisor() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"bpf": ["icmp\nudp port 53"]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc bpf_drop flush "{host_ifc}"') + vm._ubridge_send.assert_any_call(f'tc bpf_drop add "{host_ifc}" 10 "icmp"') + vm._ubridge_send.assert_any_call(f'tc bpf_drop add "{host_ifc}" 11 "udp port 53"') + + +@pytest.mark.asyncio +async def test_update_nio_kernel_flushes_bpf_drops_when_removed(vm): + + vm._ubridge_hypervisor = _caps_hypervisor() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"bpf": ["icmp"]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.reset_mock() + nio.filters = {} + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc bpf_drop flush "{host_ifc}"') + for c in vm._ubridge_send.call_args_list: + assert "bpf_drop add" not in c.args[0] + + +@pytest.mark.asyncio +async def test_apply_bpf_drops_requires_cbpf_capability(vm): + + vm._ubridge_hypervisor = _caps_hypervisor(cbpf="0") + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="upgrade uBridge"): + await vm._ubridge_apply_bpf_drops(host_ifc, {"bpf": ["icmp"]}) + + +@pytest.mark.asyncio +async def test_apply_bpf_drops_compile_failure_warns(vm): + """ + A line that compiles for the controller's tcpdump but not for this + uBridge's libpcap is skipped with a warning — mirrors the relay path. + """ + + vm._ubridge_hypervisor = _caps_hypervisor() + host_ifc, _ = vm._veth_names(0, 0) + + async def send(command): + if "bpf_drop add" in command: + raise UbridgeError("209-Cannot compile filter 'icmp': can't parse filter expression: syntax error") + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + with patch("gns3server.compute.docker.docker_vm.log.warning") as mock_log: + await vm._ubridge_apply_bpf_drops(host_ifc, {"bpf": ["icmp"]}) # must not raise + assert mock_log.called + + +@pytest.mark.asyncio +async def test_apply_bpf_drops_skips_old_ubridge_without_lines(vm): + """ + No bpf lines and no capability probe yet: nothing was ever installed on + an old uBridge, so it must not be touched at all (no unknown-module error). + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + await vm._ubridge_apply_bpf_drops(host_ifc, {}) + vm._ubridge_send.assert_not_called() + vm._ubridge_hypervisor.send.assert_not_called() + + +@pytest.mark.asyncio +async def test_ubridge_tc_capabilities_probed_once(vm): + + vm._ubridge_hypervisor = _caps_hypervisor() + caps = await vm._ubridge_tc_capabilities() + assert caps == {"netem": "delay", "ebpf": "0", "cbpf": "1"} + await vm._ubridge_tc_capabilities() + assert vm._ubridge_hypervisor.send.call_count == 1 + + @pytest.mark.asyncio async def test_update_nio_kernel_applies_netem(vm): diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index b9301c049..87770a86e 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -103,8 +103,9 @@ async def test_kernel_datapath_eligible_with_netem_filters(project): @pytest.mark.asyncio async def test_kernel_datapath_not_eligible_with_relay_only_filters(project): """ - frequency_drop and bpf only exist in the uBridge userspace relay — a - link carrying one stays on the relay. + frequency_drop only exists in the uBridge userspace relay (its eBPF + classifier is not delivered yet) — a link carrying one stays on the + relay. bpf is kernel-capable (cls_bpf match-drop) and eligible. """ link, node1, node2 = await _kernel_link(project) @@ -112,7 +113,7 @@ async def test_kernel_datapath_not_eligible_with_relay_only_filters(project): assert link._kernel_datapath_eligible(node1, node2) is False link._filters = {"bpf": ["icmp"]} - assert link._kernel_datapath_eligible(node1, node2) is False + assert link._kernel_datapath_eligible(node1, node2) is True @pytest.mark.asyncio @@ -267,8 +268,25 @@ async def test_update_relay_only_filters_rejected_on_kernel_link(project): with pytest.raises(ControllerError, match="kernel-datapath"): await link.update_filters({"frequency_drop": [10]}) - with pytest.raises(ControllerError, match="kernel-datapath"): - await link.update_filters({"bpf": ["icmp"]}) + + +@pytest.mark.asyncio +async def test_update_bpf_filters_accepted_on_kernel_link(project): + """ + bpf expressions ride the kernel NIO to BOTH endpoints (each end's + cls_bpf classifiers cover the traffic entering that container — the + same both-directions-once net effect as the relay's single filtered + bridge). + """ + + link, node1, node2 = await _kernel_link(project) + await link._prepare() + + await link.update_filters({"bpf": ["icmp\nudp port 53"]}) + assert link.filters == {"bpf": ["icmp\nudp port 53"]} + by_node = {entry[0].id: entry[3] for entry in (await link._prepare())} + assert by_node[node1.id]["filters"] == {"bpf": ["icmp\nudp port 53"]} + assert by_node[node2.id]["filters"] == {"bpf": ["icmp\nudp port 53"]} @pytest.mark.asyncio From b1eb0984bc5c71e05329892e0976c8d013f2360c Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 29 Sep 2026 18:14:23 +0800 Subject: [PATCH 010/101] fix(docker): bring the veth host end up before a relay attach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unified veth host end is born admin-down (carrier off until a link attaches), and libpcap cannot open a packet socket on an admin-down interface — its netlink promiscuous-mode transaction returns ENOENT, so 'bridge add_nio_ethernet' failed on every runtime relay link (and would on any relay link at container start). Bring the interface up for the attach; the carrier pass in the caller refines the state afterwards (a suspended NIO sets it back down). Verified live with a direct uBridge probe: attach on a down veth -> ENOENT, after 'link set up' -> OK. --- gns3server/compute/docker/docker_vm.py | 7 +++++++ tests/compute/docker/test_docker_kernel_datapath.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 63ca5a089..2c95b23ee 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1959,6 +1959,13 @@ class DockerVM(BaseNode): if host_ifc is not None and bridge_name not in self._bridges: await self._ubridge_send(f"bridge create {bridge_name}") self._bridges.add(bridge_name) + # libpcap cannot open a packet socket on an admin-down interface + # (its netlink promiscuous-mode transaction returns ENOENT), and + # the unified veth host end is born down (carrier off until a + # link attaches) — bring it up for the relay attach. The carrier + # pass in the caller refines the state afterwards (a suspended + # NIO sets it back down). + await self._ubridge_send(f'link set "{host_ifc}" up') await self._ubridge_send(f'bridge add_nio_ethernet {bridge_name} "{host_ifc}"') await self._ubridge_send( "bridge add_nio_udp {bridge_name} {lport} {rhost} {rport}".format( diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index a467a21c9..6cf1f2ca1 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -222,6 +222,9 @@ async def test_add_ubridge_connection_udp_relays_over_veth(vm): [ call.send(f'docker create_veth "{host_ifc}" "{vm._veth_names(0, 0)[1]}"'), call.send("bridge create bridge0"), + # the host end is born down; libpcap cannot attach to an + # admin-down interface, so the relay brings it up first + call.send(f'link set "{host_ifc}" up'), call.send(f'bridge add_nio_ethernet bridge0 "{host_ifc}"'), call.send("bridge add_nio_udp bridge0 4242 127.0.0.1 4343"), call.send("bridge start bridge0"), From d4b57d49fbdca2df49d06b12d8e24e5b3ed1f54c Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 29 Sep 2026 18:14:54 +0800 Subject: [PATCH 011/101] feat(docker): expose netem-extension packet filters on kernel links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface the netem extensions delivered in uBridge feature/tc-netem-ext as GNS3 filter types: rate (tc-style bandwidth string), reorder (needs delay), gemodel (Gilbert-Elliot bursty loss, exclusive with packet_loss), duplicate, seed and limit, plus the delay jitter distribution and the loss/dup correlation parameters. All are kernel-datapath only — the relay's packet-filter registry has no equivalent, so available_filters hides them on relay links, setting one on a created relay link returns 409, and the project-load path drops them with a warning (update_filters runs before the datapath is decided, so it cannot reject there). Validation mirrors the tc grammar: reorder requires delay, gemodel and packet_loss are mutually exclusive, a distribution requires jitter > 0, rate is integer+unit capped at 100gbit. The translation in DockerVM._netem_command_parts emits the frozen grammar order and gates extension keywords on the 'tc capabilities' netem token list (probed once per uBridge process; a plain delay/loss/corrupt/dup filter never probes, keeping old uBridge builds untouched; the correl suffix has no token and gates on 'rate' as the extension-build marker). Reconcile became reset-before-set: the kernel's netem change merges optional attributes (rate, correlation, reorder, corrupt, gemodel, distribution, seed keep their previous value when the new message omits them), so removing a parameter needs an explicit 'tc reset' — the bpf_drop filters are re-added right after in the same flow, so the clsact churn is invisible. The capability check runs before the reset so a failed update leaves the previous qdisc in place. e2e (isolated instance, kernel link): 24/24 — rate 512kbit adds exactly 2x the per-packet serialization to the RTT (45.5 ms for 1400-byte pings), paretonormal jitter + rate-cleared leak check, reorder gap 5, gemodel 50% bursty loss, duplicate, loss 30% correl 50%, seed 42 + limit 5000 visible in the qdisc dump, suspend/resume and node restart keep the filters, relay link rejects rate with 409, teardown leaves no netem. Full unit suite: 1916 passed. --- docs/design/ubridge-kernel-impairment-spec.md | 47 +++- docs/features/docker-kernel-datapath.md | 89 +++++-- gns3server/compute/base_manager.py | 14 ++ gns3server/compute/base_node.py | 11 + gns3server/compute/docker/docker_vm.py | 153 ++++++++++-- gns3server/controller/link.py | 88 ++++++- gns3server/controller/udp_link.py | 34 ++- gns3server/schemas/compute/nios.py | 19 +- gns3server/utils/packet_filter_validation.py | 235 +++++++++++++++--- .../docker/test_docker_kernel_datapath.py | 181 ++++++++++++++ tests/compute/test_base_node.py | 14 ++ tests/controller/test_kernel_datapath_link.py | 121 +++++++++ tests/utils/test_packet_filter_validation.py | 114 ++++++++- 13 files changed, 1012 insertions(+), 108 deletions(-) diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md index 52b36399a..356b3a5b4 100644 --- a/docs/design/ubridge-kernel-impairment-spec.md +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -4,14 +4,29 @@ See LICENSE file for licensing information. --> > Frozen requirements spec for the **uBridge** project. Delivery status: -> parts **A** (netem keyword extensions, uBridge `feature/tc-netem-ext`) and -> **C + D** (cBPF match-drop, full-restore idempotent `tc reset`, -> `tc capabilities` with `cbpf` probe — uBridge `feature/tc-bpf-drop`) are -> **delivered and integrated** on gns3-server branch -> `feat/docker-kernel-bpf-drop`. Part **B** (eBPF stateful classifier) is -> still pending. The originally shipped netem surface (delay/jitter/loss/ -> dup/corrupt) exists in uBridge 1.2.3+ and is verified against gns3-server -> branch `feat/docker-kernel-filters`. +> parts **A** (netem keyword extensions, uBridge `feature/tc-netem-ext`), **C** +> (cBPF match-drop) and **D + E** (full-restore idempotent `tc reset`, +> `tc capabilities` — uBridge `feature/tc-bpf-drop`) are **delivered and +> integrated** on gns3-server branches `feat/docker-kernel-bpf-drop` (C/D/E) +> and `feat/docker-kernel-netem-ext` (A: rate/reorder/gemodel/duplicate/ +> seed/limit/distributions/correlation exposed as GNS3 filter types). +> Part **B** (eBPF stateful classifier) is delivered in the uBridge +> `feature/tc-precision` branch but **not yet consumed** by gns3-server. +> The originally shipped netem surface (delay/jitter/loss/dup/corrupt) +> exists in uBridge 1.2.3+ and is verified against gns3-server branch +> `feat/docker-kernel-filters`. +> +> **Deviation found during integration (part A/D):** the delivered +> `tc netem set` uses NLM_F_REPLACE, but the *kernel's* netem change merges +> optional attributes — rate, correlation, reorder, corrupt, gemodel, +> distribution and seed keep their previous value when the new message +> omits them, so "netem set atomically replaces" (D's reconcile +> assumption) does not hold for parameter *removals*. gns3-server works +> around it by sending `tc reset` before every `netem set` (the bpf_drop +> filters are re-added right after in the same flow, so clsact churn is +> invisible). If uBridge later emits explicit zero/absent clears for every +> optional attribute, the server-side reset-before-set can be dropped; the +> two behaviours are compatible either way. # uBridge kernel impairment: tc netem extensions + eBPF classifiers @@ -272,11 +287,17 @@ corresponding filter types per capability. ## gns3-server alignment (informational — not uBridge scope) -* P6a types map 1:1 onto new GNS3 filter entries (rate, reorder, gemodel…), - translated in `DockerVM._ubridge_apply_netem`. -* `frequency_drop` / `bpf` move out of `KERNEL_UNSUPPORTED_FILTERS` once - `tc capabilities` reports `ebpf=1` / `cbpf=1` (per-compute probe, cached); - without the capability the link stays relay-wired exactly as today. +* **Done** (`feat/docker-kernel-netem-ext`): the P6a types map 1:1 onto new + GNS3 filter entries (rate, reorder, gemodel, duplicate, seed, limit, plus + the delay `distribution` and loss/dup `correlation` parameters), translated + in `DockerVM._ubridge_apply_netem`; extension keywords are gated on the + `tc capabilities` netem token list (old uBridge builds are never probed + for the original surface). +* **Done** (`feat/docker-kernel-bpf-drop`): `bpf` runs as cBPF match-drop + (`KERNEL_UNSUPPORTED_FILTERS` shrank to `frequency_drop`). +* `frequency_drop` moves out of `KERNEL_UNSUPPORTED_FILTERS` once the eBPF + classifier (part B, uBridge `feature/tc-precision`) is consumed on the + server side, keyed on `ebpf=1` per compute. * Every veth end owns one qdisc + its filters: per-direction impairment is an architectural freebie to expose later (API `direction` field), aligned with marker `dir` semantics. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 38764531b..fad28bbb7 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -24,13 +24,13 @@ Frames never leave the kernel. | Link | uBridge bridge, TAP fd ↔ UDP, userspace copy | per-link kernel bridge `gns3{link_id[:11]}`, zero-copy | | Relay latency | ~0.3 ms RTT | ~0.05 ms RTT (measured) | | Link attach at runtime | yes (relay attaches the pre-existing TAP) | yes (brctl addif / add_nio_ethernet — interface untouched) | -| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt); bpf as cls_bpf match-drop classifiers; frequency_drop stays relay-only (eBPF classifier pending) | +| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt / duplicate + the netem extensions rate, reorder, gemodel, seed, limit, jitter distributions, loss correlation); bpf as cls_bpf match-drop classifiers; frequency_drop stays relay-only (eBPF classifier pending) | | Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | -The work landed in six stages on branch stack `feat/docker-kernel-datapath` → +The work landed in seven stages on branch stack `feat/docker-kernel-datapath` → `feat/docker-kernel-capture` → `feat/docker-kernel-markers` → `feat/docker-veth-everywhere` → `feat/docker-kernel-filters` → -`feat/docker-kernel-bpf-drop`: +`feat/docker-kernel-bpf-drop` → `feat/docker-kernel-netem-ext`: 1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based suspend, crash-safe reconciliation. @@ -49,6 +49,14 @@ The work landed in six stages on branch stack `feat/docker-kernel-datapath` → uBridge pcap-compiles the line to classic BPF, the kernel migrates it to eBPF internally — no CAP_BPF needed). Requires a uBridge reporting `cbpf=1` via `tc capabilities`; older builds get a clear upgrade error. +7. **netem extensions** — the netem-extension surface delivered in uBridge + `feature/tc-netem-ext` is exposed as GNS3 filter types: `rate`, + `reorder`, `gemodel`, `duplicate`, `seed`, `limit`, plus the delay + `distribution` and loss/dup `correlation` parameters. Kernel-datapath + links only (the relay has no equivalent); capability-gated per uBridge + process. Reconcile became reset-before-set: the kernel's netem replace + merges optional attributes, so removing a parameter needs the explicit + reset. ## Adapter interface types @@ -171,9 +179,15 @@ ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): | GNS3 filter | kernel implementation | Notes | |---|---|---| -| `delay [ms, jitter]` | netem `delay X jitter Y` | jitter 0 omitted | -| `packet_loss [%]` | netem `loss P` | per direction (see below) | +| `delay [ms, jitter, dist]` | netem `delay X jitter Y [distribution D]` | jitter 0 omitted; dist ∈ uniform/normal/pareto/paretonormal (kernel-only parameter) | +| `packet_loss [%, correl]` | netem `loss P [correl C]` | per direction (see below); correlation is a kernel-only parameter | | `corrupt [%]` | netem `corrupt P` | | +| `duplicate [%, correl]` | netem `dup P [correl C]` | kernel-only type | +| `rate ["512kbit"]` | netem `rate B` | tc-style integer+unit (bit/kbit/mbit/gbit/bps/kbps/mbps, ≤100gbit); kernel-only type | +| `reorder [%, correl, gap]` | netem `reorder P [correl C] [gap G]` | requires `delay`; kernel-only type | +| `gemodel [p, r, 1-h]` | netem `loss gemodel P R H` | Gilbert-Elliot bursty loss; mutually exclusive with `packet_loss`; kernel-only type | +| `seed [u32]` | netem `seed N` | reproducible random draws; kernel-only type | +| `limit [pkts]` | netem `limit N` | queue depth above the 1000 default (rate+delay BDP); kernel-only type | | `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | | `frequency_drop` | — not yet (eBPF stateful classifier spec'd: [ubridge-kernel-impairment-spec](../design/ubridge-kernel-impairment-spec.md), part B pending) | 409 on kernel links; relay fallback | @@ -184,32 +198,45 @@ Semantics: so every direction of the link is impaired exactly once — the same net effect as the relay, where both directions cross the single filtered bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` - measures ≈51 % round-trip (1 − 0.7²). -* **Reconcile = full re-apply.** `netem set` is an atomic replace (NLM_F_REPLACE), - so every NIO update just rebuilds the qdisc from the current filters; no - per-parameter diffing. An empty filter set detaches it (`tc reset`, - ENOENT-tolerated). + measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the + per-packet serialization time to the RTT. +* **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES + optional attributes (rate, correlation, reorder, corrupt, gemodel, + distribution: an absent attr keeps its previous value), so re-applying a + filter set with a parameter *removed* would silently keep the old value. + Every apply therefore starts with `tc reset` (also drops clsact and its + bpf_drop filters — re-added right after in the same flow) followed by one + `netem set` built from the current filters; no per-parameter diffing. An + empty filter set is the reset alone (ENOENT-tolerated). +* **Extension gating.** The netem-extension keywords (rate, reorder, + gemodel, dist, seed, limit, and the correl suffixes) probe + `tc capabilities` once per uBridge process and require the tokens; + a plain delay/loss/corrupt/dup filter never probes — an old uBridge + serves the original surface untouched. +* **kernel-only vs relay-only.** The extension types have no uBridge relay + equivalent: `available_filters` hides them on relay links, setting one on + a created relay link returns 409, and a project loaded with such filters + on a link that cannot be kernel-wired drops them with a warning (invalid + filters are dropped the same way at load). Symmetrically, + `frequency_drop` stays relay-only. +* **Validation** mirrors the tc grammar: `reorder` requires `delay`, + `gemodel` and `packet_loss` are mutually exclusive (both map to the netem + loss keyword), `distribution` requires jitter > 0, rate must be integer + + unit ≤ 100gbit. * **Node restart** restores the qdisc from the NIO (like capture and markers); **link deletion** detaches it explicitly — the veth survives the link and an orphaned qdisc would keep impairing the next one. * **Suspend** keeps the qdisc (carrier-driven loss); the first packet after resume takes one extra delay interval (known netem idle-baseline behaviour) and steady state is exact. -* A link carrying `frequency_drop` is wired on the **relay** (its eBPF - stateful classifier is not delivered yet); `available_filters` hides it on - kernel links and setting it returns 409 with a clear message. The remaining - kernel-side roadmap is frozen in - [docs/design/ubridge-kernel-impairment-spec.md](../design/ubridge-kernel-impairment-spec.md) - (eBPF stateful classifier for `frequency_drop` + quota/window/flow modes, - plus netem keyword extensions: rate, reorder, gemodel loss, jitter - distributions — parts C and D of that spec are delivered). * **bpf reconcile = flush + re-add** (`tc bpf_drop flush`, then one add per line at priorities 10, 11, …). `tc reset` (link delete / filter clear) is the full restore in uBridge: filters → clsact → root qdisc, idempotent. * **Observability caveat:** clsact drops happen before the AF_PACKET tap points — markers/captures on the same veth do not observe cls_bpf-dropped frames (on the relay, filter-vs-mark ordering determined visibility - instead). + instead). Also note `tc qdisc show` cannot print a distribution's name + (the kernel stores only the sampled table). ## uBridge command surface @@ -222,7 +249,9 @@ brctl create / delete / addif / delif / show link set up|down capture start_kernel / stop_kernel marker add_kernel / delete_kernel / enable_kernel -tc netem set [delay ] [jitter ] [loss <%>] [dup <%>] [corrupt <%>] +tc netem set [delay ] [jitter ] [loss <%> [correl <%>] | loss gemodel

<1-h>] + [dup <%> [correl <%>]] [corrupt <%>] [reorder <%> [correl <%>] [gap ]] + [rate ] [limit ] [distribution uniform|normal|pareto|paretonormal] [seed ] tc bpf_drop add "" / tc bpf_drop flush tc reset # full restore: filters -> clsact -> root qdisc, idempotent tc capabilities # "netem=;ebpf=0|1;cbpf=0|1", probed once per process @@ -266,3 +295,23 @@ recreate leaves no residual filters; `frequency_drop` still 409; `available_filters` shows `bpf` and hides `frequency_drop`. Unit tests: `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. + +netem extensions (same setup, 24/24 checks): `rate 512kbit` on both veths, +1400-byte pings measure 45.5 ms RTT = baseline + 2× the 22.2 ms +serialization (exact); `delay 100 20 paretonormal` applies delay+jitter and +clears the previous rate (leak detector for the kernel's merge-on-replace); +`reorder 25 gap 5` visible in the qdisc dump and traffic passes; +`gemodel 100/0/30` = 50 % bursty round-trip loss with the previous reorder +cleared; `duplicate 50` link alive; `packet_loss 30 correl 50` shows +`loss 30% 50%`; `seed 42` + `limit 5000` both visible in the dump, RTT +exactly 2×50 ms; suspend/resume and node restart keep the filters; a +docker↔ethernet-switch relay link rejects `rate` with 409 and its +`available_filters` hides the kernel-only types while the kernel link shows +them all (minus frequency_drop); `reorder` without `delay` and +`gemodel`+`packet_loss` are 409s; teardown leaves no netem on either veth. +Also verified live: a relay attach (`bridge add_nio_ethernet`) on the +admin-down veth host end fails in libpcap — the relay path now brings the +interface up before attaching. Unit tests: +`tests/utils/test_packet_filter_validation.py` (P6a class), +`tests/compute/docker/test_docker_kernel_datapath.py`, +`tests/controller/test_kernel_datapath_link.py`. diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index 30a6fc1c3..0205003fd 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -356,6 +356,20 @@ class BaseManager: except OSError as e: raise ComputeError(f"Could not create an UDP connection to {rhost}:{rport}: {e}") nio = NIOUDP(lport, rhost, rport) + # The netem-extension filters (rate, reorder, gemodel…) have no + # uBridge relay equivalent — reject them here (the controller + # already keeps them off relay links; this is the second guard + # for direct compute API use, mirroring the kernel-NIO guard). + from gns3server.utils.packet_filter_validation import kernel_only_features + + kernel_only = kernel_only_features(nio_settings.get("filters") or {}) + if kernel_only: + raise ComputeError( + "Packet filter(s) {} only run on a kernel-datapath link " + "(tc netem on the veth host end); the uBridge relay has no equivalent".format( + ", ".join(sorted(kernel_only)) + ) + ) nio.filters = nio_settings.get("filters", {}) nio.markers = nio_settings.get("markers", {}) nio.suspend = nio_settings.get("suspend", False) diff --git a/gns3server/compute/base_node.py b/gns3server/compute/base_node.py index 74f6a028c..18f4cfdc3 100644 --- a/gns3server/compute/base_node.py +++ b/gns3server/compute/base_node.py @@ -1116,6 +1116,17 @@ class BaseNode: :param filters: Array of filter dictionary """ + # The netem-extension filters (rate, reorder, gemodel…) only exist on + # the kernel datapath. The controller keeps them off relay links; + # this is the second guard for direct compute API use. + from gns3server.utils.packet_filter_validation import kernel_only_features + + kernel_only = kernel_only_features(filters) + if kernel_only: + raise NodeError( + "Packet filter(s) {} only run on a kernel-datapath link (tc netem on the " + "veth host end); the uBridge relay has no equivalent".format(", ".join(sorted(kernel_only))) + ) await self._ubridge_send("bridge reset_packet_filters " + bridge_name) for packet_filter in self._build_filter_list(filters): cmd = f"bridge add_packet_filter {bridge_name} {packet_filter}" diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 2c95b23ee..641b46f10 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -2156,35 +2156,143 @@ class DockerVM(BaseNode): return await super()._ubridge_enable_marker_filter(anchor, name, state) + @staticmethod + def _netem_command_parts(filters): + """ + Build the ``tc netem set`` keyword sequence (in the frozen grammar + order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, + limit, distribution, seed) plus the set of capability tokens the + sequence needs beyond the original netem surface. Original-surface + keywords (delay, jitter, loss, corrupt, dup) need no probe; the + extension tokens (rate, reorder, gemodel, dist, seed, limit) are + checked against ``tc capabilities`` by the caller. ``correl`` has no + token of its own in the capabilities list, so it gates on "rate" as + the netem-extension build marker. + + :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) + + :returns: (keyword sequence, required extension tokens) + """ + + def values_of(key): + # tolerate the legacy bare-value shape ({"packet_loss": 10}) + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + parts = [] + ext_tokens = set() + + delay = values_of("delay") + if delay: + parts.append(f"delay {int(delay[0])}") + if len(delay) > 1 and int(delay[1]): + parts.append(f"jitter {int(delay[1])}") + + gemodel = values_of("gemodel") + if gemodel: + segment = f"loss gemodel {int(gemodel[0])}" + if len(gemodel) > 1: + segment += f" {int(gemodel[1])}" + if len(gemodel) > 2: + segment += f" {int(gemodel[2])}" + parts.append(segment) + ext_tokens.add("gemodel") + else: + loss = values_of("packet_loss") + if loss and int(loss[0]): + parts.append(f"loss {int(loss[0])}") + if len(loss) > 1 and int(loss[1]): + parts.append(f"correl {int(loss[1])}") + ext_tokens.add("rate") # correl: extension build marker + + duplicate = values_of("duplicate") + if duplicate and int(duplicate[0]): + parts.append(f"dup {int(duplicate[0])}") + if len(duplicate) > 1 and int(duplicate[1]): + parts.append(f"correl {int(duplicate[1])}") + ext_tokens.add("rate") # correl: extension build marker + + corrupt = values_of("corrupt") + if corrupt and int(corrupt[0]): + parts.append(f"corrupt {int(corrupt[0])}") + + reorder = values_of("reorder") + if reorder: + segment = f"reorder {int(reorder[0])}" + if len(reorder) > 1 and int(reorder[1]): + segment += f" correl {int(reorder[1])}" + if len(reorder) > 2 and int(reorder[2]): + segment += f" gap {int(reorder[2])}" + parts.append(segment) + ext_tokens.add("reorder") + + rate = values_of("rate") + if rate and str(rate[0]).strip(): + parts.append(f"rate {str(rate[0]).strip()}") + ext_tokens.add("rate") + + limit = values_of("limit") + if limit and int(limit[0]): + parts.append(f"limit {int(limit[0])}") + ext_tokens.add("limit") + + if delay and len(delay) > 2 and str(delay[2]).strip().lower() not in ("", "uniform"): + # "uniform" is the kernel default — emitting nothing keeps the + # command compatible with the original netem surface. + parts.append(f"distribution {str(delay[2]).strip().lower()}") + ext_tokens.add("dist") + + seed = values_of("seed") + if seed and int(seed[0]): + # 0 is the "disabled" convention everywhere else — treat it so + # here too (the controller's inactive-filter pass drops it). + parts.append(f"seed {int(seed[0])}") + ext_tokens.add("seed") + + return parts, ext_tokens + async def _ubridge_apply_netem(self, host_ifc, filters): """ Translate the NIO's impairment filters into one tc netem qdisc on the - veth host end. The supported types (delay, packet_loss, corrupt) merge - into a single qdisc — its egress covers the traffic entering this - container, and with both link endpoints applying theirs, every - direction of the link is impaired exactly once (the same net effect - as the relay, where both directions cross the single filtered bridge). - ``netem set`` atomically replaces whatever qdisc was attached, so a - full re-apply on every NIO update *is* the incremental reconcile. An - empty filter set detaches the qdisc (``tc reset``). + veth host end. The netem-expressible types (delay, packet_loss, + corrupt and the netem extensions rate/reorder/gemodel/duplicate/ + seed/limit) merge into a single qdisc — its egress covers the traffic + entering this container, and with both link endpoints applying + theirs, every direction of the link is impaired exactly once (the + same net effect as the relay, where both directions cross the single + filtered bridge). + + The kernel's netem replace MERGES optional attributes (rate, + correlation, reorder, corrupt, gemodel, distribution: absent attr = + previous value kept), so re-applying a filter set with a parameter + *removed* would silently keep the old value. A ``tc reset`` before + every ``netem set`` makes the re-apply a true reconcile (reset also + drops clsact and its bpf_drop filters — the caller re-applies them + right after in the same flow). An empty filter set is the reset + alone. :param host_ifc: veth host-end interface name :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) """ - parts = [] - delay = filters.get("delay") - if delay: - parts.append(f"delay {int(delay[0])}") - if len(delay) > 1 and int(delay[1]): - parts.append(f"jitter {int(delay[1])}") - for filter_type, netem_kw in (("packet_loss", "loss"), ("corrupt", "corrupt")): - values = filters.get(filter_type) - if values and int(values[0]): - parts.append(f"{netem_kw} {int(values[0])}") - if parts: - await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) - return + parts, ext_tokens = self._netem_command_parts(filters) + if parts and ext_tokens: + # Extension keywords need the netem-extension uBridge. Plain + # delay/loss/corrupt/dup never probes — an old uBridge serves + # the original surface untouched (mirrors bpf_drop's guard). + # Checked before the reset so a failed update leaves the + # previously applied qdisc in place. + caps = await self._ubridge_tc_capabilities() + missing = ext_tokens - set(caps.get("netem", "").split(",")) + if missing: + raise DockerError( + "Packet filter(s) using {} need a uBridge with the netem extensions " + "(tc capabilities reports '{}'); upgrade uBridge on this compute".format( + ", ".join(sorted(missing)), caps.get("netem") or "no tc support" + ) + ) try: await self._ubridge_send(f'tc reset "{host_ifc}"') except UbridgeError as e: @@ -2192,6 +2300,9 @@ class DockerVM(BaseNode): # clean. Everything else is a real failure and must surface. if "No such file" not in str(e): raise + if not parts: + return + await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) async def _ubridge_tc_capabilities(self): """ diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index fdfaa1162..2bfa1dc32 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -26,6 +26,7 @@ from gns3server.config import Config from gns3server.utils.packet_filter_validation import ( FilterValidationError, filter_inactive_filters, + kernel_only_features, validate_all_filters, ) @@ -51,16 +52,22 @@ FILTERS = [ { "type": "packet_loss", "name": "Packet loss", - "description": "The percentage represents the chance for a packet to be lost", - "parameters": [{"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}], + "description": "The percentage represents the chance for a packet to be lost. The optional correlation " + "(kernel-datapath links only) makes consecutive losses dependent, like a bursty real network", + "parameters": [ + {"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "Correlation", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + ], }, { "type": "delay", "name": "Delay", - "description": "Delay packets in milliseconds. You can add jitter in milliseconds (+/-) of the delay", + "description": "Delay packets in milliseconds. You can add jitter in milliseconds (+/-) of the delay, and " + "(kernel-datapath links only) shape the jitter with a distribution", "parameters": [ {"name": "Latency", "minimum": 1, "maximum": 32767, "unit": "ms", "type": "int"}, {"name": "Jitter (-/+)", "minimum": 0, "maximum": 32767, "unit": "ms", "type": "int"}, + {"name": "Distribution", "type": "str", "unit": "uniform|normal|pareto|paretonormal"}, ], }, { @@ -75,6 +82,60 @@ FILTERS = [ "description": "This filter will drop any packet matching a BPF expression. Put one expression per line", "parameters": [{"name": "Filters", "type": "text"}], }, + # Everything below runs on kernel-datapath links only (tc netem on the + # veth host end; the uBridge relay has no equivalent). + { + "type": "rate", + "name": "Bandwidth limit", + "description": "Shape the link to a maximum bandwidth, tc-style value (kernel-datapath links only)", + "parameters": [{"name": "Rate", "type": "str", "unit": "e.g. 512kbit, 10mbit"}], + }, + { + "type": "reorder", + "name": "Reorder", + "description": "The percentage represents the chance for a packet to be reordered (held until a later " + "packet passes it). Requires the delay filter; kernel-datapath links only", + "parameters": [ + {"name": "Reorder", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "Correlation", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "Gap", "minimum": 1, "maximum": 1000, "type": "int", "unit": "packets"}, + ], + }, + { + "type": "gemodel", + "name": "Gilbert-Elliot loss", + "description": "Bursty loss model: p is the loss chance in the bad state, r in the good state, 1-h the " + "chance of moving from good to bad. Mutually exclusive with packet loss; kernel-datapath links only", + "parameters": [ + {"name": "p (bad-state loss)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "r (good-state loss)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "1-h (good-to-bad)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + ], + }, + { + "type": "duplicate", + "name": "Duplicate", + "description": "The percentage represents the chance for a packet to be duplicated; the optional " + "correlation makes consecutive duplicates dependent (kernel-datapath links only)", + "parameters": [ + {"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "Correlation", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + ], + }, + { + "type": "seed", + "name": "Random seed", + "description": "Make the netem random draws (loss, duplication, jitter) reproducible for repeated " + "experiments (kernel-datapath links only)", + "parameters": [{"name": "Seed", "minimum": 0, "maximum": 4294967295, "type": "int", "unit": ""}], + }, + { + "type": "limit", + "name": "Queue limit", + "description": "Queue depth of the impairment qdisc in packets — raise it above the 1000 default when " + "combining a low rate with a long delay (kernel-datapath links only)", + "parameters": [{"name": "Limit", "minimum": 1, "maximum": 1000000, "type": "int", "unit": "packets"}], + }, ] @@ -247,6 +308,21 @@ class Link: ", ".join(sorted(unsupported)) ) ) + elif self._created: + # The reverse guard: netem-extension filters have no relay + # equivalent. Only enforced on created links — while loading a + # project the datapath is not decided yet (a link that will be + # wired on the kernel datapath must accept them), and the relay + # prepare path drops what it cannot run with a warning. + conflicts = kernel_only_features(new_filters) + if conflicts: + raise ControllerError( + "Packet filter(s) {} only run on a kernel-datapath link (tc netem on the " + "veth host end); this link is wired on the uBridge relay — delete and " + "recreate the link to switch it to the kernel datapath".format( + ", ".join(sorted(conflicts)) + ) + ) if new_filters != self.filters: self._filters = new_filters @@ -652,7 +728,11 @@ class Link: from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS return [f for f in FILTERS if f["type"] not in KERNEL_UNSUPPORTED_FILTERS] - return FILTERS + # Relay links: hide the netem-extension types the relay cannot run + # (they would be rejected with 409 on update). + from gns3server.utils.packet_filter_validation import KERNEL_ONLY_FILTERS + + return [f for f in FILTERS if f["type"] not in KERNEL_ONLY_FILTERS] return [] def _get_filter_node(self): diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 494bfc976..e943dba46 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -22,6 +22,7 @@ import logging from gns3server.config import Config from gns3server.utils.packet_filter_validation import ( KERNEL_UNSUPPORTED_FILTERS, + split_kernel_only_features, validate_bpf_syntax, ) @@ -111,10 +112,23 @@ class UDPLink(Link): kernel = self.kernel_datapath if kernel: return self._filters, self._filters + # Relay datapath: the netem-extension filters (rate, reorder, gemodel…) + # have no relay equivalent. update_filters rejects them on created + # relay links, but a project loaded with such filters on a link that + # cannot be kernel-wired (cross-compute, non-docker node, relay-only + # filter mix…) reaches this point — drop what cannot run with a + # warning, exactly like invalid filters are dropped at load time. + relay_filters, dropped = split_kernel_only_features(self.get_active_filters()) + if dropped: + log.warning( + "Link %s: dropping packet filter(s) %s — no kernel datapath available on this link (uBridge relay wiring)", + self._id, + ", ".join(sorted(dropped)), + ) filter_node = self._get_filter_node() return ( - self.get_active_filters() if filter_node == node1 else {}, - self.get_active_filters() if filter_node == node2 else {}, + relay_filters if filter_node == node1 else {}, + relay_filters if filter_node == node2 else {}, ) def _markers_for_node(self, node): @@ -152,10 +166,11 @@ class UDPLink(Link): Whether this link can be wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge) instead of the uBridge UDP relay. Impairment filters with a tc netem equivalent (delay, - packet_loss, corrupt) are served on the veth host end; the others - (frequency_drop, bpf) only exist in the uBridge relay and disqualify - the kernel path. Capture and markers are served by uBridge's - AF_PACKET modules on the veth host end. + packet_loss, corrupt, bpf via cls_bpf, and the netem extensions + rate/reorder/gemodel/duplicate/seed/limit) are served on the veth + host end; frequency_drop only exists in the uBridge relay and + disqualifies the kernel path. Capture and markers are served by + uBridge's AF_PACKET modules on the veth host end. Docker adapters are born as veth pairs (unified interface), so the datapath is a runtime decision — links attach to running containers @@ -214,9 +229,10 @@ class UDPLink(Link): # Markers ride the NIO like on the relay datapath, routed by # capture node; they attach to the veth host end via uBridge's # AF_PACKET marker module instead of a relay `mark` filter. - # Filters (delay/packet_loss/corrupt — eligibility guarantees no - # other type is present) become one tc netem qdisc per veth host - # end, pushed to both endpoints. + # Filters (delay/packet_loss/corrupt plus the netem extensions + # rate/reorder/gemodel/duplicate/seed/limit — eligibility + # guarantees frequency_drop is absent) become one tc netem qdisc + # per veth host end, pushed to both endpoints. bridge_name = "gns3" + self._id.replace("-", "")[:11] node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True) node1_markers, node2_markers = self._get_node_markers(node1, node2) diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 13e403af4..cfc9fd43f 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -35,7 +35,11 @@ class UDPNIO(BaseModel): rhost: str = Field(..., description="Remote host") rport: int = Field(..., gt=0, le=65535, description="Remote port") suspend: Optional[bool] = Field(None, description="Suspend the NIO") - filters: Optional[dict] = Field(None, description="Packet filters") + filters: Optional[dict] = Field( + None, + description="Packet filters (relay-supported types only: delay/packet_loss/corrupt/frequency_drop/bpf — " + "the netem extensions need a kernel-datapath NIO)", + ) markers: Optional[dict] = Field(None, description="Traffic-insight markers") @@ -74,10 +78,12 @@ class BridgeNIO(BaseModel): Kernel-datapath bridge Network Input/Output properties. The NIO tells the node to enslave its veth host end into the named kernel bridge instead of wiring a uBridge UDP relay. Impairment filters run on the veth host end: - delay/packet_loss/corrupt as one tc netem qdisc, bpf as cls_bpf match-drop - classifiers (needs a uBridge reporting cbpf). frequency_drop still has no - kernel equivalent and is rejected. Markers and packet capture are served - by uBridge's AF_PACKET modules on the veth host end. + delay/packet_loss/corrupt and the netem extensions (rate, reorder, + gemodel, duplicate, seed, limit, jitter distributions, loss correlation) + as one tc netem qdisc, bpf as cls_bpf match-drop classifiers (needs a + uBridge reporting cbpf). frequency_drop still has no kernel equivalent + and is rejected. Markers and packet capture are served by uBridge's + AF_PACKET modules on the veth host end. """ type: BridgeNIOType @@ -85,7 +91,8 @@ class BridgeNIO(BaseModel): suspend: Optional[bool] = Field(None, description="Suspend the NIO") filters: Optional[dict] = Field( None, - description="Packet filters served on the veth host end (delay/packet_loss/corrupt via netem, bpf via cls_bpf)", + description="Packet filters served on the veth host end (delay/packet_loss/corrupt and the netem extensions " + "via one tc netem qdisc, bpf via cls_bpf)", ) markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the veth host end)") diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index 6a0337552..7f2f6d0e4 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -3,8 +3,9 @@ Packet filter parameter validation utilities. """ import logging +import re import subprocess -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set, Tuple log = logging.getLogger(__name__) @@ -22,6 +23,29 @@ class FilterValidationError(Exception): # cBPF match-drop classifier (tc bpf_drop) when uBridge reports cbpf=1. KERNEL_UNSUPPORTED_FILTERS = frozenset({"frequency_drop"}) +# Packet filters with no uBridge *relay* equivalent: the tc netem extensions +# (uBridge feature/tc-netem-ext) serve them on the kernel datapath only. The +# relay's packet-filter registry only knows frequency_drop / packet_loss / +# delay / corrupt / bpf / mark. +KERNEL_ONLY_FILTERS = frozenset({"rate", "reorder", "gemodel", "duplicate", "seed", "limit"}) + +# Jitter distributions embedded in the netem-extension uBridge (tc_netem_dist). +NETEM_DISTRIBUTIONS = ("uniform", "normal", "pareto", "paretonormal") + +# Bandwidth values follow the tc netem grammar: integer + unit, capped at +# 100gbit (uBridge NETEM_RATE_MAX_BPS). bps-family units are bytes/s. +_RATE_RE = re.compile(r"^(\d+)(bit|kbit|mbit|gbit|bps|kbps|mbps)$", re.IGNORECASE) +_RATE_UNITS = { + "bit": 1, + "kbit": 1000, + "mbit": 1000**2, + "gbit": 1000**3, + "bps": 8, + "kbps": 8000, + "mbps": 8000 * 1000, +} +_RATE_MAX_BITS = 100 * 1000**3 + def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Optional[str]]: """ @@ -73,6 +97,64 @@ def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Optional[str]]: return {"valid": False, "error": f"BPF validation error: {e!s}"} +def _validate_distribution_value(value: Any, name: str) -> str: + """ + Validate a netem jitter distribution name (delay filter, 3rd parameter). + Returns the normalized (lowercase) name; an empty value is treated as + "not set" and returns "". + """ + + if not isinstance(value, str): + raise FilterValidationError(f"{name} must be one of {', '.join(NETEM_DISTRIBUTIONS)} (or empty), got: {value}") + value = value.strip().lower() + if not value: + return "" # empty = not set + if value not in NETEM_DISTRIBUTIONS: + raise FilterValidationError(f"{name} must be one of {', '.join(NETEM_DISTRIBUTIONS)} (or empty), got: {value}") + return value + + +def _validate_rate_value(value: Any, name: str) -> None: + """ + Validate a tc-style bandwidth value (rate filter), e.g. "512kbit". + """ + + if not isinstance(value, str): + raise FilterValidationError(f"{name} must be a string like '512kbit', got: {value}") + match = _RATE_RE.match(value.strip()) + if not match: + raise FilterValidationError( + f"{name} must be an integer plus a unit (bit, kbit, mbit, gbit, bps, kbps, mbps), got: {value}" + ) + if int(match.group(1)) * _RATE_UNITS[match.group(2).lower()] > _RATE_MAX_BITS: + raise FilterValidationError(f"{name} exceeds the 100gbit maximum, got: {value}") + + +def _validate_int_parameter(filter_type: str, rules: dict, i: int, value: Any) -> None: + """ + Integer parameter validation: type coercion plus range check. + """ + + try: + if isinstance(value, str): + value = value.strip() + int_value = int(value) + else: + int_value = int(value) + except (ValueError, TypeError): + raise FilterValidationError( + f"{filter_type} parameter {rules['names'][i]} must be an integer, got: {value}" + ) + + # Range validation + min_val, max_val = rules["ranges"][i] + if int_value < min_val or int_value > max_val: + raise FilterValidationError( + f"{filter_type} parameter {rules['names'][i]} must be between " + f"{min_val} and {max_val} {rules['units'][i]}, got: {int_value}" + ) + + def validate_filter_parameters(filter_type: str, values: List[Any]) -> None: """ Validate packet filter parameters. @@ -85,33 +167,73 @@ def validate_filter_parameters(filter_type: str, values: List[Any]) -> None: FilterValidationError: If parameters are invalid """ - # Define validation rules based on ubridge implementation + # Define validation rules based on ubridge implementation. params_count is + # (min, max): trailing parameters are optional (clients sending only the + # first N fields keep working). String parameters are declared per index + # in "string_params" with their own validators. VALIDATION_RULES: Dict[str, Dict[str, Any]] = { "frequency_drop": { - "params_count": 1, + "params_count": (1, 1), "ranges": [(-1, 32767)], # min, max "names": ["Frequency"], "units": ["th packet"], }, - "packet_loss": {"params_count": 1, "ranges": [(0, 100)], "names": ["Chance"], "units": ["%"]}, - "delay": { - "params_count": 2, # latency, jitter - "ranges": [(1, 32767), (0, 32767)], # ubridge rejects latency <= 0 - "names": ["Latency", "Jitter"], - "units": ["ms", "ms"], + "packet_loss": { + # 2nd parameter (loss correlation) needs the netem-extension + # uBridge and only runs on the kernel datapath. + "params_count": (1, 2), + "ranges": [(0, 100), (0, 100)], + "names": ["Chance", "Correlation"], + "units": ["%", "%"], }, - "corrupt": {"params_count": 1, "ranges": [(0, 100)], "names": ["Chance"], "units": ["%"]}, - "bpf": {"params_count": 1, "is_text": True, "names": ["Filters"]}, + "delay": { + # 3rd parameter (jitter distribution) needs the netem-extension + # uBridge and only runs on the kernel datapath. + "params_count": (1, 3), + "ranges": [(1, 32767), (0, 32767)], # ubridge rejects latency <= 0 + "names": ["Latency", "Jitter", "Distribution"], + "units": ["ms", "ms", ""], + "string_params": {2: _validate_distribution_value}, + }, + "corrupt": {"params_count": (1, 1), "ranges": [(0, 100)], "names": ["Chance"], "units": ["%"]}, + "duplicate": { + "params_count": (1, 2), + "ranges": [(0, 100), (0, 100)], + "names": ["Chance", "Correlation"], + "units": ["%", "%"], + }, + "rate": {"params_count": (1, 1), "ranges": [], "names": ["Rate"], "string_params": {0: _validate_rate_value}}, + "reorder": { + "params_count": (1, 3), + "ranges": [(0, 100), (0, 100), (1, 1000)], + "names": ["Reorder", "Correlation", "Gap"], + "units": ["%", "%", "packets"], + }, + "gemodel": { + # Gilbert-Elliot loss model, tc semantics: p = loss probability in + # the bad state, r = loss probability in the good state, 1-h = + # probability of moving from the good to the bad state. + "params_count": (1, 3), + "ranges": [(0, 100), (0, 100), (0, 100)], + "names": ["p (bad-state loss)", "r (good-state loss)", "1-h (good-to-bad transition)"], + "units": ["%", "%", "%"], + }, + "seed": {"params_count": (1, 1), "ranges": [(0, 4294967295)], "names": ["Seed"], "units": [""]}, + "limit": {"params_count": (1, 1), "ranges": [(1, 1000000)], "names": ["Limit"], "units": ["packets"]}, + "bpf": {"params_count": (1, 1), "is_text": True, "names": ["Filters"]}, } if filter_type not in VALIDATION_RULES: raise FilterValidationError(f"Unknown filter type: {filter_type}") rules = VALIDATION_RULES[filter_type] + string_params = rules.get("string_params", {}) # Check parameter count - if len(values) != rules["params_count"]: - raise FilterValidationError(f"{filter_type} expects {rules['params_count']} parameter(s), got {len(values)}") + min_count, max_count = rules["params_count"] + if not (min_count <= len(values) <= max_count): + expected = str(min_count) if min_count == max_count else f"{min_count} to {max_count}" + raise FilterValidationError(f"{filter_type} expects {expected} parameter(s), got {len(values)}") # Validate each parameter for i, value in enumerate(values): @@ -136,26 +258,19 @@ def validate_filter_parameters(filter_type: str, values: List[Any]) -> None: f"{filter_type} parameter {rules['names'][i]} line {line_num + 1} " f"has invalid syntax: {bpf_result['error']}" ) + elif i in string_params: + string_params[i](value, f"{filter_type} parameter {rules['names'][i]}") else: - # Integer parameter validation - try: - if isinstance(value, str): - value = value.strip() - int_value = int(value) - else: - int_value = int(value) - except (ValueError, TypeError): - raise FilterValidationError( - f"{filter_type} parameter {rules['names'][i]} must be an integer, got: {value}" - ) + _validate_int_parameter(filter_type, rules, i, value) - # Range validation - min_val, max_val = rules["ranges"][i] - if int_value < min_val or int_value > max_val: - raise FilterValidationError( - f"{filter_type} parameter {rules['names'][i]} must be between " - f"{min_val} and {max_val} {rules['units'][i]}, got: {int_value}" - ) + if filter_type == "delay" and len(values) >= 3 and isinstance(values[2], str) and values[2].strip(): + # The distribution shapes the jitter draw — without jitter it does + # nothing (the kernel silently stays uniform). values[1] has already + # passed integer validation at this point. + if int(values[1]) <= 0: + raise FilterValidationError( + "delay parameter Distribution requires a Jitter > 0 (the distribution shapes the jitter draw)" + ) def filter_inactive_filters(filters: Dict[str, List[Any]]) -> Dict[str, List[Any]]: @@ -221,9 +336,60 @@ def filter_inactive_filters(filters: Dict[str, List[Any]]) -> Dict[str, List[Any return active_filters +def split_kernel_only_features(filters: Dict[str, List[Any]]) -> Tuple[Dict[str, List[Any]], Set[str]]: + """ + Split a filter set into what the uBridge *relay* can run and what needs + the kernel datapath (tc netem on the veth host end). + + Kernel-only are the whole types in KERNEL_ONLY_FILTERS plus two + parameters: the delay jitter distribution (3rd) and the packet_loss + correlation (2nd) — the relay's delay/packet_loss filters take no such + argument. + + :returns: (relay-safe filters copy, names of the dropped features) + """ + + dropped: Set[str] = set() + clean: Dict[str, List[Any]] = {} + for filter_type, values in (filters or {}).items(): + if filter_type in KERNEL_ONLY_FILTERS: + dropped.add(filter_type) + continue + # tolerate the legacy bare-value shape ({"packet_loss": 10}) + if isinstance(values, (list, tuple)): + values = list(values) + else: + values = [values] if values else [] + if filter_type == "delay" and len(values) >= 3 and str(values[2]).strip(): + dropped.add("delay distribution") + values = values[:2] + elif filter_type == "packet_loss" and len(values) >= 2: + try: + correl = int(values[1]) + except (ValueError, TypeError): + correl = 0 + if correl: + dropped.add("packet_loss correlation") + values = values[:1] + clean[filter_type] = values + return clean, dropped + + +def kernel_only_features(filters: Dict[str, List[Any]]) -> Set[str]: + """ + Names of the features in *filters* that only run on the kernel datapath + (see split_kernel_only_features). + """ + + return split_kernel_only_features(filters)[1] + + def validate_all_filters(filters: Dict[str, List[Any]]) -> None: """ - Validate all packet filters. + Validate all packet filters, including cross-filter dependencies that + mirror the tc netem grammar (reorder requires delay; gemodel and + packet_loss both translate to the netem loss keyword and are mutually + exclusive). Args: filters: Dictionary mapping filter types to their values @@ -240,3 +406,8 @@ def validate_all_filters(filters: Dict[str, List[Any]]) -> None: continue validate_filter_parameters(filter_type, values) + + if "reorder" in filters and "delay" not in filters: + raise FilterValidationError("reorder requires delay") + if "gemodel" in filters and "packet_loss" in filters: + raise FilterValidationError("gemodel and packet_loss are mutually exclusive (both map to the netem loss keyword)") diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 6cf1f2ca1..1868ee577 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -846,3 +846,184 @@ def test_veth_names_deterministic_and_bounded(vm): # multi-port adapters can reach two-digit adapter/port numbers long_host, long_guest = vm._veth_names(12, 8) assert len(long_host) <= 15 and len(long_guest) <= 15 + + +# --------------------------------------------------------------------------- +# P6a: netem extensions (rate / reorder / gemodel / duplicate / seed / limit / +# distribution / correlation) — translation and capability gating +# --------------------------------------------------------------------------- + + +def _caps_hypervisor_netem(tokens="delay,jitter,loss,dup,corrupt,rate,reorder,gemodel,dist,seed,limit"): + """A hypervisor mock whose direct send() answers `tc capabilities` with + the full netem-extension surface.""" + hyp = MagicMock() + hyp.is_running.return_value = True + hyp.send = AsyncioMagicMock(return_value=[f"netem={tokens};ebpf=1;cbpf=1"]) + return hyp + + +@pytest.mark.asyncio +async def test_update_nio_kernel_applies_netem_extensions(vm): + """ + The netem-extension filters merge into the single netem qdisc command, + in the frozen grammar order. + """ + + vm._ubridge_hypervisor = _caps_hypervisor_netem() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = { + "delay": [100, 20, "normal"], + "packet_loss": [5, 25], + "duplicate": [10, 50], + "corrupt": [2], + "reorder": [25, 50, 5], + "rate": ["512kbit"], + "limit": [5000], + "seed": [42], + } + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call( + f'tc netem set "{host_ifc}" delay 100 jitter 20 loss 5 correl 25 dup 10 correl 50 corrupt 2 ' + "reorder 25 correl 50 gap 5 rate 512kbit limit 5000 distribution normal seed 42" + ) + + +@pytest.mark.asyncio +async def test_update_nio_kernel_applies_gemodel(vm): + """ + gemodel replaces the plain loss keyword and is mutually exclusive with + packet_loss (the controller validates; the translation just prefers + gemodel when both somehow arrive). + """ + + vm._ubridge_hypervisor = _caps_hypervisor_netem() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"delay": [50, 5], "gemodel": [100, 0, 30]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc netem set "{host_ifc}" delay 50 jitter 5 loss gemodel 100 0 30') + + +@pytest.mark.asyncio +async def test_apply_netem_extension_needs_capability(vm): + """ + Extension keywords on an old uBridge (no tc capabilities / no netem-ext + tokens) raise a clear upgrade error instead of a raw parse failure. + """ + + # old uBridge: no tc module at all -> empty capabilities + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_hypervisor.is_running.return_value = True + vm._ubridge_hypervisor.send = AsyncioMagicMock(side_effect=UbridgeError("201-Unknown command")) + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="netem extensions"): + await vm._ubridge_apply_netem(host_ifc, {"rate": ["512kbit"]}) + vm._ubridge_send.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_netem_extension_partial_capability(vm): + """ + A uBridge that reports some but not all needed tokens fails on the + missing ones (per-token check, not just build detection). + """ + + vm._ubridge_hypervisor = _caps_hypervisor_netem(tokens="delay,jitter,loss,dup,corrupt") + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="rate, reorder"): + await vm._ubridge_apply_netem(host_ifc, {"delay": [50], "reorder": [25], "rate": ["1mbit"]}) + + +@pytest.mark.asyncio +async def test_apply_netem_plain_surface_never_probes(vm): + """ + Original-surface filters (delay/loss/corrupt, and plain dup) must not + touch the capability probe at all — an old uBridge serves them as-is. + The reset-before-set is unconditional (kernel netem replace merges + optional attrs, so a removed parameter would otherwise leak). + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + await vm._ubridge_apply_netem(host_ifc, {"delay": [50, 5], "packet_loss": [10], "duplicate": [10], "corrupt": [2]}) + assert vm._ubridge_send.call_args_list == [ + call(f'tc reset "{host_ifc}"'), + call(f'tc netem set "{host_ifc}" delay 50 jitter 5 loss 10 dup 10 corrupt 2'), + ] + vm._ubridge_hypervisor.send.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_netem_reset_leak_on_removal(vm): + """ + A re-apply with a parameter REMOVED (rate gone, delay kept) must reset + first: the kernel's netem change merges optional attributes, so a bare + replace would keep the old rate shaping forever. + """ + + vm._ubridge_hypervisor = _caps_hypervisor_netem() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + await vm._ubridge_apply_netem(host_ifc, {"delay": [50], "rate": ["1mbit"]}) + vm._ubridge_send.reset_mock() + await vm._ubridge_apply_netem(host_ifc, {"delay": [50]}) + + assert vm._ubridge_send.call_args_list == [ + call(f'tc reset "{host_ifc}"'), + call(f'tc netem set "{host_ifc}" delay 50'), + ] + + +@pytest.mark.asyncio +async def test_apply_netem_correl_gates_on_extension_build(vm): + """ + Loss/dup correlation is an extension (the capabilities string has no + correl token) — it gates on the netem-extension build marker "rate". + """ + + vm._ubridge_hypervisor = _caps_hypervisor_netem(tokens="delay,jitter,loss,dup,corrupt") + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="netem extensions"): + await vm._ubridge_apply_netem(host_ifc, {"packet_loss": [5, 25]}) + + +def test_create_nio_bridge_accepts_netem_extension_filters(vm): + + nio = vm.manager.create_nio( + {"type": "nio_bridge", "bridge": BRIDGE, "filters": {"rate": ["512kbit"], "reorder": [25, 0, 5]}} + ) + assert isinstance(nio, NIOBridge) + assert nio.filters == {"rate": ["512kbit"], "reorder": [25, 0, 5]} + + +def test_create_nio_udp_rejects_kernel_only_filters(vm): + """ + The relay NIO rejects netem-extension filter types (second guard behind + the controller). + """ + + with pytest.raises(ComputeError, match="kernel-datapath"): + vm.manager.create_nio( + {"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1", "filters": {"gemodel": [100, 0, 30]}} + ) diff --git a/tests/compute/test_base_node.py b/tests/compute/test_base_node.py index 9d4e15374..7e0c34601 100644 --- a/tests/compute/test_base_node.py +++ b/tests/compute/test_base_node.py @@ -169,6 +169,20 @@ async def test_ubridge_apply_filters(node): node._ubridge_send.assert_any_call("bridge add_packet_filter VPCS-10 filter0 latency 10") +@pytest.mark.asyncio +async def test_ubridge_apply_filters_rejects_kernel_only(node): + """ + Netem-extension filters have no relay equivalent: applying them on a + relay bridge must fail loudly instead of pushing an unknown filter type + to uBridge. + """ + + node._ubridge_send = AsyncioMagicMock() + with pytest.raises(NodeError, match="kernel-datapath"): + await node._ubridge_apply_filters("VPCS-10", {"delay": [100, 10], "rate": ["512kbit"]}) + node._ubridge_send.assert_not_called() + + @pytest.mark.asyncio async def test_ubridge_apply_bpf_filters(node): diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 87770a86e..645ab32f1 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -52,6 +52,22 @@ async def _kernel_link(project, **kwargs): return link, node1, node2 +def _mock_compute_http(compute): + """Wire the async compute HTTP callbacks the relay _prepare path needs.""" + + async def subnet_callback(other): + return ("192.168.1.1", "192.168.1.2") + + compute.get_ip_on_same_subnet.side_effect = subnet_callback + + async def port_callback(path, data=None, **kwargs): + response = MagicMock() + response.json = {"udp_port": 1024} + return response + + compute.post.side_effect = port_callback + + @pytest.mark.asyncio async def test_kernel_datapath_eligible(project): @@ -380,3 +396,108 @@ async def test_update_suspend_kernel_link_sends_no_synthetic_filter(project): for body in bodies: assert body["filters"] == {} assert body["suspend"] is True + + +# --------------------------------------------------------------------------- +# P6a: netem-extension filters (rate / reorder / gemodel / duplicate / seed / +# limit / distribution / correlation) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_with_netem_extension_filters(project): + """ + All netem-extension types have a tc netem equivalent — they keep the + link kernel-eligible (only frequency_drop still forces the relay). + """ + + link, node1, node2 = await _kernel_link(project) + link._filters = { + "delay": [100, 20, "normal"], + "packet_loss": [5, 25], + "duplicate": [10], + "reorder": [25, 0, 5], + "rate": ["512kbit"], + "gemodel": [100, 0, 30], + "seed": [42], + "limit": [5000], + } + # gemodel/packet_loss are mutually exclusive — drop packet_loss for the + # eligibility check (validation rejects the mix elsewhere) + link._filters.pop("packet_loss") + assert link._kernel_datapath_eligible(node1, node2) is True + + +@pytest.mark.asyncio +async def test_update_netem_extension_filters_accepted_on_kernel_link(project): + """ + The extension filters ride the kernel NIO to both endpoints like the + base netem surface. + """ + + link, node1, node2 = await _kernel_link(project) + await link._prepare() + + filters = {"delay": [100, 20, "normal"], "rate": ["512kbit"], "reorder": [25, 0, 5]} + await link.update_filters(filters) + assert link.filters == filters + by_node = {entry[0].id: entry[3] for entry in (await link._prepare())} + assert by_node[node1.id]["filters"] == filters + assert by_node[node2.id]["filters"] == filters + + +@pytest.mark.asyncio +async def test_update_kernel_only_filters_rejected_on_relay_link(project): + """ + Netem-extension filters have no relay equivalent: setting them on a + created relay link is a 409. (frequency_drop in the stored filters + forces this link onto the relay datapath.) + """ + + link, _n1, _n2 = await _kernel_link(project) + _mock_compute_http(link._nodes[0]["node"].compute) + link._filters = {"frequency_drop": [7]} + await link._prepare() + assert link.kernel_datapath is False + link._created = True # the computes hold the UDP NIOs + + with pytest.raises(ControllerError, match="kernel-datapath"): + await link.update_filters({"rate": ["512kbit"]}) + # the stored filters are untouched + assert link.filters == {"frequency_drop": [7]} + + +@pytest.mark.asyncio +async def test_update_kernel_only_filters_accepted_before_creation(project): + """ + While loading a project the datapath is not decided yet (update_filters + runs before _prepare) — a link that will be kernel-wired must accept the + extension filters. Only _prepare decides, dropping what the relay cannot + run. + """ + + link, _n1, _n2 = await _kernel_link(project) + await link.update_filters({"rate": ["512kbit"], "reorder": [25, 0, 5], "delay": [100, 10]}) + assert link.filters["rate"] == ["512kbit"] + + +@pytest.mark.asyncio +async def test_prepare_relay_link_strips_kernel_only_filters(project): + """ + A relay-wired link (here: frequency_drop in the stored filters) built + from a topology carrying extension filters drops them with a warning + instead of pushing an unknown filter type to uBridge. + """ + + link, node1, node2 = await _kernel_link(project) + _mock_compute_http(link._nodes[0]["node"].compute) + link._filters = {"frequency_drop": [7], "rate": ["512kbit"], "delay": [100, 20, "normal"], "packet_loss": [5, 25]} + entries = await link._prepare() + assert link.kernel_datapath is False + by_node = {entry[0].id: entry[3] for entry in entries} + assert by_node[node1.id]["type"] == "nio_udp" + # the relay's filter node keeps what it can run; rate, the distribution + # and the loss correlation are kernel-only and stripped (the far side + # carries no filters — the relay filters one bridge) + assert by_node[node1.id]["filters"] == {"frequency_drop": [7], "delay": [100, 20], "packet_loss": [5]} + assert by_node[node2.id]["filters"] == {} diff --git a/tests/utils/test_packet_filter_validation.py b/tests/utils/test_packet_filter_validation.py index 462e23a4d..034114f65 100644 --- a/tests/utils/test_packet_filter_validation.py +++ b/tests/utils/test_packet_filter_validation.py @@ -7,6 +7,7 @@ import pytest from gns3server.utils.packet_filter_validation import ( FilterValidationError, filter_inactive_filters, + split_kernel_only_features, validate_all_filters, validate_filter_parameters, ) @@ -131,9 +132,11 @@ class TestPacketFilterValidation: with pytest.raises(FilterValidationError, match="expects 1 parameter"): validate_filter_parameters("frequency_drop", [1, 2]) - # delay expects 2 parameters - with pytest.raises(FilterValidationError, match="expects 2 parameter"): - validate_filter_parameters("delay", [100]) + # delay takes 1 to 3 parameters (latency, optional jitter, + # optional distribution) — latency alone is valid, a 4th is not + validate_filter_parameters("delay", [100]) + with pytest.raises(FilterValidationError, match="expects 1 to 3 parameter"): + validate_filter_parameters("delay", [100, 50, "normal", "extra"]) def test_string_to_int_conversion(self): """Test string to integer conversion.""" @@ -258,3 +261,108 @@ class TestFilterInactiveFilters: """Test None filters.""" result = filter_inactive_filters(None) assert result == {} + + +class TestNetemExtensionFilters: + """P6a: the netem-extension filter types (rate, reorder, gemodel, + duplicate, seed, limit) and the kernel-only parameter extensions.""" + + def test_rate_valid(self): + for value in ("512kbit", "10mbit", "1gbit", "64000bps", "8000kbps", "100mbps", "1544bit"): + validate_filter_parameters("rate", [value]) + + def test_rate_invalid(self): + # bare number, unknown unit, float, over 100gbit + for value in ("512", "512kbits", "0.5mbit", "101gbit", "", "mbit"): + with pytest.raises(FilterValidationError, match="Rate"): + validate_filter_parameters("rate", [value]) + + def test_rate_rejects_non_string(self): + with pytest.raises(FilterValidationError, match="must be a string"): + validate_filter_parameters("rate", [512000]) + + def test_reorder_valid(self): + validate_filter_parameters("reorder", [25]) + validate_filter_parameters("reorder", [25, 50]) + validate_filter_parameters("reorder", [25, 50, 5]) + + def test_reorder_invalid_ranges(self): + with pytest.raises(FilterValidationError, match="Reorder"): + validate_filter_parameters("reorder", [101]) + with pytest.raises(FilterValidationError, match="Gap"): + validate_filter_parameters("reorder", [25, 0, 0]) # gap < 1 + with pytest.raises(FilterValidationError, match="Gap"): + validate_filter_parameters("reorder", [25, 0, 1001]) + + def test_reorder_requires_delay(self): + with pytest.raises(FilterValidationError, match="reorder requires delay"): + validate_all_filters({"reorder": [25]}) + # with delay present it passes + validate_all_filters({"reorder": [25, 0, 5], "delay": [100, 10]}) + + def test_gemodel_valid_and_exclusive(self): + validate_filter_parameters("gemodel", [100]) + validate_filter_parameters("gemodel", [100, 0]) + validate_filter_parameters("gemodel", [100, 0, 30]) + with pytest.raises(FilterValidationError, match="mutually exclusive"): + validate_all_filters({"gemodel": [100, 0, 30], "packet_loss": [10]}) + + def test_gemodel_invalid_range(self): + with pytest.raises(FilterValidationError, match="bad-state"): + validate_filter_parameters("gemodel", [101, 0, 30]) + + def test_duplicate_valid(self): + validate_filter_parameters("duplicate", [10]) + validate_filter_parameters("duplicate", [10, 25]) + + def test_seed_and_limit(self): + validate_filter_parameters("seed", [42]) + validate_filter_parameters("seed", [4294967295]) + with pytest.raises(FilterValidationError, match="Seed"): + validate_filter_parameters("seed", [4294967296]) + validate_filter_parameters("limit", [5000]) + with pytest.raises(FilterValidationError, match="Limit"): + validate_filter_parameters("limit", [0]) + with pytest.raises(FilterValidationError, match="Limit"): + validate_filter_parameters("limit", [1000001]) + + def test_delay_distribution(self): + for dist in ("uniform", "normal", "pareto", "paretonormal"): + validate_filter_parameters("delay", [100, 20, dist]) + with pytest.raises(FilterValidationError, match="Distribution"): + validate_filter_parameters("delay", [100, 20, "poisson"]) + # distribution without jitter does nothing -> rejected + with pytest.raises(FilterValidationError, match="requires a Jitter"): + validate_filter_parameters("delay", [100, 0, "normal"]) + # empty distribution = not set + validate_filter_parameters("delay", [100, 20, ""]) + + def test_packet_loss_correlation(self): + validate_filter_parameters("packet_loss", [10]) + validate_filter_parameters("packet_loss", [10, 25]) + with pytest.raises(FilterValidationError, match="Correlation"): + validate_filter_parameters("packet_loss", [10, 101]) + + def test_kernel_only_features(self): + clean, dropped = split_kernel_only_features( + { + "delay": [100, 20, "normal"], + "packet_loss": [10, 25], + "rate": ["512kbit"], + "corrupt": [2], + } + ) + assert dropped == {"rate", "delay distribution", "packet_loss correlation"} + assert clean == {"delay": [100, 20], "packet_loss": [10], "corrupt": [2]} + + def test_kernel_only_features_all_relay_safe(self): + clean, dropped = split_kernel_only_features( + {"delay": [100, 20, ""], "packet_loss": [10, 0], "corrupt": [2], "bpf": ["icmp"]} + ) + assert dropped == set() + # zero correlation and empty distribution are kept as-is (no-ops) + assert clean == {"delay": [100, 20, ""], "packet_loss": [10, 0], "corrupt": [2], "bpf": ["icmp"]} + + def test_kernel_only_features_empty(self): + assert split_kernel_only_features({}) == ({}, set()) + assert split_kernel_only_features(None) == ({}, set()) From 0c4ec8c38cc02ca5c6be529fb8d8441ecef3e5c4 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 29 Sep 2026 23:35:03 +0800 Subject: [PATCH 012/101] feat(docker): eBPF stateful packet filters on kernel links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consume the eBPF impairment classifier delivered in uBridge feature/tc-precision: frequency_drop becomes the exact every-Nth mode (tc nth_drop; -1 = drop everything maps to every 1st) and the new kernel-only quota type is the byte-cap mode (tc quota_drop, 100 = hard cutoff). KERNEL_UNSUPPORTED_FILTERS is gone entirely — kernel/relay eligibility is now purely topological. Both modes gate on ebpf=1 from the per-process tc capabilities probe (error names the setcap fix); the cleanup path (modes off) only talks to the eBPF commands when the probe reported them. Reconcile is a plain re-set: the netem reset-before-set already detaches the program and clears uBridge's per-interface registry, so a mode set after it is a fresh load+attach. Documented semantic difference: the relay counts both directions through one bridge counter, the kernel counts per direction — N=3 round-trip loss is 1-(2/3)^2 = 55.6% on kernel vs ~1/N on relay (measured 57%). window_drop is NOT exposed: the delivered implementation advances the window with no gap, making 'outside the window packets pass' unreachable (needs outage/period fields — recorded in the spec); flow_drop awaits a parameter-shape decision. Also fixes a pre-existing restart bug the e2e exposed: _bridges (relay bridge names) was never cleared on stop, so restarting a node with a relay link skipped 'bridge create' and failed on add_nio_udp. e2e (isolated instance): 13/13 — pref-1 tc_impair classifier on both veths, 57% round-trip for N=3, -1 = exact 100%, quota hard cutoff, delay coexistence, clean teardown, suspend/resume, node restart, relay-leg frequency_drop regression through an Ethernet switch. Full suite: 1925 passed. --- docs/design/ubridge-kernel-impairment-spec.md | 48 +++-- docs/features/docker-kernel-datapath.md | 83 +++++++-- gns3server/compute/base_manager.py | 11 -- gns3server/compute/docker/docker_vm.py | 92 ++++++++-- gns3server/controller/link.py | 49 +++-- gns3server/controller/udp_link.py | 23 +-- gns3server/schemas/compute/nios.py | 12 +- gns3server/utils/packet_filter_validation.py | 33 ++-- .../docker/test_docker_kernel_datapath.py | 172 ++++++++++++++++-- tests/controller/test_kernel_datapath_link.py | 68 ++++--- tests/utils/test_packet_filter_validation.py | 17 ++ 11 files changed, 449 insertions(+), 159 deletions(-) diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md index 356b3a5b4..ad795cc2e 100644 --- a/docs/design/ubridge-kernel-impairment-spec.md +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -4,17 +4,16 @@ See LICENSE file for licensing information. --> > Frozen requirements spec for the **uBridge** project. Delivery status: -> parts **A** (netem keyword extensions, uBridge `feature/tc-netem-ext`), **C** -> (cBPF match-drop) and **D + E** (full-restore idempotent `tc reset`, -> `tc capabilities` — uBridge `feature/tc-bpf-drop`) are **delivered and -> integrated** on gns3-server branches `feat/docker-kernel-bpf-drop` (C/D/E) -> and `feat/docker-kernel-netem-ext` (A: rate/reorder/gemodel/duplicate/ -> seed/limit/distributions/correlation exposed as GNS3 filter types). -> Part **B** (eBPF stateful classifier) is delivered in the uBridge -> `feature/tc-precision` branch but **not yet consumed** by gns3-server. -> The originally shipped netem surface (delay/jitter/loss/dup/corrupt) -> exists in uBridge 1.2.3+ and is verified against gns3-server branch -> `feat/docker-kernel-filters`. +> **all parts delivered and integrated.** A (netem keyword extensions, +> uBridge `feature/tc-netem-ext`), C + D + E (cBPF match-drop, full-restore +> idempotent `tc reset`, `tc capabilities` — uBridge `feature/tc-bpf-drop`) +> and **B (eBPF stateful classifier — uBridge `feature/tc-precision` +> 59e2b38, including the non-root setcap load fix)** are consumed by +> gns3-server on the branch stack ending at `feat/docker-kernel-ebpf-drops` +> (B: `frequency_drop` → `tc nth_drop`, the new kernel-only `quota` type → +> `tc quota_drop`). The originally shipped netem surface (delay/jitter/ +> loss/dup/corrupt) exists in uBridge 1.2.3+ and is verified against +> gns3-server branch `feat/docker-kernel-filters`. > > **Deviation found during integration (part A/D):** the delivered > `tc netem set` uses NLM_F_REPLACE, but the *kernel's* netem change merges @@ -27,6 +26,20 @@ See LICENSE file for licensing information. > invisible). If uBridge later emits explicit zero/absent clears for every > optional attribute, the server-side reset-before-set can be dropped; the > two behaviours are compatible either way. +> +> **Deviation found during B integration:** `window_drop`'s implementation +> advances the window start by one length on expiry with no gap, making the +> windows back-to-back — after the first `start_ms` the "outside the window +> packets pass" behaviour described in B.2 is unreachable (inside ≡ always). +> gns3-server therefore does **not** expose window_drop yet; it needs either +> a second length field (outage vs period) or a next-start semantics fix on +> the uBridge side. `flow_drop` is delivered and functional but not yet +> exposed as a GNS3 filter type (its mask/target parameter shape needs a +> UX decision). Additionally, the delivered B.1 program had to fold the +> flow-hash L4 port reads to constant offsets (IHL==20 only) because the +> verifier prohibits variable packet-pointer arithmetic for non-root — +> even with CAP_BPF (Spectre-mitigation gating by uid); documented in +> uBridge's doc/tc.md. # uBridge kernel impairment: tc netem extensions + eBPF classifiers @@ -293,11 +306,14 @@ corresponding filter types per capability. in `DockerVM._ubridge_apply_netem`; extension keywords are gated on the `tc capabilities` netem token list (old uBridge builds are never probed for the original surface). -* **Done** (`feat/docker-kernel-bpf-drop`): `bpf` runs as cBPF match-drop - (`KERNEL_UNSUPPORTED_FILTERS` shrank to `frequency_drop`). -* `frequency_drop` moves out of `KERNEL_UNSUPPORTED_FILTERS` once the eBPF - classifier (part B, uBridge `feature/tc-precision`) is consumed on the - server side, keyed on `ebpf=1` per compute. +* **Done** (`feat/docker-kernel-bpf-drop`): `bpf` runs as cBPF match-drop. +* **Done** (`feat/docker-kernel-ebpf-drops`): the eBPF classifier (part B) + is consumed — `frequency_drop` → `tc nth_drop` (-1 → every 1st), the new + kernel-only `quota` type → `tc quota_drop`, both keyed on `ebpf=1` from + the per-process `tc capabilities` probe. `KERNEL_UNSUPPORTED_FILTERS` is + gone entirely: kernel/relay eligibility is purely topological now. + Not yet exposed: `window_drop` (implementation deviation above), + `flow_drop` (parameter-shape UX decision). * Every veth end owns one qdisc + its filters: per-direction impairment is an architectural freebie to expose later (API `direction` field), aligned with marker `dir` semantics. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index fad28bbb7..0f5bacf48 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -24,13 +24,14 @@ Frames never leave the kernel. | Link | uBridge bridge, TAP fd ↔ UDP, userspace copy | per-link kernel bridge `gns3{link_id[:11]}`, zero-copy | | Relay latency | ~0.3 ms RTT | ~0.05 ms RTT (measured) | | Link attach at runtime | yes (relay attaches the pre-existing TAP) | yes (brctl addif / add_nio_ethernet — interface untouched) | -| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt / duplicate + the netem extensions rate, reorder, gemodel, seed, limit, jitter distributions, loss correlation); bpf as cls_bpf match-drop classifiers; frequency_drop stays relay-only (eBPF classifier pending) | +| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt / duplicate + the netem extensions rate, reorder, gemodel, seed, limit, jitter distributions, loss correlation); bpf as cls_bpf match-drop classifiers; frequency_drop and quota as the eBPF stateful classifier — every filter type has a kernel equivalent | | Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | -The work landed in seven stages on branch stack `feat/docker-kernel-datapath` → +The work landed in eight stages on branch stack `feat/docker-kernel-datapath` → `feat/docker-kernel-capture` → `feat/docker-kernel-markers` → `feat/docker-veth-everywhere` → `feat/docker-kernel-filters` → -`feat/docker-kernel-bpf-drop` → `feat/docker-kernel-netem-ext`: +`feat/docker-kernel-bpf-drop` → `feat/docker-kernel-netem-ext` → +`feat/docker-kernel-ebpf-drops`: 1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based suspend, crash-safe reconciliation. @@ -57,6 +58,14 @@ The work landed in seven stages on branch stack `feat/docker-kernel-datapath` process. Reconcile became reset-before-set: the kernel's netem replace merges optional attributes, so removing a parameter needs the explicit reset. +8. **eBPF stateful drops** — `frequency_drop` becomes the eBPF classifier's + exact every-Nth mode (`tc nth_drop`, uBridge `feature/tc-precision`; + -1 = drop everything → every 1st) and the new kernel-only `quota` type + the byte-cap mode (`tc quota_drop`). Needs a uBridge reporting `ebpf=1` + (setcap cap_bpf,cap_net_admin,cap_net_raw=ep — verified working for a + non-root server process, the production credential shape). With this, + no filter type forces the relay anymore: kernel/relay is a purely + topological choice. ## Adapter interface types @@ -89,8 +98,8 @@ Kernel link between two containers on the same compute: └───────────────────┘ ``` -Relay link on the same unified veth (fallback for filtered links or when -`enable_kernel_datapath` is off — e.g. to reach another compute): +Relay link on the same unified veth (fallback for mixed node types, cross- +compute wiring or when `enable_kernel_datapath` is off): ``` ┌────────────┐ ┌────────────────────────────┐ ┌────────────┐ @@ -110,10 +119,11 @@ time; the compute side only reacts to the NIO type: * both endpoints Docker (any class except unix-socket containers), same compute -* no active packet filters (filters live in the relay; kernelization via tc - netem is planned) * `Server.enable_kernel_datapath` enabled (default) +Filters never disqualify the kernel path anymore — every type has a kernel +equivalent (netem / cls_bpf / eBPF classifier). + There is **no stopped-node requirement**. Since every adapter is a veth, links attach to running containers: `brctl addif` (kernel) or `bridge add_nio_ethernet` (relay) are runtime-safe operations on the host end. @@ -189,7 +199,8 @@ ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): | `seed [u32]` | netem `seed N` | reproducible random draws; kernel-only type | | `limit [pkts]` | netem `limit N` | queue depth above the 1000 default (rate+delay BDP); kernel-only type | | `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | -| `frequency_drop` | — not yet (eBPF stateful classifier spec'd: [ubridge-kernel-impairment-spec](../design/ubridge-kernel-impairment-spec.md), part B pending) | 409 on kernel links; relay fallback | +| `frequency_drop [N]` | `tc nth_drop N` — the eBPF stateful classifier's exact every-Nth mode (clsact egress prio 1) | needs uBridge `ebpf=1`; -1 = drop everything → every 1st; per-direction counters (see below) | +| `quota [bytes, %]` | `tc quota_drop ` — after the byte quota is consumed, each further packet drops with the given chance (100 = hard cutoff) | kernel-only type; needs `ebpf=1`; per-direction byte counters | Semantics: @@ -213,12 +224,24 @@ Semantics: `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. -* **kernel-only vs relay-only.** The extension types have no uBridge relay - equivalent: `available_filters` hides them on relay links, setting one on - a created relay link returns 409, and a project loaded with such filters - on a link that cannot be kernel-wired drops them with a warning (invalid - filters are dropped the same way at load). Symmetrically, - `frequency_drop` stays relay-only. +* **kernel-only vs relay-available.** The extension types and `quota` have + no uBridge relay equivalent: `available_filters` hides them on relay + links, setting one on a created relay link returns 409, and a project + loaded with such filters on a link that cannot be kernel-wired drops them + with a warning (invalid filters are dropped the same way at load). + `frequency_drop` runs on both datapaths (relay userspace filter / eBPF + every-Nth). +* **frequency_drop counting semantics differ per datapath.** The relay's + single filtered bridge counts packets of BOTH directions through one + counter; the kernel attaches one classifier per veth end, so each + direction drops every Nth independently. For a round-trip measurement + with every-Nth N: relay ≈ 1/N of frames (phase-locked), kernel = 1 − + ((N−1)/N)² (e.g. N=3 → 55.6 % round-trip loss, measured 57 %). The + kernel count is exact (atomic counter), unlike netem's stochastic loss. +* **Classifier order** on clsact egress: the eBPF stateful filter runs at + prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the + stateful modes never reaches the expression drops, and dropped packets + never reach netem. * **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + @@ -253,7 +276,9 @@ tc netem set [delay ] [jitter ] [loss <%> [correl <%>] | loss gemod [dup <%> [correl <%>]] [corrupt <%>] [reorder <%> [correl <%>] [gap ]] [rate ] [limit ] [distribution uniform|normal|pareto|paretonormal] [seed ] tc bpf_drop add "" / tc bpf_drop flush -tc reset # full restore: filters -> clsact -> root qdisc, idempotent +tc nth_drop # eBPF exact every-Nth (frequency_drop) +tc quota_drop | off # eBPF byte cap (quota) +tc reset # full restore: eBPF filter -> bpf_drops -> clsact -> root qdisc, idempotent tc capabilities # "netem=;ebpf=0|1;cbpf=0|1", probed once per process bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capture ``` @@ -280,7 +305,8 @@ pairs), capture freeze on stop, server restart reconciliation. Filters (two-container kernel link, 27/27 checks): baseline 0.06 ms → `delay 100` = 200.2 ms RTT (2× per direction) → `delay 50` = 100.2 ms (atomic replace) → clear = baseline (qdisc detached); `packet_loss 30` = -48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop` = 409; +48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop` = 409 (at that +stage — superseded by the eBPF stage below); suspend with delay active = 100 % loss, resume keeps the qdisc (200.1 ms); node restart restores it (200.1 ms); link delete detaches it (`/usr/sbin/tc qdisc show` — no netem left); re-created link has no residual @@ -291,8 +317,9 @@ ends' clsact; size-discriminating expression (`greater 150`) = small pings pass / 300-byte pings dropped (real byte-level BPF match); two-line OR; coexistence with netem (delay 50 → small pings 100.2 ms while big dropped); clear removes clsact entirely; suspend/resume keeps the drops; link delete + -recreate leaves no residual filters; `frequency_drop` still 409; -`available_filters` shows `bpf` and hides `frequency_drop`. Unit tests: +recreate leaves no residual filters; `frequency_drop` still 409 and hidden +(at that stage — superseded by the eBPF stage below); `available_filters` shows +`bpf`. Unit tests: `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. @@ -307,11 +334,27 @@ cleared; `duplicate 50` link alive; `packet_loss 30 correl 50` shows exactly 2×50 ms; suspend/resume and node restart keep the filters; a docker↔ethernet-switch relay link rejects `rate` with 409 and its `available_filters` hides the kernel-only types while the kernel link shows -them all (minus frequency_drop); `reorder` without `delay` and +them all (frequency_drop included since the eBPF stage); `reorder` without `delay` and `gemodel`+`packet_loss` are 409s; teardown leaves no netem on either veth. Also verified live: a relay attach (`bridge add_nio_ethernet`) on the admin-down veth host end fails in libpcap — the relay path now brings the -interface up before attaching. Unit tests: +interface up before attaching. + +eBPF stateful drops (same setup, 13/13 checks): `frequency_drop 3` installs +the tc_impair classifier on both veths (clsact egress pref 1, visible in +`tc filter show`); round-trip loss 57 % against the per-direction theory 1 − +(2/3)² = 55.6 % (each direction drops every 3rd independently — the relay's +single shared counter would give ≈ 1/3); `frequency_drop -1` = exactly +100 % loss; `quota 3000B/100 %` = pings pass until the byte cap then hard +cutoff; coexistence with `delay 50` (RTT 100.3 ms with every-4th drops); +clearing filters leaves no residual bpf filter; suspend = 100 % while down +with drops persisting after resume; node restart restores every-Nth on the +fresh veth; `available_filters` shows frequency_drop + quota on kernel +links and hides quota on relay links; a two-leg relay path through an +Ethernet switch still serves frequency_drop from the userspace filter +(56 % round-trip for N=3). The e2e also exposed and fixed a pre-existing +restart bug: the relay bridge-name registry was not cleared on stop, so a +node restart with a relay link skipped `bridge create` and failed. Unit tests: `tests/utils/test_packet_filter_validation.py` (P6a class), `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index 0205003fd..d63817152 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -375,17 +375,6 @@ class BaseManager: nio.suspend = nio_settings.get("suspend", False) elif nio_settings["type"] == "nio_bridge": nio = NIOBridge(nio_settings["bridge"]) - # Impairment filters with a tc netem equivalent run on the veth - # host end; the relay-only types are rejected here (the controller - # already keeps them off kernel links — this is the second guard). - from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS - - unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(nio_settings.get("filters") or {}) - if unsupported: - raise ComputeError( - "Packet filter(s) {} cannot run on a kernel-datapath link " - "(no uBridge relay in the forwarding path)".format(", ".join(sorted(unsupported))) - ) nio.filters = nio_settings.get("filters", {}) # Markers ride the NIO and attach to the veth host interface via # uBridge's AF_PACKET marker module (marker add_kernel). diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 641b46f10..62316dd7b 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1438,6 +1438,11 @@ class DockerVM(BaseNode): # The next start spawns a fresh uBridge process — its capabilities # must be re-probed (see _ubridge_tc_capabilities). self._ubridge_tc_caps = None + # The uBridge process dies with every bridge it hosted — drop the + # relay bridge names too, or the next start would skip + # "bridge create" for them (the gate checks the name's presence) + # and fail on "bridge add_nio_udp: bridge doesn't exist". + self._bridges.clear() # veth host ends live in the root namespace and survive container # death — remove them while the uBridge control channel is still up. await self._remove_kernel_veths() @@ -1944,9 +1949,11 @@ class DockerVM(BaseNode): await self._ubridge_apply_markers(host_ifc, nio) # Impairment filters become one tc netem qdisc on the veth host # end (restored here on node restart, like the capture above); - # bpf expressions become cls_bpf match-drop classifiers. + # bpf expressions become cls_bpf match-drop classifiers and + # frequency_drop/quota become the eBPF stateful classifier. await self._ubridge_apply_netem(host_ifc, nio.filters) await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) + await self._ubridge_apply_ebpf_drops(host_ifc, nio.filters) return # Relay NIO. On a veth-backed adapter (the unified Docker interface) @@ -2015,28 +2022,19 @@ class DockerVM(BaseNode): if self.ubridge: if isinstance(nio, NIOBridge): - # The controller keeps frequency_drop off kernel links (no - # kernel equivalent until the eBPF classifier lands) and - # create_nio rejects it on the POST path; this PUT path - # mutates the NIO in place, so guard here too. bpf is fine — - # it becomes cls_bpf match-drop classifiers (cbpf-probed). - unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(nio.filters or {}) - if unsupported: - raise DockerError( - "Packet filter(s) {} cannot run on a kernel-datapath link " - "(no uBridge relay in the forwarding path)".format(", ".join(sorted(unsupported))) - ) if self.status == "started": host_ifc = self._kernel_veths.get((adapter_number, port_number)) if host_ifc is not None: # Incremental marker reconcile on the veth anchor — # the relay-datapath equivalent of the branch below. await self._ubridge_apply_markers(host_ifc, nio) - # netem set is an atomic replace, so a full re-apply - # IS the incremental reconcile for impairment filters; - # bpf drops reconcile as flush + re-add. + # The netem apply resets the interface first (the + # kernel merges optional netem attrs on replace), so + # everything anchored on clsact must re-apply after + # it: bpf drops flush + re-add, eBPF modes re-set. await self._ubridge_apply_netem(host_ifc, nio.filters) await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) + await self._ubridge_apply_ebpf_drops(host_ifc, nio.filters) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) return @@ -2376,6 +2374,70 @@ class DockerVM(BaseNode): log.warning(message) self.project.emit("log.warning", {"message": message}) + async def _ubridge_apply_ebpf_drops(self, host_ifc, filters): + """ + Translate the NIO's stateful filters into uBridge's eBPF impairment + classifier on the veth host end's clsact egress (prio 1, below + bpf_drop's 10-99): ``frequency_drop`` becomes the exact every-Nth + mode (``tc nth_drop``; -1 = drop everything maps to every 1st), and + the kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``). + + Reconcile is a plain re-set on every apply: the preceding netem + apply resets the interface (which detaches the program and drops + uBridge's per-interface registry entry), so a mode set re-loads the + program from scratch, and a mode left absent is turned off + explicitly (idempotent) so a removed filter stops dropping. + + Semantic difference vs the relay, documented server-side: the relay + counts packets of BOTH directions through its single filtered + bridge, while each veth end's classifier counts only the traffic + entering that container — with both endpoints applying theirs, each + direction drops every Nth independently. + + :param host_ifc: veth host-end interface name + :param filters: NIO filters dictionary + """ + + def values_of(key): + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + frequency = values_of("frequency_drop") + quota = values_of("quota") + if not frequency and not quota and self._ubridge_tc_caps is None: + # Nothing to set and nothing was ever installed (no probe yet) — + # skip touching an old uBridge entirely. + return + caps = await self._ubridge_tc_capabilities() + if not frequency and not quota: + if caps.get("ebpf") == "1": + # Only when the eBPF module answers can anything be + # installed; "off" is idempotent in uBridge. + await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') + await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') + return + if caps.get("ebpf") != "1": + raise DockerError( + "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " + "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " + "the uBridge binary); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(sorted(f for f in ("frequency_drop", "quota") if f in filters))) + ) + if frequency: + # relay semantics: -1 = drop everything, N = every Nth packet + nth = 1 if int(frequency[0]) == -1 else int(frequency[0]) + await self._ubridge_send(f'tc nth_drop "{host_ifc}" {nth}') + else: + await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') + if quota: + await self._ubridge_send( + 'tc quota_drop "{ifc}" {bytes} {pct}'.format(ifc=host_ifc, bytes=int(quota[0]), pct=int(quota[1])) + ) + else: + await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') + async def _remove_kernel_markers(self, host_ifc): """ Tear down every kernel marker anchored to *host_ifc*: the veth survives diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 2bfa1dc32..0b69981a4 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -46,7 +46,8 @@ FILTERS = [ { "type": "frequency_drop", "name": "Frequency drop", - "description": "It will drop everything with a -1 frequency, drop every Nth packet with a positive frequency, or drop nothing", + "description": "It will drop everything with a -1 frequency, drop every Nth packet with a positive " + "frequency, or drop nothing. On kernel-datapath links this is exact (eBPF every-Nth counter)", "parameters": [{"name": "Frequency", "minimum": -1, "maximum": 32767, "type": "int", "unit": "th packet"}], }, { @@ -136,6 +137,16 @@ FILTERS = [ "combining a low rate with a long delay (kernel-datapath links only)", "parameters": [{"name": "Limit", "minimum": 1, "maximum": 1000000, "type": "int", "unit": "packets"}], }, + { + "type": "quota", + "name": "Byte quota", + "description": "After the byte quota is consumed, each further packet drops with the given chance " + "(100 = hard cutoff) — a data cap, like a mobile plan (kernel-datapath links only)", + "parameters": [ + {"name": "Quota", "minimum": 1, "maximum": 1000000000000000, "type": "int", "unit": "bytes"}, + {"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + ], + }, ] @@ -296,24 +307,12 @@ class Link: except FilterValidationError as e: raise ControllerError(f"Invalid packet filter parameters: {e!s}") - if self.kernel_datapath: - from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS - - unsupported = KERNEL_UNSUPPORTED_FILTERS.intersection(new_filters or {}) - if unsupported: - raise ControllerError( - "Packet filter(s) {} cannot run on a kernel-datapath link (no uBridge relay " - "in the forwarding path; delay, packet loss and corrupt are served by tc " - "netem on the veth); delete and recreate the link to use them".format( - ", ".join(sorted(unsupported)) - ) - ) - elif self._created: - # The reverse guard: netem-extension filters have no relay - # equivalent. Only enforced on created links — while loading a - # project the datapath is not decided yet (a link that will be - # wired on the kernel datapath must accept them), and the relay - # prepare path drops what it cannot run with a warning. + if self._created and not self.kernel_datapath: + # Kernel-only filters (the netem extensions, the eBPF quota mode) + # have no relay equivalent. Only enforced on created links — while + # loading a project the datapath is not decided yet (a link that + # will be wired on the kernel datapath must accept them), and the + # relay prepare path drops what it cannot run with a warning. conflicts = kernel_only_features(new_filters) if conflicts: raise ControllerError( @@ -722,13 +721,11 @@ class Link: filter_node = self._get_filter_node() if filter_node: if self.kernel_datapath: - # Kernel-datapath links serve filters via tc netem on the - # veth host end; types without a netem equivalent (which - # only run in the uBridge relay) are hidden from the picker. - from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS - - return [f for f in FILTERS if f["type"] not in KERNEL_UNSUPPORTED_FILTERS] - # Relay links: hide the netem-extension types the relay cannot run + # Kernel-datapath links serve every filter type: netem for + # delay/loss/corrupt and the netem extensions, cls_bpf for + # bpf, the eBPF classifier for frequency_drop and quota. + return FILTERS + # Relay links: hide the kernel-only types the relay cannot run # (they would be rejected with 409 on update). from gns3server.utils.packet_filter_validation import KERNEL_ONLY_FILTERS diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index e943dba46..e2e9602d7 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -165,11 +165,10 @@ class UDPLink(Link): """ Whether this link can be wired on the kernel datapath (veth pairs enslaved into a per-link Linux bridge) instead of the uBridge UDP - relay. Impairment filters with a tc netem equivalent (delay, - packet_loss, corrupt, bpf via cls_bpf, and the netem extensions - rate/reorder/gemodel/duplicate/seed/limit) are served on the veth - host end; frequency_drop only exists in the uBridge relay and - disqualifies the kernel path. Capture and markers are served by + relay. Every GNS3 filter type has a kernel equivalent — netem for + delay/packet_loss/corrupt and the netem extensions, cls_bpf for bpf, + the eBPF stateful classifier for frequency_drop — so filters no + longer disqualify the kernel path. Capture and markers are served by uBridge's AF_PACKET modules on the veth host end. Docker adapters are born as veth pairs (unified interface), so the @@ -185,12 +184,6 @@ class UDPLink(Link): return False if node1.compute.id != node2.compute.id: return False - # Look at the *stored* filters, not get_active_filters(): a suspended - # link reports the synthetic frequency_drop emulation, which is - # relay-only mechanics and must not flip a kernel link to the relay - # on reopen/reset. - if KERNEL_UNSUPPORTED_FILTERS.intersection(self._filters or {}): - return False if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): return False return True @@ -229,10 +222,10 @@ class UDPLink(Link): # Markers ride the NIO like on the relay datapath, routed by # capture node; they attach to the veth host end via uBridge's # AF_PACKET marker module instead of a relay `mark` filter. - # Filters (delay/packet_loss/corrupt plus the netem extensions - # rate/reorder/gemodel/duplicate/seed/limit — eligibility - # guarantees frequency_drop is absent) become one tc netem qdisc - # per veth host end, pushed to both endpoints. + # Filters become one tc netem qdisc per veth host end (the netem + # surface and its extensions), plus cls_bpf match-drop (bpf) and + # the eBPF stateful classifier (frequency_drop, quota) — every + # GNS3 filter type has a kernel equivalent now. bridge_name = "gns3" + self._id.replace("-", "")[:11] node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True) node1_markers, node2_markers = self._get_node_markers(node1, node2) diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index cfc9fd43f..9946cf755 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -38,7 +38,7 @@ class UDPNIO(BaseModel): filters: Optional[dict] = Field( None, description="Packet filters (relay-supported types only: delay/packet_loss/corrupt/frequency_drop/bpf — " - "the netem extensions need a kernel-datapath NIO)", + "the netem extensions and the quota mode need a kernel-datapath NIO)", ) markers: Optional[dict] = Field(None, description="Traffic-insight markers") @@ -81,9 +81,9 @@ class BridgeNIO(BaseModel): delay/packet_loss/corrupt and the netem extensions (rate, reorder, gemodel, duplicate, seed, limit, jitter distributions, loss correlation) as one tc netem qdisc, bpf as cls_bpf match-drop classifiers (needs a - uBridge reporting cbpf). frequency_drop still has no kernel equivalent - and is rejected. Markers and packet capture are served by uBridge's - AF_PACKET modules on the veth host end. + uBridge reporting cbpf), frequency_drop and quota as the eBPF stateful + classifier (needs ebpf). Markers and packet capture are served by + uBridge's AF_PACKET modules on the veth host end. """ type: BridgeNIOType @@ -91,8 +91,8 @@ class BridgeNIO(BaseModel): suspend: Optional[bool] = Field(None, description="Suspend the NIO") filters: Optional[dict] = Field( None, - description="Packet filters served on the veth host end (delay/packet_loss/corrupt and the netem extensions " - "via one tc netem qdisc, bpf via cls_bpf)", + description="Packet filters served on the veth host end (netem surface + extensions via one tc netem qdisc, " + "bpf via cls_bpf, frequency_drop/quota via the eBPF classifier)", ) markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the veth host end)") diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index 7f2f6d0e4..c986453ed 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -16,18 +16,18 @@ class FilterValidationError(Exception): pass -# Packet filters with no kernel-datapath equivalent yet: frequency_drop needs -# the eBPF stateful classifier (frozen in -# docs/design/ubridge-kernel-impairment-spec.md, part B — not delivered), -# so it still only runs in the uBridge userspace relay. bpf is served by the -# cBPF match-drop classifier (tc bpf_drop) when uBridge reports cbpf=1. -KERNEL_UNSUPPORTED_FILTERS = frozenset({"frequency_drop"}) - -# Packet filters with no uBridge *relay* equivalent: the tc netem extensions -# (uBridge feature/tc-netem-ext) serve them on the kernel datapath only. The -# relay's packet-filter registry only knows frequency_drop / packet_loss / -# delay / corrupt / bpf / mark. -KERNEL_ONLY_FILTERS = frozenset({"rate", "reorder", "gemodel", "duplicate", "seed", "limit"}) +# Every GNS3 filter type now has a kernel-datapath equivalent: delay / +# packet_loss / corrupt / the netem extensions run as one tc netem qdisc, +# bpf as cBPF match-drop classifiers, and frequency_drop as the eBPF +# stateful classifier's exact every-Nth mode (tc nth_drop — needs a uBridge +# reporting ebpf=1). No type is relay-only anymore; the controller-side +# kernel/relay choice is purely topological (same compute, docker/docker). +# +# Packet filters with no uBridge *relay* equivalent: the netem extensions +# plus the eBPF quota mode run on the kernel datapath only. The relay's +# packet-filter registry only knows frequency_drop / packet_loss / delay / +# corrupt / bpf / mark. +KERNEL_ONLY_FILTERS = frozenset({"rate", "reorder", "gemodel", "duplicate", "seed", "limit", "quota"}) # Jitter distributions embedded in the netem-extension uBridge (tc_netem_dist). NETEM_DISTRIBUTIONS = ("uniform", "normal", "pareto", "paretonormal") @@ -220,6 +220,15 @@ def validate_filter_parameters(filter_type: str, values: List[Any]) -> None: }, "seed": {"params_count": (1, 1), "ranges": [(0, 4294967295)], "names": ["Seed"], "units": [""]}, "limit": {"params_count": (1, 1), "ranges": [(1, 1000000)], "names": ["Limit"], "units": ["packets"]}, + "quota": { + # eBPF stateful classifier (uBridge tc quota_drop): after the byte + # quota is consumed, each further packet drops with the given + # chance (100 = hard cutoff). Kernel-datapath only. + "params_count": (2, 2), + "ranges": [(1, 10**15), (0, 100)], + "names": ["Quota", "Chance"], + "units": ["bytes", "%"], + }, "bpf": {"params_count": (1, 1), "is_text": True, "names": ["Filters"]}, } diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 1868ee577..14fd81679 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -34,7 +34,7 @@ from tests.utils import AsyncioMagicMock from gns3server.compute.docker import Docker from gns3server.compute.docker.docker_vm import DockerVM -from gns3server.compute.docker.docker_error import DockerError +from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error from gns3server.compute.compute_error import ComputeError from gns3server.compute.nios.nio_udp import NIOUDP from gns3server.compute.nios.nio_bridge import NIOBridge @@ -102,16 +102,28 @@ def test_create_nio_bridge_accepts_bpf_filters(vm): assert nio.filters == {"bpf": ["icmp"]} -def test_create_nio_bridge_rejects_relay_only_filters(vm): +def test_create_nio_bridge_accepts_frequency_drop(vm): + """ + frequency_drop runs on the kernel datapath as the eBPF classifier's + exact every-Nth mode — every filter type is accepted on a kernel NIO. + """ - with pytest.raises(ComputeError, match="kernel-datapath"): - vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"frequency_drop": [10]}}) + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"frequency_drop": [10]}}) + assert isinstance(nio, NIOBridge) + assert nio.filters == {"frequency_drop": [10]} + + +def test_create_nio_bridge_accepts_quota(vm): + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "filters": {"quota": [1000000, 100]}}) + assert isinstance(nio, NIOBridge) + assert nio.filters == {"quota": [1000000, 100]} def test_create_nio_bridge_accepts_markers(vm): """ Markers ride the kernel NIO (attached to the veth host end via uBridge's - AF_PACKET marker module); only filters are rejected. + AF_PACKET marker module). """ nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE, "markers": {"m": {"bpf": "icmp"}}}) @@ -421,26 +433,134 @@ async def test_connect_nio_udp_on_veth_is_idempotent(vm): @pytest.mark.asyncio -async def test_update_nio_kernel_rejects_relay_only_filters(vm): +async def test_update_nio_kernel_applies_ebpf_drops(vm): """ - The PUT route mutates the NIO in place (no create_nio), so the update - path carries its own guard against relay-only filter types. + frequency_drop becomes the eBPF every-Nth mode and quota the byte-cap + mode, applied after the netem/bpf pass on the same NIO update. + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1") + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"frequency_drop": [3], "quota": [1000000, 50]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" 3') + vm._ubridge_send.assert_any_call(f'tc quota_drop "{host_ifc}" 1000000 50') + + +@pytest.mark.asyncio +async def test_update_nio_kernel_ebpf_drop_everything(vm): + """ + frequency_drop -1 (drop everything) maps to every 1st packet. + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1") + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"frequency_drop": [-1]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" 1') + # the quota mode is explicitly off (not left to chance) + vm._ubridge_send.assert_any_call(f'tc quota_drop "{host_ifc}" off') + + +@pytest.mark.asyncio +async def test_update_nio_kernel_turns_ebpf_modes_off(vm): + """ + A re-apply with the stateful filters removed must turn both modes off + (the netem reset already detached the program; the offs are idempotent). + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1") + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"frequency_drop": [3]} + + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.reset_mock() + nio.filters = {"delay": [50]} + await vm.adapter_update_nio_binding(0, nio) + + vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" off') + vm._ubridge_send.assert_any_call(f'tc quota_drop "{host_ifc}" off') + for c in vm._ubridge_send.call_args_list: + assert "nth_drop" not in c.args[0] or "off" in c.args[0] + + +@pytest.mark.asyncio +async def test_apply_ebpf_drops_requires_ebpf_capability(vm): + """ + An old or non-capped uBridge (ebpf=0) gets a clear upgrade error naming + the setcap fix. + """ + + hyp = MagicMock() + hyp.is_running.return_value = True + hyp.send = AsyncioMagicMock(return_value=["netem=delay;ebpf=0;cbpf=1"]) + vm._ubridge_hypervisor = hyp + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="setcap cap_bpf"): + await vm._ubridge_apply_ebpf_drops(host_ifc, {"frequency_drop": [3]}) + vm._ubridge_send.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_ebpf_drops_skips_old_ubridge_without_filters(vm): + """ + No stateful filters and no capability probe yet: nothing was ever + installed on an old uBridge, so it must not be touched at all. """ vm._ubridge_hypervisor = MagicMock() - nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) - vm._ethernet_adapters[0].add_nio(0, nio) - nio.filters = {"frequency_drop": [10]} + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) - with pytest.raises(DockerError, match="kernel-datapath"): - await vm.adapter_update_nio_binding(0, nio) + await vm._ubridge_apply_ebpf_drops(host_ifc, {}) + vm._ubridge_send.assert_not_called() + vm._ubridge_hypervisor.send.assert_not_called() -def _caps_hypervisor(cbpf="1"): +@pytest.mark.asyncio +async def test_apply_ebpf_drops_cleanup_only_probes_with_ebpf(vm): + """ + The filter-removal path (modes off) only talks to the eBPF commands + when the capability probe reported them — a netem-ext uBridge without + the eBPF module must not see an unknown command. + """ + + hyp = MagicMock() + hyp.is_running.return_value = True + hyp.send = AsyncioMagicMock(return_value=["netem=delay,rate;ebpf=0;cbpf=1"]) + vm._ubridge_hypervisor = hyp + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._ubridge_tc_caps = None # force the probe (something was set before) + + await vm._ubridge_apply_ebpf_drops(host_ifc, {}) + vm._ubridge_send.assert_not_called() + + +def _caps_hypervisor(cbpf="1", ebpf="0"): """A hypervisor mock whose direct send() answers `tc capabilities`.""" hyp = MagicMock() hyp.is_running.return_value = True - hyp.send = AsyncioMagicMock(return_value=[f"netem=delay;ebpf=0;cbpf={cbpf}"]) + hyp.send = AsyncioMagicMock(return_value=[f"netem=delay;ebpf={ebpf};cbpf={cbpf}"]) return hyp @@ -1027,3 +1147,25 @@ def test_create_nio_udp_rejects_kernel_only_filters(vm): vm.manager.create_nio( {"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1", "filters": {"gemodel": [100, 0, 30]}} ) + + +@pytest.mark.asyncio +async def test_stop_clears_relay_bridge_registry(vm): + """ + The uBridge process dies with every bridge it hosted — stop() must drop + the relay bridge names, or the next start skips "bridge create" (the + gate checks the name's presence) and fails on add_nio_udp. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm._bridges.add("bridge1") + vm._ubridge_tc_caps = {"netem": "delay"} + + async def query(method, path, *args, **kwargs): + raise DockerHttp404Error("no such container") + + vm.manager.query = AsyncioMagicMock(side_effect=query) + await vm.stop() + assert vm._bridges == set() + assert vm._ubridge_tc_caps is None diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 645ab32f1..a55a6aa52 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -68,6 +68,21 @@ def _mock_compute_http(compute): compute.post.side_effect = port_callback +async def _relay_link(project): + """A docker↔qemu link (mixed node types) — never kernel-eligible, so the + relay _prepare path runs (UDP ports reserved through the mocked compute).""" + + compute = MagicMock() + compute.id = "compute-1" + _mock_compute_http(compute) + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "qemu1", node_type="qemu") + link = UDPLink(project) + await link.add_node(node1, 0, 0, batch=True, dump=False) + await link.add_node(node2, 0, 0, batch=True, dump=False) + return link, node1, node2 + + @pytest.mark.asyncio async def test_kernel_datapath_eligible(project): @@ -117,18 +132,16 @@ async def test_kernel_datapath_eligible_with_netem_filters(project): @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_with_relay_only_filters(project): +async def test_kernel_datapath_eligible_with_frequency_drop(project): """ - frequency_drop only exists in the uBridge userspace relay (its eBPF - classifier is not delivered yet) — a link carrying one stays on the - relay. bpf is kernel-capable (cls_bpf match-drop) and eligible. + frequency_drop runs on the kernel datapath too (the eBPF classifier's + exact every-Nth mode) — no filter type forces the relay anymore; only + topology does (mixed node types, cross-compute, unix-socket + containers, config off). """ link, node1, node2 = await _kernel_link(project) - link._filters = {"delay": [10, 0], "frequency_drop": [7]} - assert link._kernel_datapath_eligible(node1, node2) is False - - link._filters = {"bpf": ["icmp"]} + link._filters = {"delay": [10, 0], "frequency_drop": [7], "quota": [1000000, 100]} assert link._kernel_datapath_eligible(node1, node2) is True @@ -277,13 +290,20 @@ async def test_update_netem_filters_accepted_on_kernel_link(project): @pytest.mark.asyncio -async def test_update_relay_only_filters_rejected_on_kernel_link(project): +async def test_update_frequency_drop_accepted_on_kernel_link(project): + """ + frequency_drop rides the kernel NIO to both endpoints like every other + filter type — the compute translates it to the eBPF every-Nth mode. + """ - link, _n1, _n2 = await _kernel_link(project) + link, node1, node2 = await _kernel_link(project) await link._prepare() - with pytest.raises(ControllerError, match="kernel-datapath"): - await link.update_filters({"frequency_drop": [10]}) + await link.update_filters({"frequency_drop": [10]}) + assert link.filters == {"frequency_drop": [10]} + by_node = {entry[0].id: entry[3] for entry in (await link._prepare())} + assert by_node[node1.id]["filters"] == {"frequency_drop": [10]} + assert by_node[node2.id]["filters"] == {"frequency_drop": [10]} @pytest.mark.asyncio @@ -449,14 +469,12 @@ async def test_update_netem_extension_filters_accepted_on_kernel_link(project): @pytest.mark.asyncio async def test_update_kernel_only_filters_rejected_on_relay_link(project): """ - Netem-extension filters have no relay equivalent: setting them on a - created relay link is a 409. (frequency_drop in the stored filters - forces this link onto the relay datapath.) + Netem-extension / quota filters have no relay equivalent: setting them + on a created relay link is a 409. (The link is relay-wired because the + endpoints are mixed node types.) """ - link, _n1, _n2 = await _kernel_link(project) - _mock_compute_http(link._nodes[0]["node"].compute) - link._filters = {"frequency_drop": [7]} + link, _n1, _n2 = await _relay_link(project) await link._prepare() assert link.kernel_datapath is False link._created = True # the computes hold the UDP NIOs @@ -464,6 +482,10 @@ async def test_update_kernel_only_filters_rejected_on_relay_link(project): with pytest.raises(ControllerError, match="kernel-datapath"): await link.update_filters({"rate": ["512kbit"]}) # the stored filters are untouched + assert link.filters == {} + + # frequency_drop still runs on the relay (its uBridge userspace filter) + await link.update_filters({"frequency_drop": [7]}) assert link.filters == {"frequency_drop": [7]} @@ -484,13 +506,13 @@ async def test_update_kernel_only_filters_accepted_before_creation(project): @pytest.mark.asyncio async def test_prepare_relay_link_strips_kernel_only_filters(project): """ - A relay-wired link (here: frequency_drop in the stored filters) built - from a topology carrying extension filters drops them with a warning - instead of pushing an unknown filter type to uBridge. + A relay-wired link (mixed node types) built from a topology carrying + extension filters drops them with a warning instead of pushing an + unknown filter type to uBridge. frequency_drop stays — the relay's + userspace filter still serves it. """ - link, node1, node2 = await _kernel_link(project) - _mock_compute_http(link._nodes[0]["node"].compute) + link, node1, node2 = await _relay_link(project) link._filters = {"frequency_drop": [7], "rate": ["512kbit"], "delay": [100, 20, "normal"], "packet_loss": [5, 25]} entries = await link._prepare() assert link.kernel_datapath is False diff --git a/tests/utils/test_packet_filter_validation.py b/tests/utils/test_packet_filter_validation.py index 034114f65..83fcf2faf 100644 --- a/tests/utils/test_packet_filter_validation.py +++ b/tests/utils/test_packet_filter_validation.py @@ -366,3 +366,20 @@ class TestNetemExtensionFilters: def test_kernel_only_features_empty(self): assert split_kernel_only_features({}) == ({}, set()) assert split_kernel_only_features(None) == ({}, set()) + + def test_quota_valid_and_invalid(self): + validate_filter_parameters("quota", [1000000, 100]) + validate_filter_parameters("quota", [1, 0]) + with pytest.raises(FilterValidationError, match="Quota"): + validate_filter_parameters("quota", [0, 100]) + with pytest.raises(FilterValidationError, match="Chance"): + validate_filter_parameters("quota", [1000, 101]) + with pytest.raises(FilterValidationError, match="expects 2 parameter"): + validate_filter_parameters("quota", [1000]) + + def test_quota_is_kernel_only(self): + clean, dropped = split_kernel_only_features({"quota": [1000, 50], "frequency_drop": [7]}) + assert dropped == {"quota"} + # frequency_drop runs on both datapaths (relay userspace filter / + # eBPF every-Nth) and stays + assert clean == {"frequency_drop": [7]} From 316e7002c10458130d192fafe5dae2625aea0bfd Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 01:28:59 +0800 Subject: [PATCH 013/101] feat(docker): expose the eBPF time-window filter on kernel links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uBridge's window mode gained the semantics the B.2 spec asked for — `tc window_drop [ []]`: a single outage that traffic passes before *and* after, recurring flaps with period >= outage, and a per-cycle jitter — so the kernel-only `window_drop` filter type is now exposed: - validation mirrors the tc grammar (3-5 params, ranges, period >= outage, jitter cap) and adds window_drop to KERNEL_ONLY_FILTERS; filter_inactive_filters keeps start=0 — unlike every other filter a zero first parameter means "the outage starts now", not "disabled". - `_ubridge_apply_ebpf_drops` sends `tc window_drop` (positional pass-through) and turns the mode off explicitly when absent, under the same ebpf=1 gate. - FILTERS entry (Start/Outage/Chance/Period/Jitter) + available_filters. The schedule is relative to the apply: reconcile re-sets the classifier on every apply, so any filter change, node restart or link reset restarts the window. Verified on a two-container kernel link, 15/15: a single outage [1000, 1500, 100] arrives per-seq as [0..4, 13..19] (passing before the window, dropped inside, passing after); recurring 1000/2000 ms = 52 % round trip (duty cycle); period == outage = permanent outage; period < outage = 409 before uBridge; jittered flap keeps the ~50 % duty cycle; 30 % chance on a permanent window = 53 % round trip against the per-direction theory 1-0.7^2 = 51 %; coexist with netem; node restart restores; available_filters kernel vs relay (409 + hidden); teardown clean. P6b re-run as a regression on the same uBridge build: 13/13. Worth recording for the next rebuild: the first delivered window program loaded as root but reported ebpf=0 through gns3-server — the verifier's non-root path loses the loop counter's bound, treats the catch-up loop as unbounded and rejects the *whole* program (E2BIG, 8193 pending states), which also took frequency_drop and quota down, with ebpf=0 as the only symptom. Smaller trip caps did not help; the loop had to go (uBridge 572ca0c), on top of a Makefile header-dependency fix (53e34bb) that had been shipping a stale program object. --- docs/design/ubridge-kernel-impairment-spec.md | 74 ++++++++++++------- docs/features/docker-kernel-datapath.md | 71 +++++++++++++++++- gns3server/compute/docker/docker_vm.py | 36 +++++++-- gns3server/controller/link.py | 20 ++++- gns3server/utils/packet_filter_validation.py | 32 ++++++-- .../docker/test_docker_kernel_datapath.py | 49 ++++++++++++ tests/controller/test_kernel_datapath_link.py | 33 ++++++++- tests/utils/test_packet_filter_validation.py | 33 +++++++++ 8 files changed, 300 insertions(+), 48 deletions(-) diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md index ad795cc2e..454879a45 100644 --- a/docs/design/ubridge-kernel-impairment-spec.md +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -9,11 +9,12 @@ See LICENSE file for licensing information. > idempotent `tc reset`, `tc capabilities` — uBridge `feature/tc-bpf-drop`) > and **B (eBPF stateful classifier — uBridge `feature/tc-precision` > 59e2b38, including the non-root setcap load fix)** are consumed by -> gns3-server on the branch stack ending at `feat/docker-kernel-ebpf-drops` -> (B: `frequency_drop` → `tc nth_drop`, the new kernel-only `quota` type → -> `tc quota_drop`). The originally shipped netem surface (delay/jitter/ -> loss/dup/corrupt) exists in uBridge 1.2.3+ and is verified against -> gns3-server branch `feat/docker-kernel-filters`. +> gns3-server on the branch stack ending at `feat/docker-kernel-window-drop` +> (B: `frequency_drop` → `tc nth_drop`, the kernel-only `quota` type → +> `tc quota_drop`, and the kernel-only `window_drop` type → `tc window_drop` +> against uBridge `feature/tc-window`). The originally shipped netem surface +> (delay/jitter/loss/dup/corrupt) exists in uBridge 1.2.3+ and is verified +> against gns3-server branch `feat/docker-kernel-filters`. > > **Deviation found during integration (part A/D):** the delivered > `tc netem set` uses NLM_F_REPLACE, but the *kernel's* netem change merges @@ -27,19 +28,32 @@ See LICENSE file for licensing information. > optional attribute, the server-side reset-before-set can be dropped; the > two behaviours are compatible either way. > -> **Deviation found during B integration:** `window_drop`'s implementation -> advances the window start by one length on expiry with no gap, making the -> windows back-to-back — after the first `start_ms` the "outside the window -> packets pass" behaviour described in B.2 is unreachable (inside ≡ always). -> gns3-server therefore does **not** expose window_drop yet; it needs either -> a second length field (outage vs period) or a next-start semantics fix on -> the uBridge side. `flow_drop` is delivered and functional but not yet -> exposed as a GNS3 filter type (its mask/target parameter shape needs a -> UX decision). Additionally, the delivered B.1 program had to fold the -> flow-hash L4 port reads to constant offsets (IHL==20 only) because the -> verifier prohibits variable packet-pointer arithmetic for non-root — -> even with CAP_BPF (Spectre-mitigation gating by uid); documented in -> uBridge's doc/tc.md. +> **Deviation found during B integration, resolved uBridge-side +> (2026-09-29):** `window_drop` as first delivered advanced the window +> start by one length on expiry with no gap, making the windows +> back-to-back — after the first `start_ms` the "outside the window +> packets pass" behaviour described in B.2 was unreachable (inside ≡ +> always), so gns3-server did not expose it. uBridge now takes +> `tc window_drop [ +> []]`: 4-arg = single window (pass before *and after*), +> 5-arg = recurring outage with `period ≥ outage`, 6-arg adds a uniform +> ±jitter re-drawn per cycle (0 = exactly the fixed schedule) for +> randomized flap. The program advances whole cycles in a handful of +> straight-line steps — **no loop at all**, because the verifier's +> non-root path (the one production ubridge runs under: `CAP_BPF`, never +> uid 0) cannot keep a loop counter's bound and rejects the *whole* +> program with `E2BIG` past its jump-sequence budget, which took all four +> modes down on the gns3-server host while the same binary verified fine +> as root; uBridge's `doc/tc.md` has the trace. The steps are resumable by +> each further packet, so the +> B.2 semantics are now reachable; no caller ever depended on the old +> back-to-back form (it was never exposed). `flow_drop` is delivered and +> functional but not yet exposed as a GNS3 filter type (its mask/target +> parameter shape needs a UX decision). Additionally, the delivered B.1 +> program had to fold the flow-hash L4 port reads to constant offsets +> (IHL==20 only) because the verifier prohibits variable packet-pointer +> arithmetic for non-root — even with CAP_BPF (Spectre-mitigation gating +> by uid); documented in uBridge's doc/tc.md. # uBridge kernel impairment: tc netem extensions + eBPF classifiers @@ -176,7 +190,7 @@ no program reload on parameter changes. ``` tc nth_drop tc quota_drop | off -tc window_drop | off +tc window_drop [ []] | off tc flow_drop | off ``` @@ -189,10 +203,15 @@ tc flow_drop | off hash = Jenkins/equal-fold over the selected header fields modulo `flow_target` == remainder 0 → drop. Server documents: per-flow drop only (a filter cannot delay; delay stays netem's job). -* Window semantics: `win_start_ns`/`win_len_ns` define a recurring window on - the monotonic clock — inside the window drop with `win_pct`; outside pass. - The server computes the first start; the program advances by `len` when - `now >= start + len` (single writer — the CFG update — so this is safe). +* Window semantics: `win_start_ns`/`win_len_ns` define a window on the + monotonic clock — inside the window drop with `win_pct`; outside pass. + The program computes the first start (command time + `start_ms`) and then + advances whole cycles, catching up in a bounded loop when it fell behind + (single writer — the CFG update — so this is safe). Omitting `period` + makes it a **single** window: after `start + outage` nothing drops. With + `period ≥ outage` the outage recurs; `jitter` re-draws each cycle's + outage and period uniformly in nominal ± jitter (0 = fixed schedule), on + the integer-ms grid so the multiply-shift draw cannot overflow. ### B.3 Capability requirement @@ -312,8 +331,13 @@ corresponding filter types per capability. kernel-only `quota` type → `tc quota_drop`, both keyed on `ebpf=1` from the per-process `tc capabilities` probe. `KERNEL_UNSUPPORTED_FILTERS` is gone entirely: kernel/relay eligibility is purely topological now. - Not yet exposed: `window_drop` (implementation deviation above), - `flow_drop` (parameter-shape UX decision). +* **Done** (`feat/docker-kernel-window-drop`): `window_drop` is exposed + with the corrected semantics (uBridge `feature/tc-window`) as the + kernel-only `window_drop` filter type + `[start, outage, pct, period?, jitter?]` → `tc window_drop`; the server + mirrors the grammar's `period ≥ outage` rule and treats `start = 0` as + active (an immediate outage). + Not yet exposed: `flow_drop` (parameter-shape UX decision). * Every veth end owns one qdisc + its filters: per-direction impairment is an architectural freebie to expose later (API `direction` field), aligned with marker `dir` semantics. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 0f5bacf48..426485902 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -24,14 +24,14 @@ Frames never leave the kernel. | Link | uBridge bridge, TAP fd ↔ UDP, userspace copy | per-link kernel bridge `gns3{link_id[:11]}`, zero-copy | | Relay latency | ~0.3 ms RTT | ~0.05 ms RTT (measured) | | Link attach at runtime | yes (relay attaches the pre-existing TAP) | yes (brctl addif / add_nio_ethernet — interface untouched) | -| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt / duplicate + the netem extensions rate, reorder, gemodel, seed, limit, jitter distributions, loss correlation); bpf as cls_bpf match-drop classifiers; frequency_drop and quota as the eBPF stateful classifier — every filter type has a kernel equivalent | +| Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt / duplicate + the netem extensions rate, reorder, gemodel, seed, limit, jitter distributions, loss correlation); bpf as cls_bpf match-drop classifiers; frequency_drop, quota and window_drop as the eBPF stateful classifier — every filter type has a kernel equivalent | | Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | -The work landed in eight stages on branch stack `feat/docker-kernel-datapath` → +The work landed in nine stages on branch stack `feat/docker-kernel-datapath` → `feat/docker-kernel-capture` → `feat/docker-kernel-markers` → `feat/docker-veth-everywhere` → `feat/docker-kernel-filters` → `feat/docker-kernel-bpf-drop` → `feat/docker-kernel-netem-ext` → -`feat/docker-kernel-ebpf-drops`: +`feat/docker-kernel-ebpf-drops` → `feat/docker-kernel-window-drop`: 1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based suspend, crash-safe reconciliation. @@ -66,6 +66,14 @@ The work landed in eight stages on branch stack `feat/docker-kernel-datapath` non-root server process, the production credential shape). With this, no filter type forces the relay anymore: kernel/relay is a purely topological choice. +9. **window_drop flaps** — the eBPF classifier's time-window mode + (`tc window_drop`, uBridge `feature/tc-window`): a single outage that + passes traffic before and after, a recurring flap with `period ≥ outage`, + and an optional per-cycle jitter that randomizes each beat's outage and + period. The originally delivered back-to-back window implementation + (which made "outside the window packets pass" unreachable) was fixed + uBridge-side; this stage exposes the corrected semantics as the + kernel-only `window_drop` filter type. ## Adapter interface types @@ -201,6 +209,7 @@ ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): | `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | | `frequency_drop [N]` | `tc nth_drop N` — the eBPF stateful classifier's exact every-Nth mode (clsact egress prio 1) | needs uBridge `ebpf=1`; -1 = drop everything → every 1st; per-direction counters (see below) | | `quota [bytes, %]` | `tc quota_drop ` — after the byte quota is consumed, each further packet drops with the given chance (100 = hard cutoff) | kernel-only type; needs `ebpf=1`; per-direction byte counters | +| `window_drop [start, outage, %, period?, jitter?]` | `tc window_drop [ []]` — packets drop with the given chance inside `[start, start+outage)` | kernel-only type; needs `ebpf=1`; see the window semantics below | Semantics: @@ -241,7 +250,25 @@ Semantics: * **Classifier order** on clsact egress: the eBPF stateful filter runs at prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the stateful modes never reaches the expression drops, and dropped packets - never reach netem. + never reach netem. Within the stateful program the modes evaluate in the + fixed order nth → quota → window. +* **window_drop semantics.** `start` is relative to the moment the filter + is applied — and because reconcile re-sets the classifier on every apply + (any filter change, node restart, link reset reloads the program after + the netem reset), every such event **restarts the schedule**. 3 + parameters = one single outage (`[start, start+outage)`), traffic passes + before and after — the classic "link dies at T, comes back at T+outage". + 4 parameters = recurring flap: `period ≥ outage` (validated), so the + outage occupies `outage/period` of each cycle. 5 parameters = randomized + flap: each cycle's outage and period are re-drawn uniformly in nominal + ± `jitter` (integer-ms grid; jitter 0 draws nothing and equals the fixed + schedule). Inside a window each packet drops with the given chance (100 + = full outage, lower = degraded service during the window). Both + endpoints run their own schedule, started ~simultaneously by the + controller — a round trip survives only when **both** directions are + outside their windows. Note `start = 0` (outage begins immediately) is a + valid, active configuration — unlike other filters, a zero first + parameter does not mean "disabled". * **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + @@ -278,6 +305,7 @@ tc netem set [delay ] [jitter ] [loss <%> [correl <%>] | loss gemod tc bpf_drop add "" / tc bpf_drop flush tc nth_drop # eBPF exact every-Nth (frequency_drop) tc quota_drop | off # eBPF byte cap (quota) +tc window_drop [ []] | off # eBPF time window (window_drop) tc reset # full restore: eBPF filter -> bpf_drops -> clsact -> root qdisc, idempotent tc capabilities # "netem=;ebpf=0|1;cbpf=0|1", probed once per process bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capture @@ -358,3 +386,38 @@ node restart with a relay link skipped `bridge create` and failed. Unit tests: `tests/utils/test_packet_filter_validation.py` (P6a class), `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. + +window_drop (same setup, 15/15 checks, against uBridge `feature/tc-window`): +a single outage `[1000, 1500, 100]` delivers **per-seq evidence** of the B.2 +semantics — one ping per 200 ms arrives as `[0,1,2,3,4, 13…19]`: passing +before the window, dropped inside it, passing again after; a recurring flap +`[300, 1000, 100, 2000]` measures 52 % round-trip loss (the 1000/2000 duty +cycle); `period == outage` is a permanent outage (100 %); `period < outage` +is a 409 before uBridge sees it; a jittered flap +`[0, 1000, 100, 2000, 400]` keeps the ~50 % duty cycle (62 % sampled over +4 cycles — the per-cycle re-draw widens the variance by design); a 30 % +chance on a permanent window measures 53 % round trip against the +per-direction theory 1 − 0.7² = 51 % (the same arithmetic as +`packet_loss 30`); `start = 0` is stored as an *active* filter; the window +coexists with netem (`delay 50` + a permanent window = 100 % inside the +outage); clearing removes the classifier; a node restart restores the +schedule on the fresh veth; `available_filters` shows window_drop on kernel +links and a relay link both hides it and 409s it; teardown leaves no filter +or qdisc behind. The P6b e2e was re-run as a regression on the same uBridge +build (13/13 — nth/quota unaffected by the de-looping below). + +The uBridge-side window work took two fixes that are worth recording, both +found by *this* integration rather than by unit tests: the delivered +catch-up loop tripped the verifier's 8192 **jump-sequence** budget — on the +non-root path the verifier drops the loop counter's bound +(`R4=scalar(smax=umax32=0xfffff086)`), treats the loop as unbounded and +unrolls it until the pending-state limit, so the program was rejected +wholesale (`E2BIG`, `The sequence of 8193 jumps is too complex`) and +`tc capabilities` reported `ebpf=0` — taking `frequency_drop` and `quota` +down with it, since the classifier loads as one program; lowering the trip +cap did not help (128 trips failed identically), removing the loop did +(572ca0c: 16-step unroll + outer guard, zero back edges). A latent Makefile +header-dependency gap (53e34bb) had been shipping the *old* truncated +program object — the kernel answered `jump out of range from insn 9 to +421` with `processed 0 insns`. Both matter to anyone rebuilding uBridge: +`ebpf=1` from a direct probe is the only trustworthy signal. diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 62316dd7b..0dd0dadd4 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1950,7 +1950,8 @@ class DockerVM(BaseNode): # Impairment filters become one tc netem qdisc on the veth host # end (restored here on node restart, like the capture above); # bpf expressions become cls_bpf match-drop classifiers and - # frequency_drop/quota become the eBPF stateful classifier. + # frequency_drop/quota/window_drop become the eBPF stateful + # classifier. await self._ubridge_apply_netem(host_ifc, nio.filters) await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) await self._ubridge_apply_ebpf_drops(host_ifc, nio.filters) @@ -2379,20 +2380,28 @@ class DockerVM(BaseNode): Translate the NIO's stateful filters into uBridge's eBPF impairment classifier on the veth host end's clsact egress (prio 1, below bpf_drop's 10-99): ``frequency_drop`` becomes the exact every-Nth - mode (``tc nth_drop``; -1 = drop everything maps to every 1st), and - the kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``). + mode (``tc nth_drop``; -1 = drop everything maps to every 1st), the + kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``), + and the kernel-only ``window_drop`` type the time-window mode + (``tc window_drop``: single outage, or recurring flaps with an + optional per-cycle jitter). The program evaluates the modes in the + fixed order nth → quota → window. Reconcile is a plain re-set on every apply: the preceding netem apply resets the interface (which detaches the program and drops uBridge's per-interface registry entry), so a mode set re-loads the program from scratch, and a mode left absent is turned off - explicitly (idempotent) so a removed filter stops dropping. + explicitly (idempotent) so a removed filter stops dropping. For + window_drop this also means the schedule restarts on every apply: + the first window opens Start-ms from the moment of this call. Semantic difference vs the relay, documented server-side: the relay counts packets of BOTH directions through its single filtered bridge, while each veth end's classifier counts only the traffic entering that container — with both endpoints applying theirs, each - direction drops every Nth independently. + direction drops every Nth independently (their window schedules + start ~simultaneously, so a round trip survives only when both + directions are outside their windows). :param host_ifc: veth host-end interface name :param filters: NIO filters dictionary @@ -2406,24 +2415,28 @@ class DockerVM(BaseNode): frequency = values_of("frequency_drop") quota = values_of("quota") - if not frequency and not quota and self._ubridge_tc_caps is None: + window = values_of("window_drop") + if not frequency and not quota and not window and self._ubridge_tc_caps is None: # Nothing to set and nothing was ever installed (no probe yet) — # skip touching an old uBridge entirely. return caps = await self._ubridge_tc_capabilities() - if not frequency and not quota: + if not frequency and not quota and not window: if caps.get("ebpf") == "1": # Only when the eBPF module answers can anything be # installed; "off" is idempotent in uBridge. await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') + await self._ubridge_send(f'tc window_drop "{host_ifc}" off') return if caps.get("ebpf") != "1": raise DockerError( "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " "the uBridge binary); upgrade uBridge on this compute or keep the link on the " - "relay datapath".format(", ".join(sorted(f for f in ("frequency_drop", "quota") if f in filters))) + "relay datapath".format( + ", ".join(sorted(f for f in ("frequency_drop", "quota", "window_drop") if f in filters)) + ) ) if frequency: # relay semantics: -1 = drop everything, N = every Nth packet @@ -2437,6 +2450,13 @@ class DockerVM(BaseNode): ) else: await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') + if window: + # [start, outage, chance, period?, jitter?] — start is relative + # to this apply (the schedule restarts on every reconcile). + params = " ".join(str(int(v)) for v in window) + await self._ubridge_send(f'tc window_drop "{host_ifc}" {params}') + else: + await self._ubridge_send(f'tc window_drop "{host_ifc}" off') async def _remove_kernel_markers(self, host_ifc): """ diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 0b69981a4..a6ff17822 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -147,6 +147,20 @@ FILTERS = [ {"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, ], }, + { + "type": "window_drop", + "name": "Time window drop", + "description": "Drop packets with the given chance inside a time window starting Start ms from the " + "moment the filter is applied — a single outage (traffic passes before and after), or with a " + "period recurring flaps whose per-cycle timing a jitter randomizes (kernel-datapath links only)", + "parameters": [ + {"name": "Start", "minimum": 0, "maximum": 1000000000000, "type": "int", "unit": "ms"}, + {"name": "Outage", "minimum": 1, "maximum": 1000000000000, "type": "int", "unit": "ms"}, + {"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "Period", "minimum": 1, "maximum": 1000000000000, "type": "int", "unit": "ms"}, + {"name": "Jitter", "minimum": 0, "maximum": 1000000000, "type": "int", "unit": "ms"}, + ], + }, ] @@ -308,7 +322,8 @@ class Link: raise ControllerError(f"Invalid packet filter parameters: {e!s}") if self._created and not self.kernel_datapath: - # Kernel-only filters (the netem extensions, the eBPF quota mode) + # Kernel-only filters (the netem extensions, the eBPF quota and + # window modes) # have no relay equivalent. Only enforced on created links — while # loading a project the datapath is not decided yet (a link that # will be wired on the kernel datapath must accept them), and the @@ -723,7 +738,8 @@ class Link: if self.kernel_datapath: # Kernel-datapath links serve every filter type: netem for # delay/loss/corrupt and the netem extensions, cls_bpf for - # bpf, the eBPF classifier for frequency_drop and quota. + # bpf, the eBPF classifier for frequency_drop, quota and + # window_drop. return FILTERS # Relay links: hide the kernel-only types the relay cannot run # (they would be rejected with 409 on update). diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index c986453ed..43f6a3fb2 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -24,10 +24,10 @@ class FilterValidationError(Exception): # kernel/relay choice is purely topological (same compute, docker/docker). # # Packet filters with no uBridge *relay* equivalent: the netem extensions -# plus the eBPF quota mode run on the kernel datapath only. The relay's -# packet-filter registry only knows frequency_drop / packet_loss / delay / -# corrupt / bpf / mark. -KERNEL_ONLY_FILTERS = frozenset({"rate", "reorder", "gemodel", "duplicate", "seed", "limit", "quota"}) +# plus the eBPF quota/window modes run on the kernel datapath only. The +# relay's packet-filter registry only knows frequency_drop / packet_loss / +# delay / corrupt / bpf / mark. +KERNEL_ONLY_FILTERS = frozenset({"rate", "reorder", "gemodel", "duplicate", "seed", "limit", "quota", "window_drop"}) # Jitter distributions embedded in the netem-extension uBridge (tc_netem_dist). NETEM_DISTRIBUTIONS = ("uniform", "normal", "pareto", "paretonormal") @@ -229,6 +229,19 @@ def validate_filter_parameters(filter_type: str, values: List[Any]) -> None: "names": ["Quota", "Chance"], "units": ["bytes", "%"], }, + "window_drop": { + # eBPF stateful classifier (uBridge tc window_drop): packets drop + # with the given chance inside [start, start+outage) measured from + # the moment the filter is applied — a single outage, traffic + # passes before AND after. A period makes the outage recur every + # cycle (period >= outage); a jitter re-draws each cycle's outage + # and period uniformly in nominal ± jitter (0 = the fixed + # schedule). Kernel-datapath only. + "params_count": (3, 5), + "ranges": [(0, 10**12), (1, 10**12), (0, 100), (1, 10**12), (0, 10**9)], + "names": ["Start", "Outage", "Chance", "Period", "Jitter"], + "units": ["ms", "ms", "%", "ms", "ms"], + }, "bpf": {"params_count": (1, 1), "is_text": True, "names": ["Filters"]}, } @@ -335,6 +348,12 @@ def filter_inactive_filters(filters: Dict[str, List[Any]]) -> Dict[str, List[Any else: # latency>0, normal configuration active_filters[filter_type] = values + # window_drop's first parameter is a start offset: 0 means "the + # outage starts now", not "disabled" — the filter is active whenever + # present (an outage of 0 is rejected by validation, so there is no + # natural zero-value encoding for "off"). + elif filter_type == "window_drop": + active_filters[filter_type] = values # For other filters, skip if first value is 0 or empty string (means "disabled") elif values[0] != 0 and values[0] != "": active_filters[filter_type] = values @@ -398,7 +417,7 @@ def validate_all_filters(filters: Dict[str, List[Any]]) -> None: Validate all packet filters, including cross-filter dependencies that mirror the tc netem grammar (reorder requires delay; gemodel and packet_loss both translate to the netem loss keyword and are mutually - exclusive). + exclusive) and the eBPF window grammar (period >= outage). Args: filters: Dictionary mapping filter types to their values @@ -420,3 +439,6 @@ def validate_all_filters(filters: Dict[str, List[Any]]) -> None: raise FilterValidationError("reorder requires delay") if "gemodel" in filters and "packet_loss" in filters: raise FilterValidationError("gemodel and packet_loss are mutually exclusive (both map to the netem loss keyword)") + window = filters.get("window_drop") + if window and len(window) >= 4 and int(window[3]) < int(window[1]): + raise FilterValidationError("window_drop period must be greater than or equal to the outage length") diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 14fd81679..4f3001e33 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -497,6 +497,7 @@ async def test_update_nio_kernel_turns_ebpf_modes_off(vm): vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" off') vm._ubridge_send.assert_any_call(f'tc quota_drop "{host_ifc}" off') + vm._ubridge_send.assert_any_call(f'tc window_drop "{host_ifc}" off') for c in vm._ubridge_send.call_args_list: assert "nth_drop" not in c.args[0] or "off" in c.args[0] @@ -520,6 +521,54 @@ async def test_apply_ebpf_drops_requires_ebpf_capability(vm): vm._ubridge_send.assert_not_called() +@pytest.mark.asyncio +async def test_update_nio_kernel_applies_window_drop(vm): + """ + window_drop becomes the eBPF time-window mode: the positional + [start, outage, chance, period, jitter] maps 1:1 onto the tc command + (trailing optionals omitted), and an absent window is turned off. + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1") + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + nio.filters = {"window_drop": [0, 2000, 100]} + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc window_drop "{host_ifc}" 0 2000 100') + + vm._ubridge_send.reset_mock() + nio.filters = {"window_drop": [500, 800, 100, 2400, 300]} + await vm.adapter_update_nio_binding(0, nio) + vm._ubridge_send.assert_any_call(f'tc window_drop "{host_ifc}" 500 800 100 2400 300') + + # only the window mode present: nth/quota are explicitly off + vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" off') + vm._ubridge_send.assert_any_call(f'tc quota_drop "{host_ifc}" off') + + +@pytest.mark.asyncio +async def test_apply_window_drop_requires_ebpf_capability(vm): + """ + The window mode shares the eBPF capability gate (error names setcap). + """ + + hyp = MagicMock() + hyp.is_running.return_value = True + hyp.send = AsyncioMagicMock(return_value=["netem=delay;ebpf=0;cbpf=1"]) + vm._ubridge_hypervisor = hyp + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="setcap cap_bpf"): + await vm._ubridge_apply_ebpf_drops(host_ifc, {"window_drop": [0, 2000, 100]}) + vm._ubridge_send.assert_not_called() + + @pytest.mark.asyncio async def test_apply_ebpf_drops_skips_old_ubridge_without_filters(vm): """ diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index a55a6aa52..f6ee38410 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -428,7 +428,7 @@ async def test_update_suspend_kernel_link_sends_no_synthetic_filter(project): async def test_kernel_datapath_eligible_with_netem_extension_filters(project): """ All netem-extension types have a tc netem equivalent — they keep the - link kernel-eligible (only frequency_drop still forces the relay). + link kernel-eligible (kernel/relay is a purely topological choice). """ link, node1, node2 = await _kernel_link(project) @@ -441,6 +441,8 @@ async def test_kernel_datapath_eligible_with_netem_extension_filters(project): "gemodel": [100, 0, 30], "seed": [42], "limit": [5000], + "quota": [1000000, 100], + "window_drop": [0, 800, 100, 2400, 200], } # gemodel/packet_loss are mutually exclusive — drop packet_loss for the # eligibility check (validation rejects the mix elsewhere) @@ -469,9 +471,9 @@ async def test_update_netem_extension_filters_accepted_on_kernel_link(project): @pytest.mark.asyncio async def test_update_kernel_only_filters_rejected_on_relay_link(project): """ - Netem-extension / quota filters have no relay equivalent: setting them - on a created relay link is a 409. (The link is relay-wired because the - endpoints are mixed node types.) + Netem-extension / quota / window_drop filters have no relay equivalent: + setting them on a created relay link is a 409. (The link is relay-wired + because the endpoints are mixed node types.) """ link, _n1, _n2 = await _relay_link(project) @@ -481,6 +483,11 @@ async def test_update_kernel_only_filters_rejected_on_relay_link(project): with pytest.raises(ControllerError, match="kernel-datapath"): await link.update_filters({"rate": ["512kbit"]}) + with pytest.raises(ControllerError, match="kernel-datapath"): + await link.update_filters({"window_drop": [0, 800, 100, 2400]}) + with pytest.raises(ControllerError, match="period must be greater than or equal"): + # cross-parameter rule mirrors the tc window grammar + await link.update_filters({"window_drop": [0, 2000, 100, 1000]}) # the stored filters are untouched assert link.filters == {} @@ -489,6 +496,24 @@ async def test_update_kernel_only_filters_rejected_on_relay_link(project): assert link.filters == {"frequency_drop": [7]} +@pytest.mark.asyncio +async def test_update_window_drop_accepted_on_kernel_link(project): + """ + window_drop rides the kernel NIO to both endpoints — including the + start=0 form (an immediate outage is NOT an "inactive" filter). + """ + + link, node1, node2 = await _kernel_link(project) + await link._prepare() + + filters = {"window_drop": [0, 800, 100, 2400, 200]} + await link.update_filters(filters) + assert link.filters == filters + by_node = {entry[0].id: entry[3] for entry in (await link._prepare())} + assert by_node[node1.id]["filters"] == filters + assert by_node[node2.id]["filters"] == filters + + @pytest.mark.asyncio async def test_update_kernel_only_filters_accepted_before_creation(project): """ diff --git a/tests/utils/test_packet_filter_validation.py b/tests/utils/test_packet_filter_validation.py index 83fcf2faf..d196e9d3e 100644 --- a/tests/utils/test_packet_filter_validation.py +++ b/tests/utils/test_packet_filter_validation.py @@ -383,3 +383,36 @@ class TestNetemExtensionFilters: # frequency_drop runs on both datapaths (relay userspace filter / # eBPF every-Nth) and stays assert clean == {"frequency_drop": [7]} + + def test_window_drop_valid_and_invalid(self): + # the three delivered forms: single outage, recurring, jittered + validate_filter_parameters("window_drop", [0, 2000, 100]) + validate_filter_parameters("window_drop", [500, 800, 100, 2400]) + validate_filter_parameters("window_drop", [500, 800, 100, 2400, 300]) + with pytest.raises(FilterValidationError, match="expects 3 to 5"): + validate_filter_parameters("window_drop", [500, 800]) + with pytest.raises(FilterValidationError, match="expects 3 to 5"): + validate_filter_parameters("window_drop", [500, 800, 100, 2400, 300, 1]) + with pytest.raises(FilterValidationError, match="Outage"): + validate_filter_parameters("window_drop", [0, 0, 100]) + with pytest.raises(FilterValidationError, match="Chance"): + validate_filter_parameters("window_drop", [0, 2000, 101]) + with pytest.raises(FilterValidationError, match="Jitter"): + validate_filter_parameters("window_drop", [0, 2000, 100, 2400, 10**10]) + + def test_window_drop_is_kernel_only(self): + clean, dropped = split_kernel_only_features({"window_drop": [0, 800, 100, 2400], "delay": [10, 0]}) + assert dropped == {"window_drop"} + assert clean == {"delay": [10, 0]} + + def test_window_drop_period_must_cover_outage(self): + with pytest.raises(FilterValidationError, match="period must be greater than or equal"): + validate_all_filters({"window_drop": [0, 2000, 100, 1000]}) + # period == outage is accepted (uBridge allows it); shorter is not + validate_all_filters({"window_drop": [0, 2000, 100, 2000]}) + + def test_window_drop_start_zero_is_active(self): + # start=0 means "the outage starts now" — unlike other filters a + # zero first parameter must NOT be treated as "disabled" + active = filter_inactive_filters({"window_drop": [0, 2000, 100]}) + assert active == {"window_drop": [0, 2000, 100]} From 48dbdbe4a96dd89e0f83d4da8ccd73c7f9a7220d Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 01:37:36 +0800 Subject: [PATCH 014/101] fix(docker): delete the per-link kernel bridge when a node stops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bridge teardown only hung off the link-removal path, but deleting a node or closing a project never issues a per-link delete — the veths disappeared with their containers and left the bridge behind as an empty orphan (reproduced: create a kernel link, DELETE the project, the gns3{link_id} bridge survives). `_remove_kernel_veths` now deletes the node's kernel-NIO bridges alongside the veth host ends: both endpoints do it, the peer's still-enslaved port answers EBUSY (suppressed, last one wins) exactly like link deletion, a relay NIO's uBridge bridge is left alone (it dies with the process), and a node restart rebuilds the bridge from the NIO in _connect_nio. Found while cleaning up the window_drop e2e: seven such bridges had accumulated on this host. Re-verified live: the bridge is gone after the project delete, and the window_drop e2e (which includes a node restart with the kernel link attached) still passes 15/15. --- docs/features/docker-kernel-datapath.md | 8 +++++- gns3server/compute/docker/docker_vm.py | 13 +++++++++ .../docker/test_docker_kernel_datapath.py | 28 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 426485902..fe7840197 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -151,7 +151,13 @@ endpoints became eligible are upgraded to the kernel datapath automatically. attached to the host end*. * **Death (container stop)** — `_remove_kernel_veths` deletes the host ends explicitly: unlike a TAP (which died with the container netns), a veth host - end outlives the container. Deleting either end destroys the pair. + end outlives the container. Deleting either end destroys the pair. The + node's per-link kernel bridges go with them — deleting a node or closing a + project never runs the link-teardown path (no per-link delete is issued), + so a bridge whose ports just disappeared would stay behind as an empty + orphan. Both endpoints do this and the peer's still-enslaved port makes + the delete fail with EBUSY (suppressed, last one wins), the same contract + as link deletion; a node restart rebuilds the bridge from the NIO. ## Link operations diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 0dd0dadd4..7e8384b51 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1883,12 +1883,25 @@ class DockerVM(BaseNode): with the container network namespace) the host end outlives the container and must be removed explicitly. Deleting an enslaved veth detaches it from its bridge automatically. + + The per-link kernel bridges go too: deleting a node (or closing a + project) never runs the link-teardown path, so a bridge whose ports + just disappeared would stay behind as an empty orphan. Both link + endpoints run this — the peer's port still enslaved makes the delete + fail with EBUSY (suppressed) and the last one wins, the same + contract as _remove_kernel_nio. A node restart re-creates the bridge + from the NIO in _connect_nio, so nothing is lost by deleting it. """ if self.ubridge: for host_ifc in self._kernel_veths.values(): with contextlib.suppress(UbridgeError): await self._ubridge_send(f'docker delete_veth "{host_ifc}"') + for adapter in self._ethernet_adapters: + for nio in adapter.ports.values(): + if isinstance(nio, NIOBridge): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delete "{nio.bridge}"') self._kernel_veths.clear() async def _get_namespace(self): diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 4f3001e33..abb86049b 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -1006,6 +1006,34 @@ async def test_remove_kernel_veths(vm): assert vm._kernel_veths == {} +@pytest.mark.asyncio +async def test_remove_kernel_veths_deletes_per_link_bridge(vm): + """ + The per-link kernel bridge dies with the node too: deleting a project + never runs the link-teardown path, so without this the bridge stays + behind as an empty orphan (EBUSY from the peer's port is suppressed — + the last endpoint wins). + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + # a relay NIO's bridge is not this node's to delete (uBridge dies with it) + relay = vm.manager.create_nio({"type": "nio_udp", "lport": 10000, "rhost": "127.0.0.1", "rport": 10001}) + vm._ethernet_adapters[0].add_nio(1, relay) + + await vm._remove_kernel_veths() + + vm._ubridge_send.assert_any_call(f'docker delete_veth "{host_ifc}"') + # exactly the kernel bridge — the relay NIO's uBridge bridge dies with + # the uBridge process and must not be touched here + deletes = [call.args[0] for call in vm._ubridge_send.call_args_list if "brctl delete" in call.args[0]] + assert deletes == [f'brctl delete "{BRIDGE}"'] + + def test_veth_names_deterministic_and_bounded(vm): host_ifc, guest_ifc = vm._veth_names(3, 2) From b4e4739d52fea65244219f72cabdf1d565439345 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 09:50:47 +0800 Subject: [PATCH 015/101] feat(compute): report uBridge tc capabilities and gate filters on them Consume uBridge's ebpf_modes contract (feature/tc-window 40c36ae): a stateful mode is usable iff ebpf=1 and its token is listed; builds predating the field keep their shipped modes (nth/quota/flow), so the pre-correction window semantics can never be installed silently. - gns3server/utils/tc_capabilities.py: shared reply parser, mode policy and filter-type mappings (compute and controller gate on the same rules) - gns3server/compute/ubridge/tc_probe.py: standalone probe spawning a throwaway uBridge as the server's own user, cached by binary identity (path/mtime/size - the version string cannot distinguish rebuilds); failures cache as unknown - GET /v3/compute/capabilities gains ubridge_tc {netem, ebpf, ebpf_modes, cbpf}; the controller forwards it (Compute.capabilities) and available_filters hides kernel-datapath filter types an endpoint compute cannot run (both ends apply their own filters, so one incapable compute hides the type); computes without a report keep the full list and apply-time 409 stays the guard - docker: _ubridge_apply_ebpf_drops gates per mode token; the cleanup path only sends "off" for modes the build declares - split DockerVM's kernel-datapath machinery (veth lifecycle, carrier, marker primitives, tc filter translation, kernel NIO teardown) into compute/docker/docker_kernel_datapath.py (DockerKernelDatapathMixin) --- docs/design/ubridge-kernel-impairment-spec.md | 20 +- docs/features/docker-kernel-datapath.md | 23 +- gns3server/api/routes/compute/capabilities.py | 17 +- .../compute/docker/docker_kernel_datapath.py | 607 ++++++++++++++++++ gns3server/compute/docker/docker_vm.py | 555 +--------------- gns3server/compute/ubridge/tc_probe.py | 114 ++++ gns3server/controller/compute.py | 19 +- gns3server/controller/link.py | 45 +- gns3server/schemas/controller/capabilities.py | 19 +- gns3server/schemas/controller/computes.py | 4 + gns3server/utils/tc_capabilities.py | 85 +++ pyproject.toml | 1 + tests/api/routes/compute/test_capabilities.py | 29 +- .../docker/test_docker_kernel_datapath.py | 77 ++- tests/compute/ubridge/test_tc_probe.py | 115 ++++ tests/controller/test_compute.py | 2 +- tests/controller/test_kernel_datapath_link.py | 42 ++ tests/utils/test_tc_capabilities.py | 77 +++ 18 files changed, 1280 insertions(+), 571 deletions(-) create mode 100644 gns3server/compute/docker/docker_kernel_datapath.py create mode 100644 gns3server/compute/ubridge/tc_probe.py create mode 100644 gns3server/utils/tc_capabilities.py create mode 100644 tests/compute/ubridge/test_tc_probe.py create mode 100644 tests/utils/test_tc_capabilities.py diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md index 454879a45..2cfa565c4 100644 --- a/docs/design/ubridge-kernel-impairment-spec.md +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -293,13 +293,23 @@ old uBridge without these commands simply stays on the relay datapath): ``` tc capabilities -100-netem=delay,jitter,loss,dup,corrupt,rate,reorder,gemodel,dist,seed,limit;ebpf=1;cbpf=1 +100-netem=delay,jitter,loss,dup,corrupt,rate,reorder,gemodel,dist,seed,limit;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow ``` `ebpf=0` when the load probe fails at startup (kernel < 5.1 or no CAP_BPF); `cbpf=0` when RTM_NEWTFILTER/cls_bpf is unavailable. gns3-server hides the corresponding filter types per capability. +`ebpf_modes` (added on uBridge `feature/tc-window`, commit 40c36ae) is a +build fact, orthogonal to the runtime probes: the same binary flips `ebpf` +between users and kernels while the mode set changes only with the binary. +A mode is usable iff `ebpf=1` **and** its token is listed — gating +`window_drop` on `ebpf=1` alone silently accepts the 4-arg form on a +pre-correction build (`feature/tc-precision`: back-to-back windows, no +gap). Builds that do not emit the field at all predate it and get legacy +treatment; an incompatible mode revision renames its token +(`window` → `window2`) rather than versioning in place. + --- ## F. Test requirements @@ -338,6 +348,14 @@ corresponding filter types per capability. mirrors the grammar's `period ≥ outage` rule and treats `start = 0` as active (an immediate outage). Not yet exposed: `flow_drop` (parameter-shape UX decision). +* **Done** (capability reporting): the `ebpf_modes` contract is consumed + end to end — the compute applies it per mode (window requires its token; + pre-field builds keep their shipped modes), the compute + `GET /capabilities` payload reports `ubridge_tc` through a standalone + spawn-as-own-user probe cached by binary identity, and the controller's + `available_filters` hides kernel-datapath filter types an endpoint + compute reports it cannot run (computes without a report keep the full + list; apply-time 409 stays the guard). * Every veth end owns one qdisc + its filters: per-direction impairment is an architectural freebie to expose later (API `direction` field), aligned with marker `dir` semantics. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index fe7840197..d560bee31 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -293,6 +293,27 @@ Semantics: frames (on the relay, filter-vs-mark ordering determined visibility instead). Also note `tc qdisc show` cannot print a distribution's name (the kernel stores only the sampled table). +* **eBPF mode tokens.** A stateful mode is usable iff `tc capabilities` + reports `ebpf=1` **and** lists the mode's token in `ebpf_modes` + (uBridge `feature/tc-window`+). Builds predating the field keep the modes + that shipped with it — everything but `window`, whose semantics were + corrected exactly when the field was added: gating on `ebpf=1` alone + would silently install the broken back-to-back-window behaviour on a + pre-correction build. Requesting an unusable mode fails the apply with an + upgrade error; the cleanup ("off") path only touches modes the build + declares. +* **Capability reporting.** The compute exposes the probe through + `GET /v3/compute/capabilities` as `ubridge_tc` (netem keyword list, + `ebpf`/`cbpf` flags, and the usable `ebpf_modes` — legacy fallback + applied), answered by a throwaway uBridge spawned as the server's own + user and cached by binary identity — the runtime `ebpf` flag flips + between users and kernels for the same binary, so a root-run probe would + lie for a non-root server. The controller forwards it through + `GET /v3/computes` and `available_filters` hides kernel-datapath filter + types an endpoint compute reports it cannot run (both ends apply their + own filters, so one incapable compute is enough). Computes that report + nothing — older servers, failed probe — keep the full list; + apply-time validation (409) stays the guard. ## uBridge command surface @@ -313,7 +334,7 @@ tc nth_drop # eBPF exact every-Nth (frequency_drop) tc quota_drop | off # eBPF byte cap (quota) tc window_drop [ []] | off # eBPF time window (window_drop) tc reset # full restore: eBPF filter -> bpf_drops -> clsact -> root qdisc, idempotent -tc capabilities # "netem=;ebpf=0|1;cbpf=0|1", probed once per process +tc capabilities # "netem=;ebpf=0|1;cbpf=0|1[;ebpf_modes=]", probed once per process bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capture ``` diff --git a/gns3server/api/routes/compute/capabilities.py b/gns3server/api/routes/compute/capabilities.py index 363b2906b..055d0fb05 100644 --- a/gns3server/api/routes/compute/capabilities.py +++ b/gns3server/api/routes/compute/capabilities.py @@ -25,19 +25,33 @@ from fastapi import APIRouter, Request from gns3server import schemas from gns3server.compute import MODULES +from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities from gns3server.utils.path import get_default_project_directory +from gns3server.utils.tc_capabilities import usable_ebpf_modes from gns3server.version import __version__ router = APIRouter() @router.get("/capabilities", response_model=schemas.Capabilities) -def get_capabilities(request: Request) -> dict: +async def get_capabilities(request: Request) -> dict: node_types = [] for module in MODULES: node_types.extend(module.node_types()) + # Kernel-datapath packet filters: what this host's uBridge can run, + # probed as the server's own user (None = unknown, old/missing uBridge). + ubridge_tc = None + caps = await probe_tc_capabilities() + if caps: + ubridge_tc = { + "netem": [keyword for keyword in caps.get("netem", "").split(",") if keyword], + "ebpf": caps.get("ebpf") == "1", + "ebpf_modes": list(usable_ebpf_modes(caps)), + "cbpf": caps.get("cbpf") == "1", + } + # record the controller hostname or IP address if request.client: request.app.state.controller_host = request.client.host @@ -49,4 +63,5 @@ def get_capabilities(request: Request) -> dict: "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, "node_types": node_types, + "ubridge_tc": ubridge_tc, } diff --git a/gns3server/compute/docker/docker_kernel_datapath.py b/gns3server/compute/docker/docker_kernel_datapath.py new file mode 100644 index 000000000..a47ada1cd --- /dev/null +++ b/gns3server/compute/docker/docker_kernel_datapath.py @@ -0,0 +1,607 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Kernel-datapath machinery for DockerVM, extracted from docker_vm.py: the +veth pair lifecycle (deterministic names, creation, teardown together with +the per-link kernel bridges), the adapter carrier control, the kernel +marker primitives and the tc packet-filter translation (netem, classic-BPF +match-drop, eBPF stateful drops). Every method runs with a DockerVM +instance as ``self`` and reads the same state (_kernel_veths, +_ubridge_tc_caps, the uBridge hypervisor through the BaseNode helpers). +""" + +import os +import contextlib + +from gns3server.utils.tc_capabilities import FILTER_EBPF_MODES, parse_tc_capabilities, usable_ebpf_modes +from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError + +from ..nios.nio_bridge import NIOBridge +from .docker_error import DockerError + +import logging + +log = logging.getLogger(__name__) + + +class DockerKernelDatapathMixin: + """ + The kernel-datapath half of a Docker node — consumed by DockerVM so + docker_vm.py stays focused on the container lifecycle. + """ + + def _veth_names(self, adapter_number, port_number=0): + """ + Deterministic veth pair names for a kernel-datapath adapter port. + The host end stays in the root namespace (enslaved to the per-link + kernel bridge); the guest end is moved into the container namespace + and renamed. The ``gv``/``gc`` prefixes keep these out of the ``gns3`` + bridge/TAP name space and serve as search keys for stale-interface + cleanup. 8 hex chars of the node id + adapter/port keep the names + unique and within IFNAMSIZ (15). + """ + + suffix = f"{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" + return f"gv{suffix}", f"gc{suffix}" + + async def _create_veth(self, adapter_number, port_number=0): + """ + Create the adapter's unified veth pair: the guest end moves into the + container namespace as eth{N} (the role the TAP used to play on the + relay path), the host end stays in the root namespace with carrier + off until a link attaches. Both datapaths anchor here — kernel links + enslave the host end into a Linux bridge; relay links attach it to + the uBridge relay via AF_PACKET. + """ + + try: + adapter = self._ethernet_adapters[adapter_number] + except IndexError: + raise DockerError( + "Adapter {adapter_number} doesn't exist on Docker container '{name}'".format( + name=self.name, adapter_number=adapter_number + ) + ) + + host_ifc, guest_ifc = self._veth_names(adapter_number, port_number) + # The host end lives in the root namespace and survives container + # death: best-effort removal of a stale pair from a previous run + # (unclean stop or crash) before recreating it. + for stale_ifc in (host_ifc, guest_ifc): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{stale_ifc}"') + + mac_address = self._adapter_mac_address(adapter_number) + try: + await self._ubridge_send(f'docker create_veth "{host_ifc}" "{guest_ifc}"') + # create_veth leaves the host end UP; carrier must be off until + # a link attaches (mirrors the old "bridge add_nio_tap ... off"). + await self._ubridge_send(f'link set "{host_ifc}" down') + try: + await self._ubridge_send(f"docker set_mac_addr {guest_ifc} {mac_address}") + except UbridgeError: + log.warning(f"Could not set MAC address {mac_address} on interface {guest_ifc}") + + ifname = self._get_container_ifname(adapter_number) + log.debug(f"Move container {self.name} adapter {guest_ifc} -> {ifname} in ns {self._namespace}") + try: + await self._ubridge_send(f"docker move_to_ns {guest_ifc} {self._namespace} {ifname}") + except UbridgeError as e: + raise UbridgeNamespaceError(e) + except UbridgeNamespaceError: + raise + except Exception: + # Don't leave a half-created pair behind (the next start would + # fail on create_veth EEXIST until the stale sweep runs). + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{host_ifc}"') + raise + + adapter.host_ifc = host_ifc + self._kernel_veths[(adapter_number, port_number)] = host_ifc + log.debug( + "Created veth adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( + adapter_number=adapter_number, + port_number=port_number, + mac_address=mac_address, + namespace=self._namespace, + ) + ) + + async def _remove_kernel_veths(self): + """ + Delete the root-namespace veth host ends. Unlike relay TAPs (which die + with the container network namespace) the host end outlives the + container and must be removed explicitly. Deleting an enslaved veth + detaches it from its bridge automatically. + + The per-link kernel bridges go too: deleting a node (or closing a + project) never runs the link-teardown path, so a bridge whose ports + just disappeared would stay behind as an empty orphan. Both link + endpoints run this — the peer's port still enslaved makes the delete + fail with EBUSY (suppressed) and the last one wins, the same + contract as _remove_kernel_nio. A node restart re-creates the bridge + from the NIO in _connect_nio, so nothing is lost by deleting it. + """ + + if self.ubridge: + for host_ifc in self._kernel_veths.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{host_ifc}"') + for adapter in self._ethernet_adapters: + for nio in adapter.ports.values(): + if isinstance(nio, NIOBridge): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delete "{nio.bridge}"') + self._kernel_veths.clear() + + async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): + """Replicate a Docker adapter's connection state on its host interface. + + Kernel-datapath adapters toggle the veth host end's admin state (the + container end loses carrier when its peer is down); relay adapters + toggle the TAP carrier from the uBridge fd holder. + """ + + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if host_ifc is not None: + state = "up" if connected else "down" + await self._ubridge_send(f'link set "{host_ifc}" {state}') + return + + bridge_name = self._bridge_name(adapter_number, port_number) + state = "on" if connected else "off" + await self._ubridge_send(f"bridge set_nio_tap_carrier {bridge_name} {state}") + + def _kernel_marker_anchor(self, anchor): + """ + Whether *anchor* names one of this container's kernel-datapath veth + host ends. Marker anchors are polymorphic: a relay bridge name on the + relay datapath, the veth host interface on the kernel datapath. A veth + name can never collide with a uBridge bridge name (relay bridges are + ``bridge{adapter}``, veths are ``gv…e…p…``). + """ + + return anchor in self._kernel_veths.values() + + async def _ubridge_add_marker_filter( + self, bridge_name, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None + ): + """ + Kernel-datapath variant of the `mark` filter attach: ``marker + add_kernel`` binds an AF_PACKET socket to the veth host end. Keyword + pairs and semantics (match → MARK signal + pcap append) are identical + to the relay ``bridge add_packet_filter … mark`` command. + """ + + if self._kernel_marker_anchor(bridge_name): + self._validate_marker_name(name) + cmd = 'marker add_kernel {name} {ifc} "{bpf}"'.format(name=name, ifc=bridge_name, bpf=bpf) + if tag is not None: + cmd += f" tag {tag}" + if link_id: + cmd += f" link {link_id}" + if direction is not None: + cmd += f" dir {direction}" + linktype = self._marker_linktype(data_link_type) + if linktype is not None: + cmd += f" linktype {linktype}" + cmd += ' pcap "{path}"'.format(path=pcap_path) + await self._ubridge_send(cmd) + return + await super()._ubridge_add_marker_filter( + bridge_name, + name, + bpf, + pcap_path, + tag=tag, + link_id=link_id, + direction=direction, + data_link_type=data_link_type, + ) + + async def _ubridge_delete_marker_filter(self, bridge_name, name): + """ + Kernel-datapath variant of the marker removal: ``marker delete_kernel`` + (idempotent in uBridge — a no-op when the marker is already gone). + """ + + if self._kernel_marker_anchor(bridge_name): + if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): + return + try: + await self._ubridge_send(f"marker delete_kernel {bridge_name} {name}") + except UbridgeError as e: + log.warning("Could not remove kernel marker '%s' from %s: %s", name, bridge_name, e) + return + await super()._ubridge_delete_marker_filter(bridge_name, name) + + async def _ubridge_enable_marker_filter(self, anchor, name, state): + """ + Kernel-datapath variant of the marker on/off toggle: ``marker + enable_kernel`` — installed but silent when off, pcap preserved. + """ + + if self._kernel_marker_anchor(anchor): + await self._ubridge_send(f"marker enable_kernel {anchor} {name} {state}") + return + await super()._ubridge_enable_marker_filter(anchor, name, state) + + @staticmethod + def _netem_command_parts(filters): + """ + Build the ``tc netem set`` keyword sequence (in the frozen grammar + order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, + limit, distribution, seed) plus the set of capability tokens the + sequence needs beyond the original netem surface. Original-surface + keywords (delay, jitter, loss, corrupt, dup) need no probe; the + extension tokens (rate, reorder, gemodel, dist, seed, limit) are + checked against ``tc capabilities`` by the caller. ``correl`` has no + token of its own in the capabilities list, so it gates on "rate" as + the netem-extension build marker. + + :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) + + :returns: (keyword sequence, required extension tokens) + """ + + def values_of(key): + # tolerate the legacy bare-value shape ({"packet_loss": 10}) + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + parts = [] + ext_tokens = set() + + delay = values_of("delay") + if delay: + parts.append(f"delay {int(delay[0])}") + if len(delay) > 1 and int(delay[1]): + parts.append(f"jitter {int(delay[1])}") + + gemodel = values_of("gemodel") + if gemodel: + segment = f"loss gemodel {int(gemodel[0])}" + if len(gemodel) > 1: + segment += f" {int(gemodel[1])}" + if len(gemodel) > 2: + segment += f" {int(gemodel[2])}" + parts.append(segment) + ext_tokens.add("gemodel") + else: + loss = values_of("packet_loss") + if loss and int(loss[0]): + parts.append(f"loss {int(loss[0])}") + if len(loss) > 1 and int(loss[1]): + parts.append(f"correl {int(loss[1])}") + ext_tokens.add("rate") # correl: extension build marker + + duplicate = values_of("duplicate") + if duplicate and int(duplicate[0]): + parts.append(f"dup {int(duplicate[0])}") + if len(duplicate) > 1 and int(duplicate[1]): + parts.append(f"correl {int(duplicate[1])}") + ext_tokens.add("rate") # correl: extension build marker + + corrupt = values_of("corrupt") + if corrupt and int(corrupt[0]): + parts.append(f"corrupt {int(corrupt[0])}") + + reorder = values_of("reorder") + if reorder: + segment = f"reorder {int(reorder[0])}" + if len(reorder) > 1 and int(reorder[1]): + segment += f" correl {int(reorder[1])}" + if len(reorder) > 2 and int(reorder[2]): + segment += f" gap {int(reorder[2])}" + parts.append(segment) + ext_tokens.add("reorder") + + rate = values_of("rate") + if rate and str(rate[0]).strip(): + parts.append(f"rate {str(rate[0]).strip()}") + ext_tokens.add("rate") + + limit = values_of("limit") + if limit and int(limit[0]): + parts.append(f"limit {int(limit[0])}") + ext_tokens.add("limit") + + if delay and len(delay) > 2 and str(delay[2]).strip().lower() not in ("", "uniform"): + # "uniform" is the kernel default — emitting nothing keeps the + # command compatible with the original netem surface. + parts.append(f"distribution {str(delay[2]).strip().lower()}") + ext_tokens.add("dist") + + seed = values_of("seed") + if seed and int(seed[0]): + # 0 is the "disabled" convention everywhere else — treat it so + # here too (the controller's inactive-filter pass drops it). + parts.append(f"seed {int(seed[0])}") + ext_tokens.add("seed") + + return parts, ext_tokens + + async def _ubridge_apply_netem(self, host_ifc, filters): + """ + Translate the NIO's impairment filters into one tc netem qdisc on the + veth host end. The netem-expressible types (delay, packet_loss, + corrupt and the netem extensions rate/reorder/gemodel/duplicate/ + seed/limit) merge into a single qdisc — its egress covers the traffic + entering this container, and with both link endpoints applying + theirs, every direction of the link is impaired exactly once (the + same net effect as the relay, where both directions cross the single + filtered bridge). + + The kernel's netem replace MERGES optional attributes (rate, + correlation, reorder, corrupt, gemodel, distribution: absent attr = + previous value kept), so re-applying a filter set with a parameter + *removed* would silently keep the old value. A ``tc reset`` before + every ``netem set`` makes the re-apply a true reconcile (reset also + drops clsact and its bpf_drop filters — the caller re-applies them + right after in the same flow). An empty filter set is the reset + alone. + + :param host_ifc: veth host-end interface name + :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) + """ + + parts, ext_tokens = self._netem_command_parts(filters) + if parts and ext_tokens: + # Extension keywords need the netem-extension uBridge. Plain + # delay/loss/corrupt/dup never probes — an old uBridge serves + # the original surface untouched (mirrors bpf_drop's guard). + # Checked before the reset so a failed update leaves the + # previously applied qdisc in place. + caps = await self._ubridge_tc_capabilities() + missing = ext_tokens - set(caps.get("netem", "").split(",")) + if missing: + raise DockerError( + "Packet filter(s) using {} need a uBridge with the netem extensions " + "(tc capabilities reports '{}'); upgrade uBridge on this compute".format( + ", ".join(sorted(missing)), caps.get("netem") or "no tc support" + ) + ) + try: + await self._ubridge_send(f'tc reset "{host_ifc}"') + except UbridgeError as e: + # "No such file or directory" = no qdisc was attached: already + # clean. Everything else is a real failure and must surface. + if "No such file" not in str(e): + raise + if not parts: + return + await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) + + async def _ubridge_tc_capabilities(self): + """ + uBridge's tc-module feature report (``tc capabilities``), probed once + per uBridge process and cached — the reply shape is + ``netem=;ebpf=0|1;cbpf=0|1[;ebpf_modes=]`` (parsed + by parse_tc_capabilities). uBridge builds without the tc module + answer "Unknown module" and old builds without the command answer + "Unknown command": both mean no kernel filter support, cached as an + empty dict. + """ + + if self._ubridge_tc_caps is None: + caps = {} + if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): + try: + reply = await self._ubridge_hypervisor.send("tc capabilities") + except UbridgeError: + reply = [] + caps = parse_tc_capabilities(reply[0] if reply else "") + self._ubridge_tc_caps = caps + return self._ubridge_tc_caps + + async def _ubridge_apply_bpf_drops(self, host_ifc, filters): + """ + Translate the NIO's ``bpf`` filter (one expression per line, any line + matching drops the packet) into cls_bpf classifiers on the veth host + end's clsact egress: ``tc bpf_drop`` compiles each line with libpcap + (same compiler as the relay's bpf filter) and drops matches with a + gact TC_ACT_SHOT. Reconcile is always flush + re-add — same-prio + resend replaces the filter node in uBridge, and the flush drops + lines the user removed. + + :param host_ifc: veth host-end interface name + :param filters: NIO filters dictionary + """ + + bpf = filters.get("bpf") + lines = [line.strip() for line in (bpf[0].split("\n") if bpf else []) if line.strip()] + if not lines and self._ubridge_tc_caps is None: + # Nothing to add and nothing was ever installed (no tc module + # probed yet) — skip touching an old uBridge entirely. + return + caps = await self._ubridge_tc_capabilities() + if not lines: + if caps: + await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') + return + if caps.get("cbpf") != "1": + raise DockerError( + "Packet filter 'bpf' on a kernel-datapath link needs a uBridge with cBPF support " + "(tc capabilities reports no cbpf); upgrade uBridge on this compute or keep the " + "link on the relay datapath" + ) + await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') + for offset, line in enumerate(lines): + try: + await self._ubridge_send( + 'tc bpf_drop add "{ifc}" {prio} "{expr}"'.format(ifc=host_ifc, prio=10 + offset, expr=line) + ) + except UbridgeError as e: + if "Cannot compile filter" not in str(e): + raise + # Mirror the relay path: a line that no longer compiles on + # this uBridge's libpcap is skipped with a warning instead + # of breaking the link (the controller already validated + # the syntax with tcpdump at create/update time). + message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {line}" + log.warning(message) + self.project.emit("log.warning", {"message": message}) + + async def _ubridge_apply_ebpf_drops(self, host_ifc, filters): + """ + Translate the NIO's stateful filters into uBridge's eBPF impairment + classifier on the veth host end's clsact egress (prio 1, below + bpf_drop's 10-99): ``frequency_drop`` becomes the exact every-Nth + mode (``tc nth_drop``; -1 = drop everything maps to every 1st), the + kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``), + and the kernel-only ``window_drop`` type the time-window mode + (``tc window_drop``: single outage, or recurring flaps with an + optional per-cycle jitter). The program evaluates the modes in the + fixed order nth → quota → window. A mode is only sent when + ``tc capabilities`` declares its token (``ebpf_modes``); builds + predating the field keep their shipped modes (everything but + window). + + Reconcile is a plain re-set on every apply: the preceding netem + apply resets the interface (which detaches the program and drops + uBridge's per-interface registry entry), so a mode set re-loads the + program from scratch, and a mode left absent is turned off + explicitly (idempotent) so a removed filter stops dropping. For + window_drop this also means the schedule restarts on every apply: + the first window opens Start-ms from the moment of this call. + + Semantic difference vs the relay, documented server-side: the relay + counts packets of BOTH directions through its single filtered + bridge, while each veth end's classifier counts only the traffic + entering that container — with both endpoints applying theirs, each + direction drops every Nth independently (their window schedules + start ~simultaneously, so a round trip survives only when both + directions are outside their windows). + + :param host_ifc: veth host-end interface name + :param filters: NIO filters dictionary + """ + + def values_of(key): + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + frequency = values_of("frequency_drop") + quota = values_of("quota") + window = values_of("window_drop") + requested = {filter_type: values_of(filter_type) for filter_type in FILTER_EBPF_MODES} + if not any(requested.values()) and self._ubridge_tc_caps is None: + # Nothing to set and nothing was ever installed (no probe yet) — + # skip touching an old uBridge entirely. + return + caps = await self._ubridge_tc_capabilities() + # A mode is usable iff ebpf=1 and the build declares its token + # (ebpf_modes); builds predating the field keep their shipped modes. + modes = usable_ebpf_modes(caps) + if not any(requested.values()): + # Only modes this build declares can ever have been installed — + # sending "off" for an undeclared mode would hit an unknown + # command; "off" is idempotent in uBridge. + for filter_type, mode in FILTER_EBPF_MODES.items(): + if mode in modes: + await self._ubridge_send(f'tc {mode}_drop "{host_ifc}" off') + return + unsupported = sorted(f for f, values in requested.items() if values and FILTER_EBPF_MODES[f] not in modes) + if unsupported: + if not modes: + raise DockerError( + "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " + "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " + "the uBridge binary); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(unsupported)) + ) + raise DockerError( + "Packet filter(s) {} on a kernel-datapath link need a newer uBridge (tc capabilities " + "reports ebpf_modes={}); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(unsupported), ",".join(modes)) + ) + if frequency: + # relay semantics: -1 = drop everything, N = every Nth packet + nth = 1 if int(frequency[0]) == -1 else int(frequency[0]) + await self._ubridge_send(f'tc nth_drop "{host_ifc}" {nth}') + elif "nth" in modes: + await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') + if quota: + await self._ubridge_send( + 'tc quota_drop "{ifc}" {bytes} {pct}'.format(ifc=host_ifc, bytes=int(quota[0]), pct=int(quota[1])) + ) + elif "quota" in modes: + await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') + if window: + # [start, outage, chance, period?, jitter?] — start is relative + # to this apply (the schedule restarts on every reconcile). + params = " ".join(str(int(v)) for v in window) + await self._ubridge_send(f'tc window_drop "{host_ifc}" {params}') + elif "window" in modes: + await self._ubridge_send(f'tc window_drop "{host_ifc}" off') + + async def _remove_kernel_markers(self, host_ifc): + """ + Tear down every kernel marker anchored to *host_ifc*: the veth survives + link deletion (it is the adapter interface, not the link), so the + AF_PACKET sockets must be closed explicitly — otherwise a deleted + link's markers would keep sniffing and signaling. Mirrors the relay + datapath where deleting the uBridge bridge silently drops its filters. + """ + + from gns3server.compute.marker.marker_manager import MarkerManager + + manager = MarkerManager.instance() + markers_dir = self.project.markers_working_directory() + for (name, link_id), anchor in list(self._marker_filter_bridges.items()): + if anchor != host_ifc: + continue + self._marker_filter_bridges.pop((name, link_id)) + self._marker_specs.pop((name, link_id), None) + await self._ubridge_delete_marker_filter(host_ifc, name) + try: + os.remove(os.path.join(markers_dir, f"{self._id}_{link_id}_{name}.pcap")) + except FileNotFoundError: + pass + except OSError as e: + log.warning("Could not remove marker pcap for '%s' on link %s: %s", name, link_id, e) + manager.unregister(self._id, name) + + async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): + """ + Detach a kernel-datapath adapter port from its per-link kernel bridge. + Both endpoints run this concurrently: deleting a bridge that still has + the peer's port enslaved fails with EBUSY, so the last endpoint to + remove its port wins the deletion and the loser's failure is expected + and suppressed. + """ + + host_ifc = self._kernel_veths.get((adapter_number, port_number)) + if host_ifc is not None: + await self._remove_kernel_markers(host_ifc) + # The veth survives link deletion, so an orphaned netem qdisc + # would keep impairing whatever attaches to the adapter next + # (kernel link or relay). Best-effort: ENOENT just means no + # qdisc was attached. + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tc reset "{host_ifc}"') + if self.status == "started": + await self._set_adapter_carrier(adapter_number, False, port_number) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delete "{nio.bridge}"') diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 7e8384b51..98feb66c1 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -48,11 +48,12 @@ from ..error import ImageMissingError from ..nios.nio_bridge import NIOBridge from ..nios.nio_udp import NIOUDP from .docker_error import DockerError, DockerHttp404Error, DockerHttp409Error +from .docker_kernel_datapath import DockerKernelDatapathMixin log = logging.getLogger(__name__) -class DockerVM(BaseNode): +class DockerVM(DockerKernelDatapathMixin, BaseNode): """ Docker container implementation. @@ -1799,134 +1800,11 @@ class DockerVM(BaseNode): mac_address = custom_mac_address return mac_address - def _veth_names(self, adapter_number, port_number=0): - """ - Deterministic veth pair names for a kernel-datapath adapter port. - The host end stays in the root namespace (enslaved to the per-link - kernel bridge); the guest end is moved into the container namespace - and renamed. The ``gv``/``gc`` prefixes keep these out of the ``gns3`` - bridge/TAP name space and serve as search keys for stale-interface - cleanup. 8 hex chars of the node id + adapter/port keep the names - unique and within IFNAMSIZ (15). - """ - - suffix = f"{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" - return f"gv{suffix}", f"gc{suffix}" - - async def _create_veth(self, adapter_number, port_number=0): - """ - Create the adapter's unified veth pair: the guest end moves into the - container namespace as eth{N} (the role the TAP used to play on the - relay path), the host end stays in the root namespace with carrier - off until a link attaches. Both datapaths anchor here — kernel links - enslave the host end into a Linux bridge; relay links attach it to - the uBridge relay via AF_PACKET. - """ - - try: - adapter = self._ethernet_adapters[adapter_number] - except IndexError: - raise DockerError( - "Adapter {adapter_number} doesn't exist on Docker container '{name}'".format( - name=self.name, adapter_number=adapter_number - ) - ) - - host_ifc, guest_ifc = self._veth_names(adapter_number, port_number) - # The host end lives in the root namespace and survives container - # death: best-effort removal of a stale pair from a previous run - # (unclean stop or crash) before recreating it. - for stale_ifc in (host_ifc, guest_ifc): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'docker delete_veth "{stale_ifc}"') - - mac_address = self._adapter_mac_address(adapter_number) - try: - await self._ubridge_send(f'docker create_veth "{host_ifc}" "{guest_ifc}"') - # create_veth leaves the host end UP; carrier must be off until - # a link attaches (mirrors the old "bridge add_nio_tap ... off"). - await self._ubridge_send(f'link set "{host_ifc}" down') - try: - await self._ubridge_send(f"docker set_mac_addr {guest_ifc} {mac_address}") - except UbridgeError: - log.warning(f"Could not set MAC address {mac_address} on interface {guest_ifc}") - - ifname = self._get_container_ifname(adapter_number) - log.debug(f"Move container {self.name} adapter {guest_ifc} -> {ifname} in ns {self._namespace}") - try: - await self._ubridge_send(f"docker move_to_ns {guest_ifc} {self._namespace} {ifname}") - except UbridgeError as e: - raise UbridgeNamespaceError(e) - except UbridgeNamespaceError: - raise - except Exception: - # Don't leave a half-created pair behind (the next start would - # fail on create_veth EEXIST until the stale sweep runs). - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'docker delete_veth "{host_ifc}"') - raise - - adapter.host_ifc = host_ifc - self._kernel_veths[(adapter_number, port_number)] = host_ifc - log.debug( - "Created veth adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( - adapter_number=adapter_number, - port_number=port_number, - mac_address=mac_address, - namespace=self._namespace, - ) - ) - - async def _remove_kernel_veths(self): - """ - Delete the root-namespace veth host ends. Unlike relay TAPs (which die - with the container network namespace) the host end outlives the - container and must be removed explicitly. Deleting an enslaved veth - detaches it from its bridge automatically. - - The per-link kernel bridges go too: deleting a node (or closing a - project) never runs the link-teardown path, so a bridge whose ports - just disappeared would stay behind as an empty orphan. Both link - endpoints run this — the peer's port still enslaved makes the delete - fail with EBUSY (suppressed) and the last one wins, the same - contract as _remove_kernel_nio. A node restart re-creates the bridge - from the NIO in _connect_nio, so nothing is lost by deleting it. - """ - - if self.ubridge: - for host_ifc in self._kernel_veths.values(): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'docker delete_veth "{host_ifc}"') - for adapter in self._ethernet_adapters: - for nio in adapter.ports.values(): - if isinstance(nio, NIOBridge): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delete "{nio.bridge}"') - self._kernel_veths.clear() - async def _get_namespace(self): result = await self.manager.query("GET", f"containers/{self._cid}/json") return int(result["State"]["Pid"]) - async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): - """Replicate a Docker adapter's connection state on its host interface. - - Kernel-datapath adapters toggle the veth host end's admin state (the - container end loses carrier when its peer is down); relay adapters - toggle the TAP carrier from the uBridge fd holder. - """ - - host_ifc = self._kernel_veths.get((adapter_number, port_number)) - if host_ifc is not None: - state = "up" if connected else "down" - await self._ubridge_send(f'link set "{host_ifc}" {state}') - return - - bridge_name = self._bridge_name(adapter_number, port_number) - state = "on" if connected else "off" - await self._ubridge_send(f"bridge set_nio_tap_carrier {bridge_name} {state}") - async def _connect_nio(self, adapter_number, nio, port_number=0): host_ifc = self._kernel_veths.get((adapter_number, port_number)) @@ -2094,435 +1972,6 @@ class DockerVM(BaseNode): log.debug(f"Docker VM '{self.name}' [{self.id}]: {adapter.host_ifc} removed from adapter {adapter_number}") - def _kernel_marker_anchor(self, anchor): - """ - Whether *anchor* names one of this container's kernel-datapath veth - host ends. Marker anchors are polymorphic: a relay bridge name on the - relay datapath, the veth host interface on the kernel datapath. A veth - name can never collide with a uBridge bridge name (relay bridges are - ``bridge{adapter}``, veths are ``gv…e…p…``). - """ - - return anchor in self._kernel_veths.values() - - async def _ubridge_add_marker_filter( - self, bridge_name, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None - ): - """ - Kernel-datapath variant of the `mark` filter attach: ``marker - add_kernel`` binds an AF_PACKET socket to the veth host end. Keyword - pairs and semantics (match → MARK signal + pcap append) are identical - to the relay ``bridge add_packet_filter … mark`` command. - """ - - if self._kernel_marker_anchor(bridge_name): - self._validate_marker_name(name) - cmd = 'marker add_kernel {name} {ifc} "{bpf}"'.format(name=name, ifc=bridge_name, bpf=bpf) - if tag is not None: - cmd += f" tag {tag}" - if link_id: - cmd += f" link {link_id}" - if direction is not None: - cmd += f" dir {direction}" - linktype = self._marker_linktype(data_link_type) - if linktype is not None: - cmd += f" linktype {linktype}" - cmd += ' pcap "{path}"'.format(path=pcap_path) - await self._ubridge_send(cmd) - return - await super()._ubridge_add_marker_filter( - bridge_name, - name, - bpf, - pcap_path, - tag=tag, - link_id=link_id, - direction=direction, - data_link_type=data_link_type, - ) - - async def _ubridge_delete_marker_filter(self, bridge_name, name): - """ - Kernel-datapath variant of the marker removal: ``marker delete_kernel`` - (idempotent in uBridge — a no-op when the marker is already gone). - """ - - if self._kernel_marker_anchor(bridge_name): - if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): - return - try: - await self._ubridge_send(f"marker delete_kernel {bridge_name} {name}") - except UbridgeError as e: - log.warning("Could not remove kernel marker '%s' from %s: %s", name, bridge_name, e) - return - await super()._ubridge_delete_marker_filter(bridge_name, name) - - async def _ubridge_enable_marker_filter(self, anchor, name, state): - """ - Kernel-datapath variant of the marker on/off toggle: ``marker - enable_kernel`` — installed but silent when off, pcap preserved. - """ - - if self._kernel_marker_anchor(anchor): - await self._ubridge_send(f"marker enable_kernel {anchor} {name} {state}") - return - await super()._ubridge_enable_marker_filter(anchor, name, state) - - @staticmethod - def _netem_command_parts(filters): - """ - Build the ``tc netem set`` keyword sequence (in the frozen grammar - order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, - limit, distribution, seed) plus the set of capability tokens the - sequence needs beyond the original netem surface. Original-surface - keywords (delay, jitter, loss, corrupt, dup) need no probe; the - extension tokens (rate, reorder, gemodel, dist, seed, limit) are - checked against ``tc capabilities`` by the caller. ``correl`` has no - token of its own in the capabilities list, so it gates on "rate" as - the netem-extension build marker. - - :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) - - :returns: (keyword sequence, required extension tokens) - """ - - def values_of(key): - # tolerate the legacy bare-value shape ({"packet_loss": 10}) - values = filters.get(key) - if isinstance(values, (list, tuple)): - return list(values) - return [values] if values else [] - - parts = [] - ext_tokens = set() - - delay = values_of("delay") - if delay: - parts.append(f"delay {int(delay[0])}") - if len(delay) > 1 and int(delay[1]): - parts.append(f"jitter {int(delay[1])}") - - gemodel = values_of("gemodel") - if gemodel: - segment = f"loss gemodel {int(gemodel[0])}" - if len(gemodel) > 1: - segment += f" {int(gemodel[1])}" - if len(gemodel) > 2: - segment += f" {int(gemodel[2])}" - parts.append(segment) - ext_tokens.add("gemodel") - else: - loss = values_of("packet_loss") - if loss and int(loss[0]): - parts.append(f"loss {int(loss[0])}") - if len(loss) > 1 and int(loss[1]): - parts.append(f"correl {int(loss[1])}") - ext_tokens.add("rate") # correl: extension build marker - - duplicate = values_of("duplicate") - if duplicate and int(duplicate[0]): - parts.append(f"dup {int(duplicate[0])}") - if len(duplicate) > 1 and int(duplicate[1]): - parts.append(f"correl {int(duplicate[1])}") - ext_tokens.add("rate") # correl: extension build marker - - corrupt = values_of("corrupt") - if corrupt and int(corrupt[0]): - parts.append(f"corrupt {int(corrupt[0])}") - - reorder = values_of("reorder") - if reorder: - segment = f"reorder {int(reorder[0])}" - if len(reorder) > 1 and int(reorder[1]): - segment += f" correl {int(reorder[1])}" - if len(reorder) > 2 and int(reorder[2]): - segment += f" gap {int(reorder[2])}" - parts.append(segment) - ext_tokens.add("reorder") - - rate = values_of("rate") - if rate and str(rate[0]).strip(): - parts.append(f"rate {str(rate[0]).strip()}") - ext_tokens.add("rate") - - limit = values_of("limit") - if limit and int(limit[0]): - parts.append(f"limit {int(limit[0])}") - ext_tokens.add("limit") - - if delay and len(delay) > 2 and str(delay[2]).strip().lower() not in ("", "uniform"): - # "uniform" is the kernel default — emitting nothing keeps the - # command compatible with the original netem surface. - parts.append(f"distribution {str(delay[2]).strip().lower()}") - ext_tokens.add("dist") - - seed = values_of("seed") - if seed and int(seed[0]): - # 0 is the "disabled" convention everywhere else — treat it so - # here too (the controller's inactive-filter pass drops it). - parts.append(f"seed {int(seed[0])}") - ext_tokens.add("seed") - - return parts, ext_tokens - - async def _ubridge_apply_netem(self, host_ifc, filters): - """ - Translate the NIO's impairment filters into one tc netem qdisc on the - veth host end. The netem-expressible types (delay, packet_loss, - corrupt and the netem extensions rate/reorder/gemodel/duplicate/ - seed/limit) merge into a single qdisc — its egress covers the traffic - entering this container, and with both link endpoints applying - theirs, every direction of the link is impaired exactly once (the - same net effect as the relay, where both directions cross the single - filtered bridge). - - The kernel's netem replace MERGES optional attributes (rate, - correlation, reorder, corrupt, gemodel, distribution: absent attr = - previous value kept), so re-applying a filter set with a parameter - *removed* would silently keep the old value. A ``tc reset`` before - every ``netem set`` makes the re-apply a true reconcile (reset also - drops clsact and its bpf_drop filters — the caller re-applies them - right after in the same flow). An empty filter set is the reset - alone. - - :param host_ifc: veth host-end interface name - :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) - """ - - parts, ext_tokens = self._netem_command_parts(filters) - if parts and ext_tokens: - # Extension keywords need the netem-extension uBridge. Plain - # delay/loss/corrupt/dup never probes — an old uBridge serves - # the original surface untouched (mirrors bpf_drop's guard). - # Checked before the reset so a failed update leaves the - # previously applied qdisc in place. - caps = await self._ubridge_tc_capabilities() - missing = ext_tokens - set(caps.get("netem", "").split(",")) - if missing: - raise DockerError( - "Packet filter(s) using {} need a uBridge with the netem extensions " - "(tc capabilities reports '{}'); upgrade uBridge on this compute".format( - ", ".join(sorted(missing)), caps.get("netem") or "no tc support" - ) - ) - try: - await self._ubridge_send(f'tc reset "{host_ifc}"') - except UbridgeError as e: - # "No such file or directory" = no qdisc was attached: already - # clean. Everything else is a real failure and must surface. - if "No such file" not in str(e): - raise - if not parts: - return - await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) - - async def _ubridge_tc_capabilities(self): - """ - uBridge's tc-module feature report (``tc capabilities``), probed once - per uBridge process and cached — the reply shape is - ``netem=;ebpf=0|1;cbpf=0|1``. uBridge builds without the tc - module answer "Unknown module" and old builds without the command - answer "Unknown command": both mean no kernel filter support, cached - as an empty dict. - """ - - if self._ubridge_tc_caps is None: - caps = {} - if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - try: - reply = await self._ubridge_hypervisor.send("tc capabilities") - except UbridgeError: - reply = [] - for entry in (reply[0] if reply else "").split(";"): - key, _, value = entry.partition("=") - if key: - caps[key.strip()] = value.strip() - self._ubridge_tc_caps = caps - return self._ubridge_tc_caps - - async def _ubridge_apply_bpf_drops(self, host_ifc, filters): - """ - Translate the NIO's ``bpf`` filter (one expression per line, any line - matching drops the packet) into cls_bpf classifiers on the veth host - end's clsact egress: ``tc bpf_drop`` compiles each line with libpcap - (same compiler as the relay's bpf filter) and drops matches with a - gact TC_ACT_SHOT. Reconcile is always flush + re-add — same-prio - resend replaces the filter node in uBridge, and the flush drops - lines the user removed. - - :param host_ifc: veth host-end interface name - :param filters: NIO filters dictionary - """ - - bpf = filters.get("bpf") - lines = [line.strip() for line in (bpf[0].split("\n") if bpf else []) if line.strip()] - if not lines and self._ubridge_tc_caps is None: - # Nothing to add and nothing was ever installed (no tc module - # probed yet) — skip touching an old uBridge entirely. - return - caps = await self._ubridge_tc_capabilities() - if not lines: - if caps: - await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') - return - if caps.get("cbpf") != "1": - raise DockerError( - "Packet filter 'bpf' on a kernel-datapath link needs a uBridge with cBPF support " - "(tc capabilities reports no cbpf); upgrade uBridge on this compute or keep the " - "link on the relay datapath" - ) - await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') - for offset, line in enumerate(lines): - try: - await self._ubridge_send( - 'tc bpf_drop add "{ifc}" {prio} "{expr}"'.format(ifc=host_ifc, prio=10 + offset, expr=line) - ) - except UbridgeError as e: - if "Cannot compile filter" not in str(e): - raise - # Mirror the relay path: a line that no longer compiles on - # this uBridge's libpcap is skipped with a warning instead - # of breaking the link (the controller already validated - # the syntax with tcpdump at create/update time). - message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {line}" - log.warning(message) - self.project.emit("log.warning", {"message": message}) - - async def _ubridge_apply_ebpf_drops(self, host_ifc, filters): - """ - Translate the NIO's stateful filters into uBridge's eBPF impairment - classifier on the veth host end's clsact egress (prio 1, below - bpf_drop's 10-99): ``frequency_drop`` becomes the exact every-Nth - mode (``tc nth_drop``; -1 = drop everything maps to every 1st), the - kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``), - and the kernel-only ``window_drop`` type the time-window mode - (``tc window_drop``: single outage, or recurring flaps with an - optional per-cycle jitter). The program evaluates the modes in the - fixed order nth → quota → window. - - Reconcile is a plain re-set on every apply: the preceding netem - apply resets the interface (which detaches the program and drops - uBridge's per-interface registry entry), so a mode set re-loads the - program from scratch, and a mode left absent is turned off - explicitly (idempotent) so a removed filter stops dropping. For - window_drop this also means the schedule restarts on every apply: - the first window opens Start-ms from the moment of this call. - - Semantic difference vs the relay, documented server-side: the relay - counts packets of BOTH directions through its single filtered - bridge, while each veth end's classifier counts only the traffic - entering that container — with both endpoints applying theirs, each - direction drops every Nth independently (their window schedules - start ~simultaneously, so a round trip survives only when both - directions are outside their windows). - - :param host_ifc: veth host-end interface name - :param filters: NIO filters dictionary - """ - - def values_of(key): - values = filters.get(key) - if isinstance(values, (list, tuple)): - return list(values) - return [values] if values else [] - - frequency = values_of("frequency_drop") - quota = values_of("quota") - window = values_of("window_drop") - if not frequency and not quota and not window and self._ubridge_tc_caps is None: - # Nothing to set and nothing was ever installed (no probe yet) — - # skip touching an old uBridge entirely. - return - caps = await self._ubridge_tc_capabilities() - if not frequency and not quota and not window: - if caps.get("ebpf") == "1": - # Only when the eBPF module answers can anything be - # installed; "off" is idempotent in uBridge. - await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') - await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') - await self._ubridge_send(f'tc window_drop "{host_ifc}" off') - return - if caps.get("ebpf") != "1": - raise DockerError( - "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " - "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " - "the uBridge binary); upgrade uBridge on this compute or keep the link on the " - "relay datapath".format( - ", ".join(sorted(f for f in ("frequency_drop", "quota", "window_drop") if f in filters)) - ) - ) - if frequency: - # relay semantics: -1 = drop everything, N = every Nth packet - nth = 1 if int(frequency[0]) == -1 else int(frequency[0]) - await self._ubridge_send(f'tc nth_drop "{host_ifc}" {nth}') - else: - await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') - if quota: - await self._ubridge_send( - 'tc quota_drop "{ifc}" {bytes} {pct}'.format(ifc=host_ifc, bytes=int(quota[0]), pct=int(quota[1])) - ) - else: - await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') - if window: - # [start, outage, chance, period?, jitter?] — start is relative - # to this apply (the schedule restarts on every reconcile). - params = " ".join(str(int(v)) for v in window) - await self._ubridge_send(f'tc window_drop "{host_ifc}" {params}') - else: - await self._ubridge_send(f'tc window_drop "{host_ifc}" off') - - async def _remove_kernel_markers(self, host_ifc): - """ - Tear down every kernel marker anchored to *host_ifc*: the veth survives - link deletion (it is the adapter interface, not the link), so the - AF_PACKET sockets must be closed explicitly — otherwise a deleted - link's markers would keep sniffing and signaling. Mirrors the relay - datapath where deleting the uBridge bridge silently drops its filters. - """ - - from gns3server.compute.marker.marker_manager import MarkerManager - - manager = MarkerManager.instance() - markers_dir = self.project.markers_working_directory() - for (name, link_id), anchor in list(self._marker_filter_bridges.items()): - if anchor != host_ifc: - continue - self._marker_filter_bridges.pop((name, link_id)) - self._marker_specs.pop((name, link_id), None) - await self._ubridge_delete_marker_filter(host_ifc, name) - try: - os.remove(os.path.join(markers_dir, f"{self._id}_{link_id}_{name}.pcap")) - except FileNotFoundError: - pass - except OSError as e: - log.warning("Could not remove marker pcap for '%s' on link %s: %s", name, link_id, e) - manager.unregister(self._id, name) - - async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): - """ - Detach a kernel-datapath adapter port from its per-link kernel bridge. - Both endpoints run this concurrently: deleting a bridge that still has - the peer's port enslaved fails with EBUSY, so the last endpoint to - remove its port wins the deletion and the loser's failure is expected - and suppressed. - """ - - host_ifc = self._kernel_veths.get((adapter_number, port_number)) - if host_ifc is not None: - await self._remove_kernel_markers(host_ifc) - # The veth survives link deletion, so an orphaned netem qdisc - # would keep impairing whatever attaches to the adapter next - # (kernel link or relay). Best-effort: ENOENT just means no - # qdisc was attached. - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tc reset "{host_ifc}"') - if self.status == "started": - await self._set_adapter_carrier(adapter_number, False, port_number) - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delete "{nio.bridge}"') - def get_nio(self, adapter_number, port_number=0): """ Gets an adapter NIO binding. diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py new file mode 100644 index 000000000..33f2962b1 --- /dev/null +++ b/gns3server/compute/ubridge/tc_probe.py @@ -0,0 +1,114 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Standalone ``tc capabilities`` probe. + +Spawns the configured uBridge outside any node lifecycle, asks its tc module +for the feature report and caches the answer by binary identity (path, +mtime, size) — the same binary never answers differently, but replacing it +must re-probe. This feeds the compute ``/capabilities`` payload so the +controller and clients can gate kernel-datapath filter types on what this +host's uBridge can actually run, as the server's own user — which is the +point: the runtime ``ebpf`` flag flips between users and kernels for the +same binary, so a probe run as root would lie for a non-root server. + +A node's own probe (DockerVM._ubridge_tc_capabilities) stays per-process; +this one only answers "what can this binary do here". Never raises: any +failure (missing binary, old build answering "Unknown command", timeout) +means unknown, reported as None and cached as such. +""" + +import os +import uuid +import shutil +import asyncio +import tempfile +import logging +import contextlib + +from gns3server.config import Config +from gns3server.utils.tc_capabilities import parse_tc_capabilities +from .hypervisor import Hypervisor +from .ubridge_error import UbridgeError + +log = logging.getLogger(__name__) + +# binary identity -> parsed report (None = probed and unusable) +_cache = {} +_lock = asyncio.Lock() + + +async def probe_tc_capabilities(timeout: float = 15.0): + """ + Probe the configured uBridge's tc module once per binary identity. + + :param timeout: overall timeout for spawn + ask, in seconds + :returns: parsed ``tc capabilities`` dict, or None when unknown + """ + + config = Config.instance() + path = shutil.which(config.settings.Server.ubridge_path) + if not path: + return None + try: + stat = os.stat(path) + key = (path, stat.st_mtime_ns, stat.st_size) + except OSError: + return None + + async with _lock: + if key in _cache: + return _cache[key] + caps = await _probe(path, config, timeout) + _cache[key] = caps + return caps + + +async def _probe(path, config, timeout): + """ + Spawn a throwaway uBridge, ask ``tc capabilities``, tear it down. + """ + + working_dir = tempfile.mkdtemp(prefix="gns3-tc-probe-") + hypervisor = Hypervisor( + None, + path, + working_dir, + config.settings.Server.ubridge_control_transport, + config.settings.Server.host, + str(uuid.uuid4()), + ) + try: + + async def ask(): + await hypervisor.start() + await hypervisor.connect() + reply = await hypervisor.send("tc capabilities") + return parse_tc_capabilities(reply[0] if reply else "") + + try: + caps = await asyncio.wait_for(ask(), timeout=timeout) + except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e: + # Old uBridge builds answer "Unknown command"/"Unknown module"; + # anything else here is a broken install — either way: unknown. + log.debug("uBridge tc capabilities probe failed: %s", e) + return None + return caps if caps else None + finally: + with contextlib.suppress(Exception): + await hypervisor.stop() + shutil.rmtree(working_dir, ignore_errors=True) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 1b3b65e38..ef2593f66 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -92,7 +92,15 @@ class Compute: self._disk_usage_percent = 0 self._last_error = None self._ssl_context = ssl_context - self._capabilities = {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": []} + self._capabilities = { + "version": "", + "platform": "", + "cpus": 0, + "memory": 0, + "disk_size": 0, + "node_types": [], + "ubridge_tc": None, + } self.name = name # Cache of interfaces on remote host self._interfaces_cache = None @@ -288,6 +296,15 @@ class Compute: def disk_usage_percent(self): return self._disk_usage_percent + @property + def capabilities(self) -> dict: + """ + Capabilities reported by this compute (fetched from GET /capabilities + at connect time). + """ + + return self._capabilities + def asdict(self, topology_dump=False): """ :param topology_dump: Filter to keep only properties require for saving on disk diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index a6ff17822..4a324bb65 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -736,11 +736,15 @@ class Link: filter_node = self._get_filter_node() if filter_node: if self.kernel_datapath: - # Kernel-datapath links serve every filter type: netem for - # delay/loss/corrupt and the netem extensions, cls_bpf for - # bpf, the eBPF classifier for frequency_drop, quota and - # window_drop. - return FILTERS + # Kernel-datapath links serve every filter type in principle: + # netem for delay/loss/corrupt and the netem extensions, + # cls_bpf for bpf, the eBPF classifier for frequency_drop, + # quota and window_drop. What is offered is gated on the + # endpoints' computes reporting a uBridge that can run each + # type; computes that report nothing (older servers, failed + # probe) keep the full list — update_filters still rejects + # with 409 on apply. + return [f for f in FILTERS if self._filter_supported_by_computes(f["type"])] # Relay links: hide the kernel-only types the relay cannot run # (they would be rejected with 409 on update). from gns3server.utils.packet_filter_validation import KERNEL_ONLY_FILTERS @@ -748,6 +752,37 @@ class Link: return [f for f in FILTERS if f["type"] not in KERNEL_ONLY_FILTERS] return [] + def _filter_supported_by_computes(self, filter_type): + """ + Whether every endpoint's compute reports a uBridge able to run + *filter_type* on a kernel-datapath link (both ends apply their own + filters, so one incapable compute is enough to hide the type). + Computes without a uBridge tc report impose no constraint. + """ + + from gns3server.utils.tc_capabilities import FILTER_EBPF_MODES, FILTER_NETEM_KEYWORDS + + reported = [] + for side in self._nodes: + capabilities = side["node"].compute.capabilities + if isinstance(capabilities, dict) and capabilities.get("ubridge_tc"): + reported.append(capabilities["ubridge_tc"]) + if not reported: + return True + for caps in reported: + if filter_type in FILTER_EBPF_MODES: + if FILTER_EBPF_MODES[filter_type] not in caps["ebpf_modes"]: + return False + elif filter_type == "bpf": + if not caps["cbpf"]: + return False + elif filter_type in FILTER_NETEM_KEYWORDS: + if FILTER_NETEM_KEYWORDS[filter_type] not in caps["netem"]: + return False + # the remaining types (delay, packet_loss, corrupt) run on any + # build that reports tc capabilities at all + return True + def _get_filter_node(self): """ Return the node where the filter will run diff --git a/gns3server/schemas/controller/capabilities.py b/gns3server/schemas/controller/capabilities.py index 6e881ede7..bd4b8832a 100644 --- a/gns3server/schemas/controller/capabilities.py +++ b/gns3server/schemas/controller/capabilities.py @@ -15,13 +15,27 @@ # along with this program. If not, see . -from typing import List +from typing import List, Optional from pydantic import BaseModel, Field from .nodes import NodeType +class UbridgeTcCapabilities(BaseModel): + """ + uBridge tc-module capabilities (kernel-datapath packet filters), probed + by the compute as its own user. + """ + + netem: List[str] = Field(..., description="netem keywords supported by this uBridge's tc module") + ebpf: bool = Field(..., description="Whether the eBPF stateful classifier loads on this host") + ebpf_modes: List[str] = Field( + ..., description="eBPF classifier modes usable on this host (ebpf is on and the mode token is declared)" + ) + cbpf: bool = Field(..., description="Whether classic-BPF match-drop classifiers are available") + + class Capabilities(BaseModel): """ Capabilities properties. @@ -33,3 +47,6 @@ class Capabilities(BaseModel): cpus: int = Field(..., description="Number of CPUs on this compute") memory: int = Field(..., description="Amount of memory on this compute") disk_size: int = Field(..., description="Disk size on this compute") + ubridge_tc: Optional[UbridgeTcCapabilities] = Field( + None, description="uBridge tc-module capabilities; absent when the probe failed or uBridge has no tc module" + ) diff --git a/gns3server/schemas/controller/computes.py b/gns3server/schemas/controller/computes.py index 3db2656cc..836ac9607 100644 --- a/gns3server/schemas/controller/computes.py +++ b/gns3server/schemas/controller/computes.py @@ -21,6 +21,7 @@ from typing import Any, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator from .base import DateTimeModelMixin +from .capabilities import UbridgeTcCapabilities from .nodes import NodeType @@ -103,6 +104,9 @@ class Capabilities(BaseModel): cpus: int = Field(..., description="Number of CPUs on this compute") memory: int = Field(..., description="Amount of memory on this compute") disk_size: int = Field(..., description="Disk size on this compute") + ubridge_tc: Optional[UbridgeTcCapabilities] = Field( + None, description="uBridge tc-module capabilities; absent when the probe failed or uBridge has no tc module" + ) class Compute(DateTimeModelMixin, ComputeBase): diff --git a/gns3server/utils/tc_capabilities.py b/gns3server/utils/tc_capabilities.py new file mode 100644 index 000000000..3fed73582 --- /dev/null +++ b/gns3server/utils/tc_capabilities.py @@ -0,0 +1,85 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +uBridge ``tc capabilities`` vocabulary: the reply parser and the server-side +policy built on top of it. Shared by the compute side (the per-node probe in +DockerVM and the standalone /capabilities probe) and the controller side +(filter availability), so both gate on the same rules. + +Reply shape (additive by contract — unknown keys pass through untouched): +``netem=;ebpf=0|1;cbpf=0|1[;ebpf_modes=]`` +""" + +# eBPF classifier modes that shipped before the ``ebpf_modes`` field existed +# (uBridge feature/tc-precision): the field was added together with the +# window-mode correction, so an ebpf=1 build that does not emit it has +# exactly these. window is the only mode whose semantics changed after +# release — it is never offered unless the build declares the token. +LEGACY_EBPF_MODES = ("nth", "quota", "flow") + +# GNS3 filter type -> eBPF classifier mode token +FILTER_EBPF_MODES = { + "frequency_drop": "nth", + "quota": "quota", + "window_drop": "window", +} + +# GNS3 filter type -> netem keyword (netem= token in the capabilities reply) +FILTER_NETEM_KEYWORDS = { + "rate": "rate", + "reorder": "reorder", + "gemodel": "gemodel", + "duplicate": "dup", + "seed": "seed", + "limit": "limit", +} + + +def parse_tc_capabilities(reply): + """ + Parse the payload of a ``tc capabilities`` reply into a dict. + + :param reply: reply payload (first reply line, no status code) + :returns: dict of keys to values, both strings + """ + + caps = {} + for entry in (reply or "").split(";"): + key, _, value = entry.partition("=") + if key: + caps[key.strip()] = value.strip() + return caps + + +def usable_ebpf_modes(caps): + """ + The eBPF classifier modes this uBridge build can actually run: a mode is + usable iff ``ebpf=1`` and its token is listed in ``ebpf_modes``. Builds + that do not emit the field predate it and keep the modes that shipped + with it (LEGACY_EBPF_MODES) — gating on ``ebpf=1`` alone would silently + accept window_drop on the pre-correction build, which drops it into + back-to-back windows. + + :param caps: parsed ``tc capabilities`` dict + :returns: tuple of usable mode tokens, in reply order + """ + + if caps.get("ebpf") != "1": + return () + if "ebpf_modes" not in caps: + return LEGACY_EBPF_MODES + return tuple(mode for mode in caps["ebpf_modes"].split(",") if mode) diff --git a/pyproject.toml b/pyproject.toml index ef7b82573..6450b621e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,7 @@ extend-immutable-calls = [ "gns3server/api/routes/controller/users.py" = ["S105", "S106"] "gns3server/compute/base_node.py" = ["C901", "RUF019"] "gns3server/compute/docker/__init__.py" = ["B006", "C901"] +"gns3server/compute/docker/docker_kernel_datapath.py" = ["C901"] "gns3server/compute/docker/docker_vm.py" = ["B006", "C901", "E731", "RUF003", "RUF006", "S108"] "gns3server/compute/docker/iol_docker_vm.py" = ["RUF002", "S108"] "gns3server/compute/docker/vendor_docker_vm.py" = ["RUF002", "S108"] diff --git a/tests/api/routes/compute/test_capabilities.py b/tests/api/routes/compute/test_capabilities.py index a401e8f24..f3216daea 100644 --- a/tests/api/routes/compute/test_capabilities.py +++ b/tests/api/routes/compute/test_capabilities.py @@ -29,8 +29,12 @@ pytestmark = pytest.mark.asyncio class TestCapabilitiesRoutes: - async def test_get(self, app: FastAPI, compute_client: AsyncClient, windows_platform) -> None: + async def test_get(self, app: FastAPI, compute_client: AsyncClient, windows_platform, monkeypatch) -> None: + async def no_probe(): + return None + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -54,10 +58,15 @@ class TestCapabilitiesRoutes: "cpus": psutil.cpu_count(logical=True), "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, + "ubridge_tc": None, } - async def test_get_on_gns3vm(self, app: FastAPI, compute_client: AsyncClient, on_gns3vm) -> None: + async def test_get_on_gns3vm(self, app: FastAPI, compute_client: AsyncClient, on_gns3vm, monkeypatch) -> None: + async def no_probe(): + return None + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -81,4 +90,20 @@ class TestCapabilitiesRoutes: "cpus": psutil.cpu_count(logical=True), "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, + "ubridge_tc": None, + } + + async def test_get_reports_ubridge_tc(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: + + async def fake_probe(): + return {"netem": "delay,rate", "ebpf": "1", "cbpf": "0", "ebpf_modes": "nth,quota,window,flow"} + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", fake_probe) + response = await compute_client.get(app.url_path_for("compute:get_capabilities")) + assert response.status_code == status.HTTP_200_OK + assert response.json()["ubridge_tc"] == { + "netem": ["delay", "rate"], + "ebpf": True, + "ebpf_modes": ["nth", "quota", "window", "flow"], + "cbpf": False, } diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index abb86049b..058288889 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -569,6 +569,66 @@ async def test_apply_window_drop_requires_ebpf_capability(vm): vm._ubridge_send.assert_not_called() +@pytest.mark.asyncio +async def test_apply_window_drop_requires_mode_token(vm): + """ + ebpf=1 is not enough: a build that does not declare the window mode + token (pre-correction feature/tc-precision semantics) must get a clear + upgrade error instead of silently installing back-to-back windows. + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1", modes="nth,quota") + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + with pytest.raises(DockerError, match="ebpf_modes=nth,quota"): + await vm._ubridge_apply_ebpf_drops(host_ifc, {"window_drop": [0, 2000, 100]}) + vm._ubridge_send.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_ebpf_drops_legacy_build_keeps_shipped_modes(vm): + """ + A build predating the ebpf_modes field (feature/tc-precision) keeps the + modes that shipped with it: nth/quota still apply, window does not. + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1", modes=None) + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + await vm._ubridge_apply_ebpf_drops(host_ifc, {"frequency_drop": [3]}) + vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" 3') + vm._ubridge_send.assert_any_call(f'tc quota_drop "{host_ifc}" off') + + vm._ubridge_send.reset_mock() + with pytest.raises(DockerError, match="newer uBridge"): + await vm._ubridge_apply_ebpf_drops(host_ifc, {"window_drop": [0, 2000, 100]}) + vm._ubridge_send.assert_not_called() + + +@pytest.mark.asyncio +async def test_mode_off_skips_undeclared_modes(vm): + """ + The cleanup path only sends "off" for modes this build declares — an + undeclared mode's command would be an unknown command on that build. + """ + + vm._ubridge_hypervisor = _caps_hypervisor(ebpf="1", modes="nth") + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + + # A first apply with a filter forces the capability probe; the second + # (filters removed) takes the cached caps down the off path. + await vm._ubridge_apply_ebpf_drops(host_ifc, {"frequency_drop": [3]}) + vm._ubridge_send.reset_mock() + await vm._ubridge_apply_ebpf_drops(host_ifc, {}) + + vm._ubridge_send.assert_any_call(f'tc nth_drop "{host_ifc}" off') + sent = [call.args[0] for call in vm._ubridge_send.call_args_list] + assert not any("quota_drop" in command or "window_drop" in command for command in sent) + + @pytest.mark.asyncio async def test_apply_ebpf_drops_skips_old_ubridge_without_filters(vm): """ @@ -605,11 +665,18 @@ async def test_apply_ebpf_drops_cleanup_only_probes_with_ebpf(vm): vm._ubridge_send.assert_not_called() -def _caps_hypervisor(cbpf="1", ebpf="0"): - """A hypervisor mock whose direct send() answers `tc capabilities`.""" +def _caps_hypervisor(cbpf="1", ebpf="0", modes="nth,quota,window"): + """ + A hypervisor mock whose direct send() answers `tc capabilities`. + modes=None emulates a build predating the ebpf_modes field. + """ + + reply = f"netem=delay;ebpf={ebpf};cbpf={cbpf}" + if modes is not None: + reply += f";ebpf_modes={modes}" hyp = MagicMock() hyp.is_running.return_value = True - hyp.send = AsyncioMagicMock(return_value=[f"netem=delay;ebpf={ebpf};cbpf={cbpf}"]) + hyp.send = AsyncioMagicMock(return_value=[reply]) return hyp @@ -682,7 +749,7 @@ async def test_apply_bpf_drops_compile_failure_warns(vm): raise UbridgeError("209-Cannot compile filter 'icmp': can't parse filter expression: syntax error") vm._ubridge_send = AsyncioMagicMock(side_effect=send) - with patch("gns3server.compute.docker.docker_vm.log.warning") as mock_log: + with patch("gns3server.compute.docker.docker_kernel_datapath.log.warning") as mock_log: await vm._ubridge_apply_bpf_drops(host_ifc, {"bpf": ["icmp"]}) # must not raise assert mock_log.called @@ -708,7 +775,7 @@ async def test_ubridge_tc_capabilities_probed_once(vm): vm._ubridge_hypervisor = _caps_hypervisor() caps = await vm._ubridge_tc_capabilities() - assert caps == {"netem": "delay", "ebpf": "0", "cbpf": "1"} + assert caps == {"netem": "delay", "ebpf": "0", "cbpf": "1", "ebpf_modes": "nth,quota,window"} await vm._ubridge_tc_capabilities() assert vm._ubridge_hypervisor.send.call_count == 1 diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py new file mode 100644 index 000000000..6bb5178cd --- /dev/null +++ b/tests/compute/ubridge/test_tc_probe.py @@ -0,0 +1,115 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +The standalone `tc capabilities` probe behind the compute /capabilities +payload: spawn a throwaway uBridge, ask once, cache by binary identity. +""" + +import pytest + +from gns3server.compute.ubridge import tc_probe +from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities +from gns3server.compute.ubridge.ubridge_error import UbridgeError + +pytestmark = pytest.mark.asyncio + + +class FakeHypervisor: + """Records spawns; answers `tc capabilities` with a canned reply.""" + + reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] + error = None + spawned = 0 + + def __init__(self, project, path, working_dir, transport, host, node_id): + + self.commands = [] + FakeHypervisor.spawned += 1 + + async def start(self): + pass + + async def connect(self): + pass + + async def send(self, command): + + self.commands.append(command) + if self.error: + raise UbridgeError(self.error) + if command == "tc capabilities": + return FakeHypervisor.reply + return ["OK"] + + async def stop(self): + pass + + +@pytest.fixture(autouse=True) +def probe_env(monkeypatch, tmp_path): + + FakeHypervisor.spawned = 0 + FakeHypervisor.error = None + FakeHypervisor.reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] + tc_probe._cache.clear() + # a real file on disk: the cache is keyed on its (path, mtime, size) + binary = tmp_path / "ubridge" + binary.write_bytes(b"") + monkeypatch.setattr(tc_probe.shutil, "which", lambda name: str(binary)) + monkeypatch.setattr(tc_probe, "Hypervisor", FakeHypervisor) + + +async def test_probe_returns_parsed_report(): + + caps = await probe_tc_capabilities() + assert caps == { + "netem": "delay,rate", + "ebpf": "1", + "cbpf": "1", + "ebpf_modes": "nth,quota,window,flow", + } + assert FakeHypervisor.spawned == 1 + + +async def test_probe_cached_by_binary_identity(): + + first = await probe_tc_capabilities() + second = await probe_tc_capabilities() + assert first is second + assert FakeHypervisor.spawned == 1 + + +async def test_probe_failure_means_unknown(): + + FakeHypervisor.error = "Unknown command" # old uBridge build + assert await probe_tc_capabilities() is None + # the failure is cached too: no respawn loop on every /capabilities hit + assert await probe_tc_capabilities() is None + assert FakeHypervisor.spawned == 1 + + +async def test_probe_missing_binary_means_unknown(monkeypatch): + + monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) + assert await probe_tc_capabilities() is None + assert FakeHypervisor.spawned == 0 + + +async def test_probe_empty_reply_means_unknown(): + + FakeHypervisor.reply = [] + assert await probe_tc_capabilities() is None diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index 4fb3e9f7e..a6f41e2d3 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -381,7 +381,7 @@ async def test_json(compute): "disk_usage_percent": 0, "connected": True, "last_error": None, - "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": []}, + "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None}, } assert compute.asdict(topology_dump=True) == { "compute_id": "my_compute_id", diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index f6ee38410..1ce85ac30 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -548,3 +548,45 @@ async def test_prepare_relay_link_strips_kernel_only_filters(project): # carries no filters — the relay filters one bridge) assert by_node[node1.id]["filters"] == {"frequency_drop": [7], "delay": [100, 20], "packet_loss": [5]} assert by_node[node2.id]["filters"] == {} + + +@pytest.mark.asyncio +async def test_available_filters_gated_by_compute_capabilities(project): + """ + Kernel links only offer the filter types the endpoints' computes report + a uBridge able to run (ebpf_modes tokens, cbpf, netem keywords). + """ + + link, node1, _node2 = await _kernel_link(project) + link._link_data = [{"type": "nio_bridge"}] + node1.compute.capabilities = { + "ubridge_tc": { + "netem": ["delay", "jitter", "loss", "rate"], + "ebpf": True, + "ebpf_modes": ["nth", "quota"], + "cbpf": False, + } + } + types = [f["type"] for f in link.available_filters()] + assert "frequency_drop" in types # nth declared + assert "quota" in types # quota declared + assert "window_drop" not in types # token missing + assert "bpf" not in types # cbpf off + assert "rate" in types # netem keyword present + assert "gemodel" not in types # netem keyword missing + assert "delay" in types # core netem runs on any tc-module build + + +@pytest.mark.asyncio +async def test_available_filters_unconstrained_without_report(project): + """ + Computes that report no uBridge tc capabilities (older servers, failed + probe) impose no constraint: the full list stays offered and apply-time + validation is the guard. + """ + + from gns3server.controller.link import FILTERS + + link, _node1, _node2 = await _kernel_link(project) + link._link_data = [{"type": "nio_bridge"}] + assert [f["type"] for f in link.available_filters()] == [f["type"] for f in FILTERS] diff --git a/tests/utils/test_tc_capabilities.py b/tests/utils/test_tc_capabilities.py new file mode 100644 index 000000000..75befaea4 --- /dev/null +++ b/tests/utils/test_tc_capabilities.py @@ -0,0 +1,77 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +The `tc capabilities` reply parser and the eBPF mode policy shared by the +compute probe and the controller filter availability. +""" + +from gns3server.utils.tc_capabilities import ( + LEGACY_EBPF_MODES, + parse_tc_capabilities, + usable_ebpf_modes, +) + + +def test_parse_full_reply(): + + caps = parse_tc_capabilities("netem=delay,jitter,loss;ebpf=1;cbpf=0;ebpf_modes=nth,quota,window,flow") + assert caps == { + "netem": "delay,jitter,loss", + "ebpf": "1", + "cbpf": "0", + "ebpf_modes": "nth,quota,window,flow", + } + + +def test_parse_legacy_reply_without_modes(): + """A build predating the field (and unknown future keys) passes through.""" + + caps = parse_tc_capabilities("netem=delay;ebpf=1;cbpf=1") + assert caps == {"netem": "delay", "ebpf": "1", "cbpf": "1"} + + +def test_parse_empty_reply(): + + assert parse_tc_capabilities("") == {} + assert parse_tc_capabilities(None) == {} + + +def test_usable_modes_require_ebpf(): + """The mode list is a build fact; ebpf is the runtime gate for all of them.""" + + assert usable_ebpf_modes({"ebpf": "0", "ebpf_modes": "nth,quota,window,flow"}) == () + assert usable_ebpf_modes({}) == () + + +def test_usable_modes_from_token_list(): + + caps = {"ebpf": "1", "ebpf_modes": "nth,quota,window,flow"} + assert usable_ebpf_modes(caps) == ("nth", "quota", "window", "flow") + + +def test_usable_modes_legacy_build_keeps_shipped_modes(): + """No field: the modes that shipped before it — everything but window.""" + + assert usable_ebpf_modes({"ebpf": "1"}) == LEGACY_EBPF_MODES + assert "window" not in LEGACY_EBPF_MODES + + +def test_window_never_usable_without_token(): + """The pre-correction build reported ebpf=1 with broken window semantics.""" + + assert "window" not in usable_ebpf_modes({"ebpf": "1"}) + assert "window" not in usable_ebpf_modes({"ebpf": "1", "ebpf_modes": "nth,quota"}) From cecaf298bba02e085cffeb1ba996a0bd0ed24961 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 10:00:38 +0800 Subject: [PATCH 016/101] feat(controller): expose kernel_datapath on the link payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runtime fact recomputed from the NIO wiring on every load (not persisted). Without it clients cannot tell a relay link from a kernel link whose compute lacks a filter capability — both show up as kernel-only types missing from available_filters. --- gns3server/controller/link.py | 5 +++++ tests/controller/test_link.py | 1 + 2 files changed, 6 insertions(+) diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 4a324bb65..337df9e29 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -847,6 +847,11 @@ class Link: "capture_file_path": self.capture_file_path, "capture_compute_id": self.capture_compute_id, "link_type": self._link_type, + # runtime fact, not persisted: recomputed from the NIO wiring on + # every load (tells clients which datapath impairs this link — + # kernel-only filter types absent from available_filters are + # then distinguishable from capability gaps) + "kernel_datapath": self.kernel_datapath, "filters": self._filters, "markers": self._markers, "suspend": self._suspended, diff --git a/tests/controller/test_link.py b/tests/controller/test_link.py index 12436abb5..66a94e708 100644 --- a/tests/controller/test_link.py +++ b/tests/controller/test_link.py @@ -228,6 +228,7 @@ async def test_json(project, compute): "suspend": False, "wireshark": False, "link_type": "ethernet", + "kernel_datapath": False, "capturing": False, "capture_file_name": None, "capture_file_path": None, From 935c00802f1952ca459fd026f9c303c244b5d05b Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 14:37:02 +0800 Subject: [PATCH 017/101] docs(mcp): refresh filter vocabulary in the MCP/copilot tool descriptions link_update, link_available_filters and the copilot packet-filter tool still advertised the pre-kernel-datapath five types. List the full thirteen with positional parameter formats, ranges and kernel-only marking, point at link_available_filters as the per-link authority (topology + compute capability gated), and document window_drop's apply-relative schedule restart. Also add kernel_datapath to the link field lists. Description-only: handlers pass filters through unchanged. --- .../gns3_copilot/gns3_client/api_handlers.py | 4 +- .../tools_v2/gns3_packet_filter.py | 20 ++++++- gns3server/agent/mcp/__init__.py | 54 ++++++++++++++----- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py b/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py index bce5ba091..a7eb0e04d 100644 --- a/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py +++ b/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py @@ -692,8 +692,8 @@ def available_filters_handler(params: dict[str, Any], gns3_ctx: dict[str, Any]) """ List the packet filter types available for a link (GNS3 API v3 only). - Returns a list of filter descriptors (frequency_drop, packet_loss, - delay, corrupt, bpf) with their parameters. + Returns a list of filter descriptors with their parameters — the + authoritative, per-link answer (topology and compute-capability gated). """ project_id = params.get("project_id") link_id = params.get("link_id") diff --git a/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py b/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py index ed563cbbd..e6aee5583 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py +++ b/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py @@ -122,10 +122,26 @@ class GNS3PacketFilterTool(BaseTool): Common filter types for fault injection: - "frequency_drop": Drop every Nth packet (parameter: frequency, -1 to 32767) - - "packet_loss": Packet loss percentage (parameter: chance, 0-100) - - "delay": Delay in ms with optional jitter (parameters: latency 0-32767, jitter 0-32767) + - "packet_loss": Packet loss percentage (parameters: chance 0-100, optional correlation) + - "delay": Delay in ms with optional jitter and distribution (parameters: latency 0-32767, + jitter 0-32767, distribution uniform|normal|pareto|paretonormal) - "corrupt": Packet corruption percentage (parameter: chance, 0-100) - "bpf": Berkeley Packet Filter (parameter: filter expression text) + - "rate": Bandwidth cap, tc-style string like "512kbit" (kernel links only) + - "reorder": Packet reordering (parameters: percentage, correlation, gap; needs delay; + kernel links only) + - "gemodel": Gilbert-Elliot bursty loss (parameters: p, r, 1-h; kernel links only) + - "duplicate": Packet duplication percentage (kernel links only) + - "seed": Reproducible netem random draws (kernel links only) + - "limit": Impairment queue depth in packets (kernel links only) + - "quota": Byte cap — after the quota, drop with a chance (parameters: bytes, chance; + kernel links only) + - "window_drop": Time-window outage (parameters: start_ms, outage_ms, chance, optional + period_ms, jitter_ms; start is relative to when the filter is applied — any filter + update restarts the schedule; kernel links only) + + Filter availability depends on the link (kernel-only types need kernel-datapath links + and compute support): always check with action "get_available" before setting. Input is a JSON object with: - project_id (str): GNS3 project UUID diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index f52b05552..0f57ac7ed 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -697,7 +697,7 @@ async def link_list( fields: Annotated[ list[str] | None, Field( - description='Optional: return only these fields. e.g. ["link_id","nodes"]. Available: link_id, project_id, link_type, nodes, suspend, filters, capturing, capture_file_name, link_style' + description='Optional: return only these fields. e.g. ["link_id","nodes"]. Available: link_id, project_id, link_type, kernel_datapath, nodes, suspend, filters, capturing, capture_file_name, link_style' ), ] = None, ) -> list[dict[str, Any]]: @@ -733,7 +733,7 @@ async def link_create( list[str] | None, Field( description="Response fields to include (default: [link_id, link_type, nodes]). " - "Available: link_id, project_id, link_type, nodes, suspend, " + "Available: link_id, project_id, link_type, kernel_datapath, nodes, suspend, " "link_style, filters, show_filters_icon, capturing, " "capture_file_name, capture_file_path, capture_compute_id, wireshark" ), @@ -787,24 +787,51 @@ async def link_update( Supported kwargs: - suspend: boolean - Suspend or resume the link - - filters: dict - Packet filters (must use array format): - * frequency_drop: [N] - Drop every Nth packet (N: -1 to 32767) - * packet_loss: [rate] - Packet loss percentage (rate: 0 to 100) - * delay: [ms, jitter] - Latency and jitter in milliseconds - * corrupt: [rate] - Packet corruption percentage (rate: 0 to 100) - * bpf: [expression] - Berkeley Packet Filter expression + - filters: dict - Packet filters (values MUST be positional arrays). Types + marked (K) run on kernel-datapath links only (Docker-to-Docker on one + compute) and are rejected with 409 on relay links; which types a given + link accepts is authoritatively answered by link_available_filters: + * frequency_drop: [N] - Drop every Nth packet (N: -1 to 32767; -1 = drop + everything). Exact on kernel links (eBPF counter) + * packet_loss: [chance, correl?] - Loss percentage (0-100); correl (K) + makes consecutive losses dependent (bursty) + * delay: [ms, jitter?, distribution?] - Latency 1-32767 ms, jitter 0-32767 + ms; distribution (K) one of uniform|normal|pareto|paretonormal, needs + jitter > 0 + * corrupt: [chance] - Corruption percentage (0-100) + * bpf: [expression] - Berkeley Packet Filter expression, one per line, + any match drops + * rate (K): ["512kbit"] - Bandwidth cap, tc-style int+unit (bit/kbit/ + mbit/gbit/bps/kbps/mbps, max 100gbit) + * reorder (K): [pct, correl?, gap?] - Requires delay to also be set + * gemodel (K): [p, r, 1-h] - Gilbert-Elliot bursty loss; mutually + exclusive with packet_loss + * duplicate (K): [pct, correl?] - Duplication percentage + * seed (K): [u32] - Makes netem random draws reproducible + * limit (K): [pkts] - Impairment queue depth (default 1000; raise for + low rate + long delay) + * quota (K): [bytes, pct] - After the byte quota is consumed, each + further packet drops with the chance (100 = hard cutoff) + * window_drop (K): [start_ms, outage_ms, pct, period_ms?, jitter_ms?] - + Drop with chance pct inside [start, start+outage): 3 args = single + outage (traffic passes before and after), 4 args = recurring flap + (period >= outage), 5 args = per-cycle timing randomized within +/- + jitter. start is measured from the moment this filter is applied and + ANY filter update / node restart restarts the schedule; start 0 = + outage begins immediately (active, not disabled) Example filters: {"filters": {"frequency_drop": [10]}} {"filters": {"delay": [100, 10]}} - {"filters": {"packet_loss": [5]}} + {"filters": {"window_drop": [0, 2000, 100]}} {"filters": {"delay": [50, 5], "packet_loss": [2]}} To clear all filters: {"filters": {}} Filters are applied **bidirectionally** — a packet crossing the link twice (e.g. ping round-trip) is filtered in both directions independently. - For example, packet_loss: [50] gives ~75% observed loss (1 - 0.5²), not 50%. + For example, packet_loss: [50] gives ~75% observed loss (1 - 0.5²), not 50%; + window_drop pct likewise applies per direction. ARP frames also pass through filters; at high loss/corrupt rates, pre-set static ARP entries to avoid false "Destination Host Unreachable" errors. """ @@ -817,8 +844,11 @@ async def link_available_filters( project_id: Annotated[str, Field(description="UUID of the project")], link_id: Annotated[str, Field(description="UUID of the link")], ) -> list[dict[str, Any]]: - """List the packet filter types available for a link (frequency_drop, packet_loss, delay, corrupt, bpf) - with their parameters. Use before setting filters with link_update.""" + """List the packet filter types available for a link with their parameters + and ranges — the authoritative answer for what link_update accepts on THIS + link (topology and compute-capability gated: kernel-only types appear only + on kernel-datapath links whose compute reports support). Call this before + setting filters; a type absent here will be rejected with 409.""" return await asyncio.to_thread( _run_handler_sync, available_filters_handler, From 5ac1521217f2d00992002b3d210f5a7d2e652f99 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 14:47:45 +0800 Subject: [PATCH 018/101] feat(mcp): accept project/node names as tool locators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLM agents transcribe UUIDs unreliably — a 2-character slip invents an id that never existed and is invisible to the eye. Tool parameters now accept names wherever the server guarantees uniqueness: project names among live projects, node names within a project. Links, drawings, appliances and snapshots have no names and stay UUID-only. Resolution sits in the single dispatch choke point (_run_handler_sync): UUID-shaped values pass through untouched, names resolve with one list call, and an unknown name fails fast listing the known names instead of a confusing downstream 404. As a backstop for residual UUID typos, error replies carrying " doesn't exist" are matched against the in-scope ids; a single candidate within edit distance 2 earns a "Did you mean ...?" hint. --- gns3server/agent/mcp/__init__.py | 157 ++++++++++++---------- gns3server/agent/mcp/locators.py | 223 +++++++++++++++++++++++++++++++ tests/agent/mcp/test_locators.py | 170 +++++++++++++++++++++++ 3 files changed, 478 insertions(+), 72 deletions(-) create mode 100644 gns3server/agent/mcp/locators.py create mode 100644 tests/agent/mcp/test_locators.py diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index 0f57ac7ed..180087905 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -122,6 +122,7 @@ from .images import ( install_images_handler, prune_images_handler, ) +from .locators import add_did_you_mean, resolve_locators from .projects import ( close_project_handler, create_project_handler, @@ -339,7 +340,19 @@ def _run_handler_sync(handler, params: dict[str, Any]) -> list[dict[str, Any]]: "jwt_username": _jwt_username_var.get(), "jwt_token_version": _jwt_token_version_var.get(), } - result = handler(params, ctx) + + def run(h, p): + return h(p, ctx) + + # Name-shaped project_id/node_id values become UUIDs before the handler + # runs; a name that matches nothing fails fast with the known names. + params, resolution_error = resolve_locators(params, run) + if resolution_error is not None: + result = resolution_error + else: + result = handler(params, ctx) + # backstop for UUID transcription slips: "Did you mean ...?" + result = add_did_you_mean(result, params, run) return [{"type": "text", "text": json.dumps(result, ensure_ascii=False, default=str)}] @@ -351,7 +364,7 @@ async def project_list() -> list[dict[str, Any]]: @mcp.tool() async def project_get( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Get detailed information about a specific project.""" return await asyncio.to_thread(_run_handler_sync, get_project_handler, {"project_id": project_id}) @@ -368,7 +381,7 @@ async def project_create( @mcp.tool() async def project_delete( - project_id: Annotated[str, Field(description="UUID of the project to delete")], + project_id: Annotated[str, Field(description="Project name or UUID to delete")], ) -> list[dict[str, Any]]: """Delete a GNS3 project permanently.""" return await asyncio.to_thread(_run_handler_sync, delete_project_handler, {"project_id": project_id}) @@ -376,7 +389,7 @@ async def project_delete( @mcp.tool() async def project_open( - project_id: Annotated[str, Field(description="UUID of the project to open")], + project_id: Annotated[str, Field(description="Project name or UUID to open")], ) -> list[dict[str, Any]]: """Open a closed GNS3 project.""" return await asyncio.to_thread(_run_handler_sync, open_project_handler, {"project_id": project_id}) @@ -384,7 +397,7 @@ async def project_open( @mcp.tool() async def project_close( - project_id: Annotated[str, Field(description="UUID of the project to close")], + project_id: Annotated[str, Field(description="Project name or UUID to close")], ) -> list[dict[str, Any]]: """Close an open GNS3 project.""" return await asyncio.to_thread(_run_handler_sync, close_project_handler, {"project_id": project_id}) @@ -392,7 +405,7 @@ async def project_close( @mcp.tool() async def project_stats( - project_id: Annotated[str, Field(description="UUID of the project to get statistics for")], + project_id: Annotated[str, Field(description="Project name or UUID to get statistics for")], ) -> list[dict[str, Any]]: """Get statistics (nodes, links, snapshots, drawings) for a project.""" return await asyncio.to_thread(_run_handler_sync, get_project_stats_handler, {"project_id": project_id}) @@ -400,7 +413,7 @@ async def project_stats( @mcp.tool() async def project_update( - project_id: Annotated[str, Field(description="UUID of the project to update")], + project_id: Annotated[str, Field(description="Project name or UUID to update")], name: Annotated[str | None, Field(description="New project name")] = None, auto_close: Annotated[bool | None, Field(description="Close project when last client leaves")] = None, auto_open: Annotated[bool | None, Field(description="Project opens when GNS3 starts")] = None, @@ -442,7 +455,7 @@ async def project_update( @mcp.tool() async def project_duplicate( - project_id: Annotated[str, Field(description="UUID of the project to duplicate")], + project_id: Annotated[str, Field(description="Project name or UUID to duplicate")], name: Annotated[str, Field(description="New project name")], reset_mac_addresses: Annotated[bool, Field(description="Reset MAC addresses for this project")] = False, ) -> list[dict[str, Any]]: @@ -455,7 +468,7 @@ async def project_duplicate( @mcp.tool() async def project_readme_get( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Get the content of a project's README.md file — the project documentation (Markdown format).""" return await asyncio.to_thread(_run_handler_sync, get_project_readme_handler, {"project_id": project_id}) @@ -463,7 +476,7 @@ async def project_readme_get( @mcp.tool() async def project_readme_update( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], content: Annotated[str, Field(description="Content to write to README.md (Markdown format)")], ) -> list[dict[str, Any]]: """Update or create a project's README.md file — the project documentation (Markdown format).""" @@ -477,7 +490,7 @@ async def project_readme_update( @mcp.tool() async def node_list( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], fields: Annotated[ list[str] | None, Field( @@ -491,8 +504,8 @@ async def node_list( @mcp.tool() async def node_get( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID")], fields: Annotated[ list[str] | None, Field( @@ -514,8 +527,8 @@ async def node_get( @mcp.tool() async def node_start( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str | None, Field(description="Node UUID (single mode)")] = None, + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str | None, Field(description="Node name or UUID (single mode)")] = None, node_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — start multiple nodes in parallel') ] = None, @@ -531,8 +544,8 @@ async def node_start( @mcp.tool() async def node_stop( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str | None, Field(description="Node UUID (single mode)")] = None, + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str | None, Field(description="Node name or UUID (single mode)")] = None, node_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — stop multiple nodes in parallel') ] = None, @@ -548,8 +561,8 @@ async def node_stop( @mcp.tool() async def node_suspend( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str | None, Field(description="Node UUID (single mode)")] = None, + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str | None, Field(description="Node name or UUID (single mode)")] = None, node_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — suspend multiple nodes in parallel') ] = None, @@ -565,7 +578,7 @@ async def node_suspend( @mcp.tool() async def node_create( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], template_id: Annotated[ str | None, Field(description="Template UUID (required for single mode; used as default in batch mode)") ] = None, @@ -626,8 +639,8 @@ async def node_create( @mcp.tool() async def node_delete( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str | None, Field(description="Node UUID (single mode)")] = None, + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str | None, Field(description="Node name or UUID (single mode)")] = None, node_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — delete multiple nodes in parallel') ] = None, @@ -643,8 +656,8 @@ async def node_delete( @mcp.tool() async def node_update( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node to update")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID to update")], **kwargs: Any, ) -> list[dict[str, Any]]: """Update a node's properties (name, position, etc.).""" @@ -654,8 +667,8 @@ async def node_update( @mcp.tool() async def node_console( - project_id: Annotated[str, Field(description="UUID of the project containing the node")], - node_id: Annotated[str, Field(description="UUID of the node to get console info for")], + project_id: Annotated[str, Field(description="Project name or UUID containing the node")], + node_id: Annotated[str, Field(description="Node name or UUID to get console info for")], ) -> list[dict[str, Any]]: """Get WebSocket console connection info for a node. @@ -693,7 +706,7 @@ async def node_console( @mcp.tool() async def link_list( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], fields: Annotated[ list[str] | None, Field( @@ -707,7 +720,7 @@ async def link_list( @mcp.tool() async def link_get( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str, Field(description="UUID of the link")], ) -> list[dict[str, Any]]: """Get detailed information about a specific link.""" @@ -716,7 +729,7 @@ async def link_get( @mcp.tool() async def link_create( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], nodes: Annotated[ list | None, Field(description="Single mode: [{node_id, adapter_number, port_number}] or compact [id, ad, pt, id, ad, pt]"), @@ -762,7 +775,7 @@ async def link_create( @mcp.tool() async def link_delete( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str | None, Field(description="Link UUID (single mode)")] = None, link_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — delete multiple links in parallel') @@ -779,7 +792,7 @@ async def link_delete( @mcp.tool() async def link_update( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str, Field(description="UUID of the link to update")], **kwargs: Any, ) -> list[dict[str, Any]]: @@ -841,7 +854,7 @@ async def link_update( @mcp.tool() async def link_available_filters( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str, Field(description="UUID of the link")], ) -> list[dict[str, Any]]: """List the packet filter types available for a link with their parameters @@ -1002,8 +1015,8 @@ async def compute_images( @mcp.tool() async def node_file_list( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID")], path: Annotated[str, Field(description="Subdirectory path within node directory (optional)")] = "", recursive: Annotated[bool, Field(description="Recursively list all files (optional, default: false)")] = False, ) -> list[dict[str, Any]]: @@ -1026,8 +1039,8 @@ async def node_file_list( @mcp.tool() async def node_file_get( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID")], file_path: Annotated[str, Field(description="Path to the file within the node directory")], offset: Annotated[int, Field(description="Line offset to start reading from (optional, default: 0)")] = 0, limit: Annotated[int, Field(description="Maximum number of lines to return (optional, default: 200)")] = 200, @@ -1056,8 +1069,8 @@ async def node_file_get( @mcp.tool() async def node_file_write( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID")], file_path: Annotated[str, Field(description="Path to the file within the node directory")], content: Annotated[str, Field(description="Content to write to the file")], ) -> list[dict[str, Any]]: @@ -1076,8 +1089,8 @@ async def node_file_write( @mcp.tool() async def node_file_delete( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID")], file_path: Annotated[str, Field(description="Path to the file within the node directory")], ) -> list[dict[str, Any]]: """Delete a file from a node directory. Cannot be undone.""" @@ -1097,7 +1110,7 @@ async def node_file_delete( @mcp.tool() async def node_start_all( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Start all nodes in a project.""" return await asyncio.to_thread( @@ -1111,7 +1124,7 @@ async def node_start_all( @mcp.tool() async def node_stop_all( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Stop all nodes in a project.""" return await asyncio.to_thread( @@ -1125,7 +1138,7 @@ async def node_stop_all( @mcp.tool() async def node_suspend_all( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Suspend all nodes in a project.""" return await asyncio.to_thread( @@ -1139,8 +1152,8 @@ async def node_suspend_all( @mcp.tool() async def node_duplicate( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node to duplicate")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID to duplicate")], x: Annotated[int, Field(description="X coordinate for the new node")] = 0, y: Annotated[int, Field(description="Y coordinate for the new node")] = 0, z: Annotated[int, Field(description="Z layer for the new node")] = 0, @@ -1161,8 +1174,8 @@ async def node_duplicate( @mcp.tool() async def node_isolate( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node to isolate")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID to isolate")], ) -> list[dict[str, Any]]: """Isolate a node by suspending all its attached links (network isolation).""" return await asyncio.to_thread( @@ -1177,8 +1190,8 @@ async def node_isolate( @mcp.tool() async def node_unisolate( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node to unisolate")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID to unisolate")], ) -> list[dict[str, Any]]: """Un-isolate a node by resuming all its suspended links.""" return await asyncio.to_thread( @@ -1193,8 +1206,8 @@ async def node_unisolate( @mcp.tool() async def node_links( - project_id: Annotated[str, Field(description="UUID of the project")], - node_id: Annotated[str, Field(description="UUID of the node")], + project_id: Annotated[str, Field(description="Project name or UUID")], + node_id: Annotated[str, Field(description="Node name or UUID")], ) -> list[dict[str, Any]]: """List all links connected to a specific node.""" return await asyncio.to_thread( @@ -1212,7 +1225,7 @@ async def node_links( @mcp.tool() async def link_reset( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str | None, Field(description="Link UUID (single mode)")] = None, link_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — reset multiple links in parallel') @@ -1239,7 +1252,7 @@ async def link_reset( @mcp.tool() async def link_capture_start( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str | None, Field(description="Link UUID (single mode)")] = None, data_link_type: Annotated[str, Field(description="Data link type (default: DLT_EN10MB)")] = "DLT_EN10MB", capture_file_name: Annotated[str | None, Field(description="Capture file name (optional)")] = None, @@ -1265,7 +1278,7 @@ async def link_capture_start( @mcp.tool() async def link_capture_stop( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str | None, Field(description="Link UUID (single mode)")] = None, link_ids: Annotated[ list[str] | None, @@ -1283,7 +1296,7 @@ async def link_capture_stop( @mcp.tool() async def link_capture_download( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str | None, Field(description="Link UUID (single mode)")] = None, link_ids: Annotated[ list[str] | None, Field(description='Batch mode: ["uuid1","uuid2"] — get download URLs for multiple captures') @@ -1314,7 +1327,7 @@ async def link_capture_download( @mcp.tool() async def link_marker( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], link_id: Annotated[str, Field(description="UUID of the link")], action: Annotated[str, Field(description="Action: create, update, or delete")], bpf: Annotated[ @@ -1382,7 +1395,7 @@ async def link_marker( @mcp.tool() async def marker_definition( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], action: Annotated[str, Field(description="Action: create, update, delete, or list")], bpf: Annotated[ str | None, Field(description="BPF expression, e.g. 'arp', 'ospf', 'tcp port 22' (required for create)") @@ -1433,7 +1446,7 @@ async def marker_definition( @mcp.tool() async def snapshot_list( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """List all snapshots of a project.""" return await asyncio.to_thread( @@ -1447,7 +1460,7 @@ async def snapshot_list( @mcp.tool() async def snapshot_create( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], name: Annotated[str, Field(description="Name for the new snapshot")], ) -> list[dict[str, Any]]: """Create a new snapshot of a project. @@ -1468,7 +1481,7 @@ async def snapshot_create( @mcp.tool() async def snapshot_delete( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], snapshot_id: Annotated[str, Field(description="UUID of the snapshot to delete")], ) -> list[dict[str, Any]]: """Delete a snapshot from a project. Cannot be undone.""" @@ -1484,7 +1497,7 @@ async def snapshot_delete( @mcp.tool() async def snapshot_restore( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], snapshot_id: Annotated[str, Field(description="UUID of the snapshot to restore")], ) -> list[dict[str, Any]]: """Restore a project to a previous snapshot state. The project may be closed and reopened.""" @@ -1503,7 +1516,7 @@ async def snapshot_restore( @mcp.tool() async def drawing_list( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """List all drawings (labels, shapes, images) on a project canvas.""" return await asyncio.to_thread( @@ -1517,7 +1530,7 @@ async def drawing_list( @mcp.tool() async def drawing_create( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], svg: Annotated[str, Field(description="SVG content for the drawing")], x: Annotated[int, Field(description="X coordinate (default: 0)")] = 0, y: Annotated[int, Field(description="Y coordinate (default: 0)")] = 0, @@ -1557,7 +1570,7 @@ async def drawing_create( @mcp.tool() async def drawing_get( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], drawing_id: Annotated[str, Field(description="UUID of the drawing")], ) -> list[dict[str, Any]]: """Get detailed information about a specific drawing.""" @@ -1573,7 +1586,7 @@ async def drawing_get( @mcp.tool() async def drawing_update( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], drawing_id: Annotated[str, Field(description="UUID of the drawing")], svg: Annotated[str | None, Field(description="New SVG content")] = None, locked: Annotated[bool | None, Field(description="Lock or unlock the drawing")] = None, @@ -1593,7 +1606,7 @@ async def drawing_update( @mcp.tool() async def drawing_delete( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], drawing_id: Annotated[str, Field(description="UUID of the drawing to delete")], ) -> list[dict[str, Any]]: """Delete a drawing from a project canvas. Cannot be undone.""" @@ -1612,7 +1625,7 @@ async def drawing_delete( @mcp.tool() async def project_lock( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Lock all drawings and nodes in a project to prevent accidental changes.""" return await asyncio.to_thread( @@ -1626,7 +1639,7 @@ async def project_lock( @mcp.tool() async def project_unlock( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Unlock a project to allow editing of drawings and nodes.""" return await asyncio.to_thread( @@ -1640,7 +1653,7 @@ async def project_unlock( @mcp.tool() async def project_locked( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], ) -> list[dict[str, Any]]: """Check whether a project is locked (preventing edits to drawings and nodes).""" return await asyncio.to_thread( @@ -1868,7 +1881,7 @@ async def image_install() -> list[dict[str, Any]]: @mcp.tool() async def device_config_send( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], device_configs: Annotated[ list, Field( @@ -1905,7 +1918,7 @@ async def device_config_send( @mcp.tool() async def device_show_run( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], device_configs: Annotated[ list, Field( @@ -1947,7 +1960,7 @@ async def device_show_run( @mcp.tool() async def vpcs_config_set( - project_id: Annotated[str, Field(description="UUID of the project")], + project_id: Annotated[str, Field(description="Project name or UUID")], device_configs: Annotated[ list, Field( diff --git a/gns3server/agent/mcp/locators.py b/gns3server/agent/mcp/locators.py new file mode 100644 index 000000000..4bffb8014 --- /dev/null +++ b/gns3server/agent/mcp/locators.py @@ -0,0 +1,223 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# Author: Yue Guobin +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Name locators for MCP tools. + +LLM agents transcribe UUIDs unreliably (a 2-character slip invents an id +that never existed and is invisible to the eye), so MCP tool parameters +accept names wherever the server guarantees uniqueness: project names are +unique among live projects (the controller rejects duplicates) and node +names are unique within a project (auto-suffixed on collision). Links, +drawings, appliances and snapshots have no names and stay UUID-only. + +Resolution happens in the single dispatch choke point (_run_handler_sync): +UUID-shaped values pass through untouched, names are resolved with one +list call, and a name that matches nothing fails fast with the known +names instead of a confusing downstream 404. As a backstop for residual +UUID typos, error replies carrying " doesn't exist" are matched +against the ids visible in scope; a single candidate within edit +distance 2 earns a "Did you mean ..." hint. +""" + +import re +from typing import Any, Callable + +from .projects import list_projects_handler +from gns3server.agent.gns3_copilot.gns3_client.api_handlers import get_links_handler, get_nodes_handler + +_UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") +_ID_IN_TEXT_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") + +# how many names to list when a locator matches nothing +_MAX_HINT_NAMES = 15 + + +def looks_like_uuid(value: Any) -> bool: + """ + Whether *value* is a UUID-shaped string (anything else is treated as a name). + """ + + return isinstance(value, str) and bool(_UUID_RE.match(value)) + + +def resolve_locators( + params: dict[str, Any], run: Callable[[Callable, dict], Any] +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """ + Rewrite name-shaped ``project_id``/``node_id`` values into UUIDs. + + :param params: tool params (top-level keys only; nested structures are + left alone — link_create's node tuples require ids) + :param run: callback running (handler, params) through the same context + :returns: (resolved params, None) on success, (None, error dict) on failure + """ + + params = dict(params) + + project_ref = params.get("project_id") + if project_ref is not None and not looks_like_uuid(project_ref): + result = run(list_projects_handler, {}) + projects = _as_items(result, "projects") + if projects is None: + return None, {"error": f"Could not resolve project '{project_ref}': {_as_error(result)}"} + match = _match_by_name(projects, project_ref) + if match is None: + return None, {"error": f"Project '{project_ref}' not found. Known projects: {_known_names(projects)}"} + params["project_id"] = match["project_id"] + + node_ref = params.get("node_id") + if node_ref is not None and not looks_like_uuid(node_ref) and params.get("project_id"): + result = run(get_nodes_handler, {"project_id": params["project_id"]}) + nodes = _as_items(result, "nodes") + if nodes is None: + return None, {"error": f"Could not resolve node '{node_ref}': {_as_error(result)}"} + match = _match_by_name(nodes, node_ref) + if match is None: + return None, {"error": f"Node '{node_ref}' not found. Known nodes: {_known_names(nodes)}"} + params["node_id"] = match["node_id"] + + return params, None + + +def add_did_you_mean( + result: Any, params: dict[str, Any], run: Callable[[Callable, dict], Any] +) -> Any: + """ + Append a "Did you mean ...?" hint to an error reply that names a + non-existent UUID, when exactly one in-scope id is within edit + distance 2 of it (the signature of a transcription slip). + """ + + if not isinstance(result, dict): + return result + error = result.get("error") + if not isinstance(error, str) or ("doesn't exist" not in error and "not found" not in error.lower()): + return result + bad_ids = set(_ID_IN_TEXT_RE.findall(error)) + if not bad_ids: + return result + + candidates = _collect_candidates(params, run) + if not candidates: + return result + + hints = [] + for bad in bad_ids: + close = [(cid, label) for cid, label in candidates if bad != cid and _edit_distance_at_most(bad.lower(), cid.lower(), 2)] + if len(close) == 1: + cid, label = close[0] + hints.append(f"Did you mean {label} ({cid})?") + if not hints: + return result + result = dict(result) + result["error"] = error.rstrip() + " " + " ".join(hints) + return result + + +def _as_items(result: Any, key: str) -> list | None: + """ + Normalize a handler reply into a list of dicts (None = failure). + """ + + if isinstance(result, list): + return result + if isinstance(result, dict): + items = result.get(key) + if isinstance(items, list): + return items + return None + + +def _as_error(result: Any) -> Any: + return result.get("error", result) if isinstance(result, dict) else result + + +def _match_by_name(items: list, name: str) -> dict | None: + """ + The unique item whose name equals *name* (exact first, then a unique + case-insensitive match); None when absent or ambiguous. + """ + + exact = [item for item in items if item.get("name") == name] + if len(exact) == 1: + return exact[0] + if not exact: + folded = [item for item in items if isinstance(item.get("name"), str) and item["name"].lower() == name.lower()] + if len(folded) == 1: + return folded[0] + return None + + +def _known_names(items: list) -> str: + names = sorted(str(item.get("name", "?")) for item in items) + if len(names) > _MAX_HINT_NAMES: + names = [*names[:_MAX_HINT_NAMES], f"... ({len(names) - _MAX_HINT_NAMES} more)"] + return ", ".join(names) + + +def _collect_candidates(params: dict[str, Any], run: Callable[[Callable, dict], Any]) -> list[tuple[str, str]]: + """ + (id, label) pairs in scope for a hint: every project, plus the nodes + and links of the project the call targeted. Best-effort — any failure + just yields what was gathered so far. + """ + + candidates = [] + try: + projects = _as_items(run(list_projects_handler, {}), "projects") or [] + candidates.extend((p["project_id"], f"project '{p.get('name', '?')}'") for p in projects if p.get("project_id")) + except Exception: # best-effort hint enrichment + pass + project_id = params.get("project_id") + if looks_like_uuid(project_id): + try: + nodes = _as_items(run(get_nodes_handler, {"project_id": project_id}), "nodes") or [] + candidates.extend((n["node_id"], f"node '{n.get('name', '?')}'") for n in nodes if n.get("node_id")) + except Exception: # best-effort hint enrichment + pass + try: + links = _as_items(run(get_links_handler, {"project_id": project_id}), "links") or [] + candidates.extend((link["link_id"], "link") for link in links if link.get("link_id")) + except Exception: # best-effort hint enrichment + pass + return candidates + + +def _edit_distance_at_most(a: str, b: str, cap: int) -> bool: + """ + Levenshtein distance with an early bail above *cap* (ids are short; + the cap keeps the DP band trivial). + """ + + if abs(len(a) - len(b)) > cap: + return False + if a == b: + return True + previous = list(range(len(b) + 1)) + for i, char_a in enumerate(a, 1): + current = [i] + best = i + for j, char_b in enumerate(b, 1): + substitution = previous[j - 1] + (char_a != char_b) + value = min(previous[j] + 1, current[j - 1] + 1, substitution) + current.append(value) + best = min(best, value) + if best > cap: + return False + previous = current + return previous[-1] <= cap diff --git a/tests/agent/mcp/test_locators.py b/tests/agent/mcp/test_locators.py new file mode 100644 index 000000000..ee4bd1cee --- /dev/null +++ b/tests/agent/mcp/test_locators.py @@ -0,0 +1,170 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Name locators: name->UUID resolution and the UUID-typo "Did you mean" hint. +""" + +from gns3server.agent.mcp import locators +from gns3server.agent.mcp.locators import add_did_you_mean, looks_like_uuid, resolve_locators + +PROJECT_ID = "d1ab2349-2e53-4e41-a9ae-b5ffcd912408" +NODE_ID = "b78a2630-2c9e-4b43-9397-c855c8053d1a" +NODE_ID_TYPO = "b78a2630-3b9e-4b43-9397-c855c8053d1a" # 2c9e -> 3b9e (distance 2) + + +def _run_factory(projects=None, nodes=None): + """A run() callback answering the two list handlers from canned data.""" + + calls = [] + + def run(handler, params): + calls.append((handler, dict(params))) + if handler is locators.list_projects_handler: + if isinstance(projects, Exception): + raise projects + return {"projects": projects if projects is not None else []} + if handler is locators.get_nodes_handler: + if isinstance(nodes, Exception): + raise nodes + return nodes if nodes is not None else [] + raise AssertionError(f"unexpected handler {handler}") + + return run, calls + + +def test_looks_like_uuid(): + + assert looks_like_uuid(PROJECT_ID) is True + assert looks_like_uuid(PROJECT_ID.upper()) is True + assert looks_like_uuid("FRRDocker-4") is False + assert looks_like_uuid("") is False + assert looks_like_uuid(None) is False + assert looks_like_uuid(1234) is False + + +def test_uuid_params_pass_through_without_calls(): + """UUID-shaped values never trigger resolution lookups.""" + + run, calls = _run_factory(projects=[{"project_id": PROJECT_ID, "name": "lab"}]) + params, error = resolve_locators({"project_id": PROJECT_ID, "node_id": NODE_ID}, run) + assert error is None + assert params == {"project_id": PROJECT_ID, "node_id": NODE_ID} + assert calls == [] + + +def test_project_name_resolved(): + """A name-shaped project_id becomes the project's UUID.""" + + run, calls = _run_factory(projects=[{"project_id": "11111111-1111-1111-1111-111111111111", "name": "other"}, + {"project_id": PROJECT_ID, "name": "my-lab"}]) + params, error = resolve_locators({"project_id": "my-lab"}, run) + assert error is None + assert params["project_id"] == PROJECT_ID + assert len(calls) == 1 # only the project list + + +def test_project_name_not_found_lists_known_names(): + + run, _calls = _run_factory(projects=[{"project_id": PROJECT_ID, "name": "my-lab"}]) + params, error = resolve_locators({"project_id": "no-such-lab"}, run) + assert params is None + assert "not found" in error["error"] + assert "my-lab" in error["error"] + + +def test_node_name_resolved_within_project(): + + run, calls = _run_factory(projects=[{"project_id": PROJECT_ID, "name": "my-lab"}], + nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}]) + params, error = resolve_locators({"project_id": "my-lab", "node_id": "FRRDocker-4"}, run) + assert error is None + assert params["project_id"] == PROJECT_ID + assert params["node_id"] == NODE_ID + # node lookup went to the already-resolved project + assert calls[1] == (locators.get_nodes_handler, {"project_id": PROJECT_ID}) + + +def test_node_name_case_insensitive_unique_match(): + + run, _calls = _run_factory(nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}]) + params, error = resolve_locators({"project_id": PROJECT_ID, "node_id": "frrdocker-4"}, run) + assert error is None + assert params["node_id"] == NODE_ID + + +def test_node_name_not_found_lists_known_nodes(): + + run, _calls = _run_factory(nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}]) + params, error = resolve_locators({"project_id": PROJECT_ID, "node_id": "vpcs-1"}, run) + assert params is None + assert "FRRDocker-4" in error["error"] + + +def test_handler_failure_during_resolution_is_reported(): + + run, _calls = _run_factory(nodes={"error": "boom"}) + params, error = resolve_locators({"project_id": PROJECT_ID, "node_id": "FRRDocker-4"}, run) + assert params is None + assert "Could not resolve node" in error["error"] + + +def test_did_you_mean_appended_for_transcription_slip(): + """The 2c9e -> 3b9e class of typo earns exactly one hint.""" + + run, _calls = _run_factory(projects=[{"project_id": PROJECT_ID, "name": "my-lab"}], + nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}]) + result = add_did_you_mean( + {"error": f"Node ID {NODE_ID_TYPO} doesn't exist"}, {"project_id": PROJECT_ID}, run + ) + assert f"Did you mean node 'FRRDocker-4' ({NODE_ID})" in result["error"] + # original message preserved + assert NODE_ID_TYPO in result["error"] + + +def test_no_hint_when_too_far_or_ambiguous(): + + far = "b78a2630-9999-4b43-9397-c855c8053d1a" # distance 3 + run, _calls = _run_factory(nodes=[{"node_id": NODE_ID, "name": "n1"}]) + result = add_did_you_mean({"error": f"Node ID {far} doesn't exist"}, {"project_id": PROJECT_ID}, run) + assert "Did you mean" not in result["error"] + + two = [NODE_ID, "b78a2630-2b9e-4b43-9397-c855c8053d1a"] # both within distance 2 of the typo + run2, _calls2 = _run_factory(nodes=[{"node_id": i, "name": f"n{i}"} for i in two]) + result2 = add_did_you_mean( + {"error": f"Node ID {NODE_ID_TYPO} doesn't exist"}, {"project_id": PROJECT_ID}, run2 + ) + assert "Did you mean" not in result2["error"] + + +def test_non_error_results_untouched(): + + run, _calls = _run_factory(nodes=[{"node_id": NODE_ID, "name": "n1"}]) + payload = [{"node_id": NODE_ID}] + assert add_did_you_mean(payload, {"project_id": PROJECT_ID}, run) is payload + assert add_did_you_mean({"filters": {}}, {"project_id": PROJECT_ID}, run) == {"filters": {}} + # errors without an id carry nothing to hint against + result = add_did_you_mean({"error": "Node not found: no id here"}, {"project_id": PROJECT_ID}, run) + assert result == {"error": "Node not found: no id here"} + + +def test_edit_distance_at_most(): + + assert locators._edit_distance_at_most("abcd", "abcd", 2) + assert locators._edit_distance_at_most("abcd", "abce", 2) + assert locators._edit_distance_at_most("2c9e", "3b9e", 2) + assert not locators._edit_distance_at_most("abcd", "dcba", 2) + assert not locators._edit_distance_at_most("abc", "abcdef", 2) From 1db9f171ca461201a010e913ee12ce638a42380f Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 17:24:20 +0800 Subject: [PATCH 019/101] fix(controller): re-arm window_drop and expose kernel_datapath Three fixes from live-calibrating the kernel impairment filters on an isolated instance (Docker-to-Docker veth link, per-sequence evidence): - update_filters skipped an unchanged filters dict as a no-op, so a one-shot window_drop outage that had already passed could never be re-armed: re-applying the same values produced no datapath command at all. A PUT still carrying window_drop now reconciles, restarting the schedule from the apply as documented. Measured: identical re-apply moves the outage window from seq 50-70 to 70-91 (arm at +8s of the new apply). - gemodel parameter semantics were documented backwards in the validation rules, available_filters, the MCP/copilot descriptions and the feature docs. Measured against the kernel (per-direction counters, offered = sent + dropped): p is the good-to-bad transition chance, r the bad-to-good transition, 1-h the loss chance in the bad state (the good state loses nothing) - mean one-way loss p/(p+r)*(1-h), e.g. [10,30,100] -> 24.8%/25.7% measured vs 25% predicted, [5,45,100] -> 10.8% both ways vs 10%. - kernel_datapath was silently stripped from every REST/MCP link payload by the response model (LinkBase carried no such field), so clients could not tell a kernel link from a relay one. Also verified on the same setup: the 3-argument window_drop form works exactly as specified ([3000,5000,100] -> a single seq 31-55 outage, [20000,6000,100] -> seq 115-144), and both generations of the command builder emit the parameters unpadded. Adds an API test for the field and controller tests for the no-op skip and the window_drop re-arm exemption. --- docs/features/docker-kernel-datapath.md | 13 ++++- .../tools_v2/gns3_packet_filter.py | 3 +- gns3server/agent/mcp/__init__.py | 10 +++- gns3server/controller/link.py | 19 ++++--- gns3server/schemas/controller/links.py | 5 ++ gns3server/utils/packet_filter_validation.py | 11 ++-- tests/api/routes/controller/test_links.py | 27 ++++++++++ tests/controller/test_link.py | 52 +++++++++++++++++++ tests/utils/test_packet_filter_validation.py | 2 +- 9 files changed, 126 insertions(+), 16 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index d560bee31..c1405a8c2 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -138,6 +138,11 @@ attach to running containers: `brctl addif` (kernel) or On project (re)open every link re-runs `_prepare`, so relay links whose endpoints became eligible are upgraded to the kernel datapath automatically. +Which datapath a link ended up on is visible in its REST payload (and the +`link_list` / `link_get` MCP tools) as the read-only boolean +`kernel_datapath`, recomputed from the NIO wiring — a relay link upgraded on +reopen flips the field. It is not persisted and never sent on create/update. + ## Adapter lifecycle * **Birth (container start)** — `_create_veth`: stale sweep of any leftover @@ -209,7 +214,7 @@ ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): | `duplicate [%, correl]` | netem `dup P [correl C]` | kernel-only type | | `rate ["512kbit"]` | netem `rate B` | tc-style integer+unit (bit/kbit/mbit/gbit/bps/kbps/mbps, ≤100gbit); kernel-only type | | `reorder [%, correl, gap]` | netem `reorder P [correl C] [gap G]` | requires `delay`; kernel-only type | -| `gemodel [p, r, 1-h]` | netem `loss gemodel P R H` | Gilbert-Elliot bursty loss; mutually exclusive with `packet_loss`; kernel-only type | +| `gemodel [p, r, 1-h]` | netem `loss gemodel P R H` | Gilbert-Elliot bursty loss: p = good→bad transition, r = bad→good transition, 1-h = loss chance in the bad state (good loses nothing) — mean one-way loss p/(p+r)×(1-h); mutually exclusive with `packet_loss`; kernel-only type | | `seed [u32]` | netem `seed N` | reproducible random draws; kernel-only type | | `limit [pkts]` | netem `limit N` | queue depth above the 1000 default (rate+delay BDP); kernel-only type | | `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | @@ -274,7 +279,11 @@ Semantics: controller — a round trip survives only when **both** directions are outside their windows. Note `start = 0` (outage begins immediately) is a valid, active configuration — unlike other filters, a zero first - parameter does not mean "disabled". + parameter does not mean "disabled". A PUT re-sending an unchanged filters + dict is normally a controller no-op, but one still carrying `window_drop` + is reconciled anyway, so re-applying the same window re-arms a one-shot + outage that has already passed (arm the filter immediately before + generating traffic, or use a period for a repeatable event). * **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + diff --git a/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py b/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py index e6aee5583..b643a26f0 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py +++ b/gns3server/agent/gns3_copilot/tools_v2/gns3_packet_filter.py @@ -130,7 +130,8 @@ class GNS3PacketFilterTool(BaseTool): - "rate": Bandwidth cap, tc-style string like "512kbit" (kernel links only) - "reorder": Packet reordering (parameters: percentage, correlation, gap; needs delay; kernel links only) - - "gemodel": Gilbert-Elliot bursty loss (parameters: p, r, 1-h; kernel links only) + - "gemodel": Gilbert-Elliot bursty loss (parameters: p good-to-bad transition, r bad-to-good + transition, 1-h loss in the bad state; mean loss = p/(p+r)*(1-h); kernel links only) - "duplicate": Packet duplication percentage (kernel links only) - "seed": Reproducible netem random draws (kernel links only) - "limit": Impairment queue depth in packets (kernel links only) diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index 180087905..1c5398233 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -817,7 +817,10 @@ async def link_update( * rate (K): ["512kbit"] - Bandwidth cap, tc-style int+unit (bit/kbit/ mbit/gbit/bps/kbps/mbps, max 100gbit) * reorder (K): [pct, correl?, gap?] - Requires delay to also be set - * gemodel (K): [p, r, 1-h] - Gilbert-Elliot bursty loss; mutually + * gemodel (K): [p, r, 1-h] - Gilbert-Elliot bursty loss; p = chance + of moving from the good to the bad state, r = chance of moving back, + 1-h = loss chance in the bad state (the good state loses nothing); + steady-state mean loss = p/(p+r)*(1-h) per direction; mutually exclusive with packet_loss * duplicate (K): [pct, correl?] - Duplication percentage * seed (K): [u32] - Makes netem random draws reproducible @@ -830,7 +833,10 @@ async def link_update( outage (traffic passes before and after), 4 args = recurring flap (period >= outage), 5 args = per-cycle timing randomized within +/- jitter. start is measured from the moment this filter is applied and - ANY filter update / node restart restarts the schedule; start 0 = + ANY filter update (including re-sending the same values) or node + restart restarts the schedule — a one-shot window that already + passed will not fire again until re-applied, so arm the filter + immediately before generating traffic (or use a period); start 0 = outage begins immediately (active, not disabled) Example filters: diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 337df9e29..b531d91ec 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -105,12 +105,14 @@ FILTERS = [ { "type": "gemodel", "name": "Gilbert-Elliot loss", - "description": "Bursty loss model: p is the loss chance in the bad state, r in the good state, 1-h the " - "chance of moving from good to bad. Mutually exclusive with packet loss; kernel-datapath links only", + "description": "Bursty loss model: p is the chance of moving from the good to the bad state, r the chance " + "of moving back, and 1-h the loss chance while in the bad state (the good state loses nothing) — the " + "steady-state mean loss is p/(p+r) x (1-h). Mutually exclusive with packet loss; kernel-datapath links " + "only", "parameters": [ - {"name": "p (bad-state loss)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, - {"name": "r (good-state loss)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, - {"name": "1-h (good-to-bad)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "p (good-to-bad)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "r (bad-to-good)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, + {"name": "1-h (bad-state loss)", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, ], }, { @@ -338,7 +340,12 @@ class Link: ) ) - if new_filters != self.filters: + # An unchanged filters dict is normally a no-op skip, but a PUT that + # still carries window_drop must reconcile: the outage schedule is + # measured from the apply, so re-sending the same window has to + # re-arm the one-shot outage (the documented contract is that any + # filter update restarts the schedule). + if new_filters != self.filters or "window_drop" in new_filters: self._filters = new_filters if self._created: await self.update() diff --git a/gns3server/schemas/controller/links.py b/gns3server/schemas/controller/links.py index 5dd8f1752..dccc33063 100644 --- a/gns3server/schemas/controller/links.py +++ b/gns3server/schemas/controller/links.py @@ -81,6 +81,11 @@ class Link(LinkBase): link_id: UUID project_id: Optional[UUID] = None link_type: Optional[LinkType] = None + kernel_datapath: Optional[bool] = Field( + None, + description="Read only property. True when this link is wired through kernel veth/bridge interfaces " + "(currently Docker-to-Docker on one compute) instead of the uBridge UDP relay", + ) capturing: Optional[bool] = Field(None, description="Read only property. True if a capture running on the link") capture_file_name: Optional[str] = Field( None, description="Read only property. The name of the capture file if a capture is running" diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index 43f6a3fb2..c9c3a6d96 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -210,12 +210,15 @@ def validate_filter_parameters(filter_type: str, values: List[Any]) -> None: "units": ["%", "%", "packets"], }, "gemodel": { - # Gilbert-Elliot loss model, tc semantics: p = loss probability in - # the bad state, r = loss probability in the good state, 1-h = - # probability of moving from the good to the bad state. + # Gilbert-Elliot loss model, tc semantics (verified against the + # kernel on kernel-datapath links): p = probability of moving + # from the good to the bad state (PGAP), r = probability of + # moving back (PGBP), 1-h = loss probability in the bad state + # (the good state loses nothing). Steady-state mean loss is + # p/(p+r) * (1-h) per direction. "params_count": (1, 3), "ranges": [(0, 100), (0, 100), (0, 100)], - "names": ["p (bad-state loss)", "r (good-state loss)", "1-h (good-to-bad transition)"], + "names": ["p (good-to-bad transition)", "r (bad-to-good transition)", "1-h (bad-state loss)"], "units": ["%", "%", "%"], }, "seed": {"params_count": (1, 1), "ranges": [(0, 4294967295)], "names": ["Seed"], "units": [""]}, diff --git a/tests/api/routes/controller/test_links.py b/tests/api/routes/controller/test_links.py index 518eb22f9..08c73a8f7 100644 --- a/tests/api/routes/controller/test_links.py +++ b/tests/api/routes/controller/test_links.py @@ -139,9 +139,36 @@ class TestLinkRoutes: assert mock.called link_id = response.json()["link_id"] assert response.json()["nodes"][0]["label"]["x"] == 42 + # the runtime datapath fact survives the response model on both POST and GET + assert response.json()["kernel_datapath"] is False response = await client.get(app.url_path_for("get_link", project_id=project.id, link_id=link_id)) assert response.status_code == status.HTTP_200_OK assert response.json()["nodes"][0]["label"]["x"] == 42 + assert response.json()["kernel_datapath"] is False + + async def test_get_link_kernel_datapath( + self, app: FastAPI, client: AsyncClient, project: Project, nodes: Tuple[Node, Node] + ) -> None: + """A kernel-wired link reports kernel_datapath true (was dropped by the schema).""" + + node1, node2 = nodes + with asyncio_patch("gns3server.controller.udp_link.UDPLink.create"): + response = await client.post( + app.url_path_for("create_link", project_id=project.id), + json={ + "nodes": [ + {"node_id": node1.id, "adapter_number": 0, "port_number": 3}, + {"node_id": node2.id, "adapter_number": 2, "port_number": 4}, + ] + }, + ) + + link_id = response.json()["link_id"] + link = project.get_link(link_id) + link._link_data = [{"type": "nio_bridge"}] + response = await client.get(app.url_path_for("get_link", project_id=project.id, link_id=link_id)) + assert response.status_code == status.HTTP_200_OK + assert response.json()["kernel_datapath"] is True async def test_update_link_suspend( self, app: FastAPI, client: AsyncClient, project: Project, nodes: Tuple[Node, Node] diff --git a/tests/controller/test_link.py b/tests/controller/test_link.py index 66a94e708..186e4c332 100644 --- a/tests/controller/test_link.py +++ b/tests/controller/test_link.py @@ -361,6 +361,58 @@ async def test_update_filters(project, compute): assert link.update.called +@pytest.mark.asyncio +async def test_update_filters_unchanged_is_noop(project, compute): + """Re-sending an identical filters dict skips the datapath reconcile.""" + + node1 = Node(project, compute, "node1", node_type="qemu") + node1._ports = [EthernetPort("E0", 0, 0, 4)] + node2 = Node(project, compute, "node2", node_type="qemu") + node2._ports = [EthernetPort("E0", 0, 0, 4)] + + link = Link(project) + link.create = AsyncioMagicMock() + link._project.emit_notification = MagicMock() + project.dump = AsyncioMagicMock() + await link.add_node(node1, 0, 4) + await link.add_node(node2, 0, 4) + + link.update = AsyncioMagicMock() + await link.update_filters({"packet_loss": [10]}) + assert link.update.call_count == 1 + await link.update_filters({"packet_loss": [10]}) + assert link.update.call_count == 1 + + +@pytest.mark.asyncio +async def test_update_filters_unchanged_window_drop_still_reconciles(project, compute): + """ + window_drop's outage schedule is measured from the apply, so an + unchanged re-send must re-arm it instead of being skipped as a no-op. + """ + + node1 = Node(project, compute, "node1", node_type="qemu") + node1._ports = [EthernetPort("E0", 0, 0, 4)] + node2 = Node(project, compute, "node2", node_type="qemu") + node2._ports = [EthernetPort("E0", 0, 0, 4)] + + link = Link(project) + link.create = AsyncioMagicMock() + link._project.emit_notification = MagicMock() + project.dump = AsyncioMagicMock() + await link.add_node(node1, 0, 4) + await link.add_node(node2, 0, 4) + # window_drop is kernel-only: this synthetic link has no real wiring, + # so declare the kernel datapath the way _link_data does at runtime + link._link_data = [{"type": "nio_bridge"}] + + link.update = AsyncioMagicMock() + await link.update_filters({"window_drop": [3000, 5000, 100]}) + assert link.update.call_count == 1 + await link.update_filters({"window_drop": [3000, 5000, 100]}) + assert link.update.call_count == 2 + + @pytest.mark.asyncio async def test_available_filters(project, compute): diff --git a/tests/utils/test_packet_filter_validation.py b/tests/utils/test_packet_filter_validation.py index d196e9d3e..2bada6f13 100644 --- a/tests/utils/test_packet_filter_validation.py +++ b/tests/utils/test_packet_filter_validation.py @@ -308,7 +308,7 @@ class TestNetemExtensionFilters: validate_all_filters({"gemodel": [100, 0, 30], "packet_loss": [10]}) def test_gemodel_invalid_range(self): - with pytest.raises(FilterValidationError, match="bad-state"): + with pytest.raises(FilterValidationError, match="good-to-bad"): validate_filter_parameters("gemodel", [101, 0, 30]) def test_duplicate_valid(self): From c1d0e6ee218eb0eb319b6cd155149dd16f6061a3 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 17:24:58 +0800 Subject: [PATCH 020/101] docs(spec): define the gemodel acceptance target The F section asked for "gemodel loss within +/-5% of target" without saying what the target is, and the A.1 table gave ranges only. Spell out the semantics the kernel implements (and that the veth calibration confirmed): p is the good-to-bad transition, r the bad-to-good transition, 1-h the loss chance inside the bad state (the good state loses nothing), mean loss p/(p+r)*(1-h) per direction. The functional bullet now names that target and how to measure it (per-direction counters, round-trip reads 1-(1-L)^2, pre-warm ARP when 1-h=100). No wire-format or grammar change. --- docs/design/ubridge-kernel-impairment-spec.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/design/ubridge-kernel-impairment-spec.md b/docs/design/ubridge-kernel-impairment-spec.md index 2cfa565c4..94446b8e6 100644 --- a/docs/design/ubridge-kernel-impairment-spec.md +++ b/docs/design/ubridge-kernel-impairment-spec.md @@ -115,6 +115,14 @@ tc netem set Max token count for the command table rises accordingly (recommend 32). +`loss gemodel` semantics (mirrors the kernel, verified against a veth link +with per-direction qdisc counters): `p` is the good→bad state transition +probability (PGAP), `r` the bad→good transition (PGBP), and `1-h` the loss +probability **inside the bad state** (the good state loses nothing). The +steady-state mean loss is `p/(p+r) * (1-h)` per direction — e.g. +`loss gemodel 10 30 100` drops 25 % of the offered packets in each +direction, `loss gemodel 5 45 100` drops 10 %. + ### A.2 Netlink encoding notes * Correlation: loss/dup/reorder use their existing nested structs with the @@ -317,7 +325,11 @@ treatment; an incompatible mode revision renames its token * **Unit**: per keyword → expected netlink attrs (assert message bytes); every 204 table row; `bpf_drop` compile-fail path; capabilities string. * **Functional** (root, veth pair + netns + nsenter ping): - delay+reorder observable (mdev), gemodel loss within ±5% of target, + delay+reorder observable (mdev), gemodel loss within ±5% of + `p/(p+r) * (1-h)` measured **per direction** — a round-trip ping figure + reads `1-(1-L)²` instead, and with `1-h=100` the ARP cache must be + pre-warmed or the neighbour-resolution blackhole during a long burst + inflates the observed loss, rate within ±10% (measured byte throughput), nth_drop **exact** pattern (ICMP seq survives 1,2,…,N-1, Nth dropped), quota/window/flow with crafted senders, `tc reset` idempotent on a clean interface, P5 surface From 0b507773b9418cb227350f92da0f299f815bf397 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 18:06:15 +0800 Subject: [PATCH 021/101] docs(spec): require L2-only host anchors in uBridge Every host-side device uBridge creates for the data plane (persistent TAP anchors, veth host ends, per-link kernel bridges) must be pure L2: no IPv4 address, no IPv6 address and no IPv6 stack activity on that device. Measured today: an idle TAP emits ~6 frames / 2 s of the host's own MLD/DAD from its own link-local, the production Docker veth host end carries fe80::5c30:e8ff:fe00:5d8d/64 and the per-link bridge fe80::f0d9:94ff:feac:e42b/64. On a kernel link the bridge floods those frames into the emulated segment, on a relay link uBridge relays them to the peer, and the host answers neighbor solicitations for its own address (a phantom IPv6 neighbour for the emulated nodes). IPv4 needs no equivalent knob: nothing self-provisions it, and the anchors hold no IPv4 address. The spec adds one command (link l2only [on|off], netlink addr-gen-mode none -- never a /proc sysctl write, which fails the DAC check for a non-root setcap'd uBridge), applied by the four creators themselves, with an idle-silence acceptance test and a known-noise note in the Docker kernel datapath document. No change to existing command grammar. --- docs/design/ubridge-l2-anchor-spec.md | 181 ++++++++++++++++++++++++ docs/features/docker-kernel-datapath.md | 9 ++ 2 files changed, 190 insertions(+) create mode 100644 docs/design/ubridge-l2-anchor-spec.md diff --git a/docs/design/ubridge-l2-anchor-spec.md b/docs/design/ubridge-l2-anchor-spec.md new file mode 100644 index 000000000..6fef6d106 --- /dev/null +++ b/docs/design/ubridge-l2-anchor-spec.md @@ -0,0 +1,181 @@ + + +> Frozen requirements spec for the **uBridge** project. Status: **not yet +> delivered.** Requested by gns3-server branch `feat/qemu-kernel-datapath` +> (QEMU persistent-TAP anchors) and, retroactively, by the shipped Docker +> kernel-datapath stack: today every host-side anchor device carries an IPv6 +> identity, so emulated links carry the *host's* IPv6 chatter. +> +> Scope guard: no wire format, no command grammar change to existing +> commands — this adds one command and one internal step in four creators. + +# L2-only host anchor devices (`link l2only`) + +## 0. Background & scope + +### What an anchor is + +The host-side device that carries a node's adapter in the root namespace: +the veth host end of a Docker adapter (`gv…`, created by `docker create_veth`), +the persistent TAP of a QEMU adapter (`gq…`, created by `tap create`), and the +per-link kernel bridge those anchors are enslaved to (`brctl create`). Links +attach *to* these devices (kernel bridge port or uBridge relay endpoint); the +devices themselves are the only thing uBridge creates on the host's behalf. + +### The problem (measured) + +A host-side anchor that is UP gets an IPv6 link-local address from the kernel +automatically — no user-space actor involved — and with it the kernel emits its +own traffic on that device: + +- a freshly created persistent TAP, brought UP: **6 frames in 2 s** of MLD + reports (`33:33:00:00:00:16`, `33:33:00:00:00:fb`) and DAD neighbor + solicitations (`33:33:ff:…`, ICMPv6 next-header 0/17/58), all sourced from the + device's own link-local address; +- production: the Docker veth host end `gv6ee8d537e1p0` carries + `fe80::5c30:e8ff:fe00:5d8d/64` (`disable_ipv6=0`), and the per-link bridge + `gns30dbb78c5718` carries `fe80::f0d9:94ff:feac:e42b/64`. + +Consequences inside an emulated topology: + +- on a **kernel-datapath** link the bridge floods those frames into the emulated + segment; on a **relay** link uBridge relays them to the peer compute; +- they land in captures and marker pcaps, so "idle link" is not quiet; +- the host **answers** neighbor solicitations for its own link-local address, so + an emulated IPv6 router can form a real adjacency with the host — a phantom + neighbour inside the emulated network. + +IPv4 needs no equivalent knob: nothing self-provisions an IPv4 address, and the +invariant we keep is *no addresses at all* (measured: `ip -4 addr` is empty on +every anchor and per-link bridge on the deployment above, with NetworkManager +reporting them `unmanaged`). IPv6 is the only protocol that lights itself up. + +### Goal + +Every host-side device uBridge creates for the data plane is **pure L2**: no +IPv4 address, no IPv6 address, and no IPv6 stack activity on that device. + +Explicitly **not** in scope: + +- the guest/container side (the peer veth end inside a container netns keeps its + normal IPv6 behaviour); +- global sysctls and any change outside the named interface; +- `brctl addip`-style L3 use of a bridge (cloud / Ethernet-switch paths) — those + devices are meant to have addresses and are not data-plane anchors. + +## A. New command: `link l2only [on|off]` + +Belongs to the `link` module (per-interface operations: `link set`, `link veth`, +`link addr`, `link delete`). + +| Arg | Description | +|-----|-------------| +| `` | An **existing** device. A missing name must fail (`208/ENODEV`), never create a transient device — mirror the `tap` module's `tap_require_existing` reasoning. | +| `on` / `off` | Default `on`. `on` = suppress IPv6 address generation and stack activity on the device; `off` = restore the kernel default. | + +Semantics of `on`: after the call the kernel has assigned **no** IPv6 +link-local address to the device, and the device's `addrgenmode` reports `none` +(`IFLA_INET6_ADDR_GEN_MODE = IN6_ADDR_GEN_MODE_NONE`), so no DAD, MLD or RS is +generated from it. If an address already exists it is removed as part of the +call (an anchor that was created and brought up before this command must end up +clean, not just "no new addresses from now on"). + +Replies: + +``` +link l2only gq1234abcd e0 +100-L2-only set on gq1234abcd e0 +link l2only gq1234abcd e0 on +100-L2-only set on gq1234abcd e0 # idempotent, same reply +link l2only gq1234abcd e0 off +100-L2-only cleared on gq1234abcd e0 +``` + +The success reply is emitted only after a read-back (`RTM_GETLINK` + +`IFLA_AF_SPEC` → `IFLA_INET6_ADDR_GEN_MODE`) confirms the device state, so a +caller may treat `100` as verified rather than requested. + +## B. Where it must be applied + +The creators apply it themselves — gns3-server never issues `link l2only` +directly, and a caller cannot forget it: + +| Creator | Device hardened | Notes | +|---|---|---| +| `tap create ` | the persistent TAP | before the success reply; the device already exists (TUNSETPERSIST) | +| `docker create_veth ` | the **host** end only | the guest end moves into a container netns and is deliberately untouched | +| `link veth ` | both ends | both ends are host-side | +| `brctl create ` | the bridge device itself | the fabric the anchors are enslaved to; the bridge's own link-local floods to every port | + +Rule for future creators: any command that creates a host-side device for the +data plane hardens it, and the acceptance checks in §E apply to it. + +## C. Implementation notes (netlink, not `/proc`) + +- Use `RTM_SETLINK` with `IFLA_AF_SPEC{ AF_INET6, IFLA_INET6_ADDR_GEN_MODE = + IN6_ADDR_GEN_MODE_NONE }` — this is what `ip link set dev X addrgenmode none` + does. Removing an already-assigned link-local is `RTM_DELADDR` on that + address (or letting the kernel drop it once generation is off; verify the end + state either way). +- Optionally also set the nested `IFLA_INET6_CONF` attributes + (`DEVCONF_DISABLE_IPV6 = 1`, `DEVCONF_ACCEPT_RA = 0`) when the running kernel + accepts writes for them. The addr-gen-mode attribute alone already removes the + address, and without an address there is no DAD/MLD/RS. +- Do **not** write `/proc/sys/net/ipv6/conf//disable_ipv6`. The sysctl files + are root-owned mode 0644, so a setcap'd, non-root uBridge can be refused by + the DAC check despite holding `CAP_NET_ADMIN`. The netlink route has no such + problem. +- Failure handling: `EOPNOTSUPP` / `EINVAL` from an old kernel without the + attribute must be **non-fatal** (log, continue) so device creation never fails + because of this hardening; `ENODEV` and any other error are real and reported. + +## D. Reply contract & error codes + +| Code | Meaning | +|------|---------| +| `100` | OK — device verified L2-only (or verified restored for `off`) | +| `203` | Bad number of parameters | +| `204` | Invalid parameter (`on`/`off` expected) | +| `208` | Object not found (`ENODEV`): no such device | +| `206` | Unable to set (netlink error other than EOPNOTSUPP/EINVAL) | + +## E. Test requirements + +1. **No addresses.** After each creator: `ip -6 addr show dev ` is empty + (in particular no `fe80::`) and `ip -4 addr show dev ` is empty; the + same for the bridge created by `brctl create`. +2. **Idle silence.** With the device UP and in its normal role (TAP with an open + fd / veth with its peer up / bridge with two attached ports), nothing appears + on it for 5 s — measured from the peer end, or via + `capture start_kernel ` while idle and by asserting the pcap has + zero packets. Baseline today: 6 frames / 2 s. +3. **Idempotency & revert.** `on` twice → `100` both times, state unchanged; + `off` restores the kernel default (link-local returns after the device is + cycled down/up). +4. **No transient device.** `link l2only nosuchif0` → `208`, and no interface is + created (check `ip -o link`). +5. **Far side untouched.** Inside the container / VM the normal IPv6 link-local + still exists — nothing in this change reaches the peer. +6. **Non-fatal on old kernels.** Simulated by testing on a kernel without the + attribute: the creators still succeed, and §E.1 is then expected to fail with + the device left in the default state. + +## gns3-server alignment (informational — not uBridge scope) + +- **Call sites unchanged**: the four creators harden the devices; the server does + not issue `link l2only`, and no NIO/JSON schema, filter or MCP tool description + changes. This is host-side hygiene only. +- **e2e assertions to add** (kernel-datapath suites, alongside the existing + capture/marker/filter checks): + - anchors and per-link bridges have no IPv4/IPv6 addresses (§E.1); + - an idle anchor captures zero frames in 5 s (§E.2); + - a silent-failure guard found while validating this spec: after attach, every + bridge port reports `brport/state == 3` (forwarding). A bridge device left + DOWN keeps its ports `DISABLED` (`state == 0`) and forwards nothing — with + no error anywhere, so the assertion is the only signal. +- **Rollout**: with a uBridge that lacks the command the anchors keep today's + behaviour (noise present). The e2e assertions skip when `link l2only` answers + `202-Unknown command`, mirroring the existing tc-capability degradation. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index c1405a8c2..fad77439a 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -163,6 +163,15 @@ reopen flips the field. It is not persisted and never sent on create/update. orphan. Both endpoints do this and the peer's still-enslaved port makes the delete fail with EBUSY (suppressed, last one wins), the same contract as link deletion; a node restart rebuilds the bridge from the NIO. +* **Host L3 noise (known, hardening specified).** An anchor is a host-side + netdev like any other, so the kernel gives it an IPv6 link-local address and + emits its own MLD/DAD from it (measured ≈6 frames / 2 s on an idle TAP; the + production anchors and per-link bridges carry `fe80::…` today). On a kernel + link the bridge floods those frames into the emulated segment — and the host + answers neighbor solicitations for its own address, which looks like a phantom + IPv6 neighbor to the emulated nodes. The fix is to make anchors pure L2 at + creation time on the uBridge side: + `docs/design/ubridge-l2-anchor-spec.md`. ## Link operations From 12d8abce45bf13c6a7763f9270bcb59bdd4814c4 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 18:13:14 +0800 Subject: [PATCH 022/101] refactor(compute): share the kernel-datapath machinery across node types Move the datapath-agnostic half of DockerKernelDatapathMixin into a new KernelDatapathMixin (compute/kernel_datapath.py) so a second node type can reuse it verbatim: the three marker primitives, the tc netem / cls_bpf / eBPF appliers, the uBridge tc-capability probe, kernel marker teardown, link detach, the per-link bridge cleanup and the adapter carrier -- plus _kernel_attach(), lifted from DockerVM._connect_nio's kernel branch. The mixin talks about "the anchor" (the host-side interface an adapter owns in the root namespace) instead of "the veth host end": a node type provides _kernel_host_ifc(), _kernel_anchors() and, when needed, _relay_carrier_bridge() and _kernel_error(). Docker keeps only the veth lifecycle and those hooks; behaviour is unchanged. The C901 baseline entry moves with _netem_command_parts (the docker module is no longer over the limit) and the bpf-compile-failure test patches the logger of the module that now owns the code -- net zero new ignores. --- .../compute/docker/docker_kernel_datapath.py | 526 ++------------- gns3server/compute/docker/docker_vm.py | 29 +- gns3server/compute/kernel_datapath.py | 627 ++++++++++++++++++ pyproject.toml | 2 +- .../docker/test_docker_kernel_datapath.py | 2 +- 5 files changed, 679 insertions(+), 507 deletions(-) create mode 100644 gns3server/compute/kernel_datapath.py diff --git a/gns3server/compute/docker/docker_kernel_datapath.py b/gns3server/compute/docker/docker_kernel_datapath.py index a47ada1cd..4138bbce2 100644 --- a/gns3server/compute/docker/docker_kernel_datapath.py +++ b/gns3server/compute/docker/docker_kernel_datapath.py @@ -15,22 +15,19 @@ # along with this program. If not, see . """ -Kernel-datapath machinery for DockerVM, extracted from docker_vm.py: the -veth pair lifecycle (deterministic names, creation, teardown together with -the per-link kernel bridges), the adapter carrier control, the kernel -marker primitives and the tc packet-filter translation (netem, classic-BPF -match-drop, eBPF stateful drops). Every method runs with a DockerVM -instance as ``self`` and reads the same state (_kernel_veths, -_ubridge_tc_caps, the uBridge hypervisor through the BaseNode helpers). +Docker's half of the kernel datapath: the veth pair lifecycle (deterministic +names, creation, teardown) plus the three hooks that tell the shared +KernelDatapathMixin which host interface plays the anchor role. Everything +datapath-agnostic — tc filters, the uBridge tc-capability probe, kernel +markers, attach/detach, adapter carrier — lives in +``gns3server.compute.kernel_datapath`` and is shared with QEMU. """ -import os import contextlib -from gns3server.utils.tc_capabilities import FILTER_EBPF_MODES, parse_tc_capabilities, usable_ebpf_modes +from gns3server.compute.kernel_datapath import KernelDatapathMixin from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError -from ..nios.nio_bridge import NIOBridge from .docker_error import DockerError import logging @@ -38,7 +35,7 @@ import logging log = logging.getLogger(__name__) -class DockerKernelDatapathMixin: +class DockerKernelDatapathMixin(KernelDatapathMixin): """ The kernel-datapath half of a Docker node — consumed by DockerVM so docker_vm.py stays focused on the container lifecycle. @@ -58,6 +55,36 @@ class DockerKernelDatapathMixin: suffix = f"{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" return f"gv{suffix}", f"gc{suffix}" + # ------------------------------------------------------------------ + # Anchor hooks (see KernelDatapathMixin) + # ------------------------------------------------------------------ + + def _kernel_host_ifc(self, adapter_number, port_number=0): + """ + The veth host end of an adapter port, or None for an adapter without + one (unix-socket vendor containers). + """ + + return self._kernel_veths.get((adapter_number, port_number)) + + def _kernel_anchors(self): + """ + Every veth host end this container currently owns. + """ + + return set(self._kernel_veths.values()) + + def _relay_carrier_bridge(self, adapter_number, port_number=0): + """ + The uBridge relay bridge whose TAP carrier replicates the adapter + state — only reachable for adapters without a veth host end. + """ + + return self._bridge_name(adapter_number, port_number) + + def _kernel_error(self, message): + return DockerError(message) + async def _create_veth(self, adapter_number, port_number=0): """ Create the adapter's unified veth pair: the guest end moves into the @@ -129,479 +156,18 @@ class DockerKernelDatapathMixin: container and must be removed explicitly. Deleting an enslaved veth detaches it from its bridge automatically. - The per-link kernel bridges go too: deleting a node (or closing a - project) never runs the link-teardown path, so a bridge whose ports - just disappeared would stay behind as an empty orphan. Both link - endpoints run this — the peer's port still enslaved makes the delete - fail with EBUSY (suppressed) and the last one wins, the same - contract as _remove_kernel_nio. A node restart re-creates the bridge - from the NIO in _connect_nio, so nothing is lost by deleting it. + The per-link kernel bridges go too (``_remove_kernel_bridges``), since + deleting a node or closing a project never runs the link-teardown + path. Both link endpoints run this — the peer's port still enslaved + makes the bridge delete fail with EBUSY (suppressed) and the last one + wins, the same contract as _remove_kernel_nio. A node restart + re-creates the bridge from the NIO in the attach path, so nothing is + lost by deleting it. """ if self.ubridge: for host_ifc in self._kernel_veths.values(): with contextlib.suppress(UbridgeError): await self._ubridge_send(f'docker delete_veth "{host_ifc}"') - for adapter in self._ethernet_adapters: - for nio in adapter.ports.values(): - if isinstance(nio, NIOBridge): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delete "{nio.bridge}"') + await self._remove_kernel_bridges() self._kernel_veths.clear() - - async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): - """Replicate a Docker adapter's connection state on its host interface. - - Kernel-datapath adapters toggle the veth host end's admin state (the - container end loses carrier when its peer is down); relay adapters - toggle the TAP carrier from the uBridge fd holder. - """ - - host_ifc = self._kernel_veths.get((adapter_number, port_number)) - if host_ifc is not None: - state = "up" if connected else "down" - await self._ubridge_send(f'link set "{host_ifc}" {state}') - return - - bridge_name = self._bridge_name(adapter_number, port_number) - state = "on" if connected else "off" - await self._ubridge_send(f"bridge set_nio_tap_carrier {bridge_name} {state}") - - def _kernel_marker_anchor(self, anchor): - """ - Whether *anchor* names one of this container's kernel-datapath veth - host ends. Marker anchors are polymorphic: a relay bridge name on the - relay datapath, the veth host interface on the kernel datapath. A veth - name can never collide with a uBridge bridge name (relay bridges are - ``bridge{adapter}``, veths are ``gv…e…p…``). - """ - - return anchor in self._kernel_veths.values() - - async def _ubridge_add_marker_filter( - self, bridge_name, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None - ): - """ - Kernel-datapath variant of the `mark` filter attach: ``marker - add_kernel`` binds an AF_PACKET socket to the veth host end. Keyword - pairs and semantics (match → MARK signal + pcap append) are identical - to the relay ``bridge add_packet_filter … mark`` command. - """ - - if self._kernel_marker_anchor(bridge_name): - self._validate_marker_name(name) - cmd = 'marker add_kernel {name} {ifc} "{bpf}"'.format(name=name, ifc=bridge_name, bpf=bpf) - if tag is not None: - cmd += f" tag {tag}" - if link_id: - cmd += f" link {link_id}" - if direction is not None: - cmd += f" dir {direction}" - linktype = self._marker_linktype(data_link_type) - if linktype is not None: - cmd += f" linktype {linktype}" - cmd += ' pcap "{path}"'.format(path=pcap_path) - await self._ubridge_send(cmd) - return - await super()._ubridge_add_marker_filter( - bridge_name, - name, - bpf, - pcap_path, - tag=tag, - link_id=link_id, - direction=direction, - data_link_type=data_link_type, - ) - - async def _ubridge_delete_marker_filter(self, bridge_name, name): - """ - Kernel-datapath variant of the marker removal: ``marker delete_kernel`` - (idempotent in uBridge — a no-op when the marker is already gone). - """ - - if self._kernel_marker_anchor(bridge_name): - if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): - return - try: - await self._ubridge_send(f"marker delete_kernel {bridge_name} {name}") - except UbridgeError as e: - log.warning("Could not remove kernel marker '%s' from %s: %s", name, bridge_name, e) - return - await super()._ubridge_delete_marker_filter(bridge_name, name) - - async def _ubridge_enable_marker_filter(self, anchor, name, state): - """ - Kernel-datapath variant of the marker on/off toggle: ``marker - enable_kernel`` — installed but silent when off, pcap preserved. - """ - - if self._kernel_marker_anchor(anchor): - await self._ubridge_send(f"marker enable_kernel {anchor} {name} {state}") - return - await super()._ubridge_enable_marker_filter(anchor, name, state) - - @staticmethod - def _netem_command_parts(filters): - """ - Build the ``tc netem set`` keyword sequence (in the frozen grammar - order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, - limit, distribution, seed) plus the set of capability tokens the - sequence needs beyond the original netem surface. Original-surface - keywords (delay, jitter, loss, corrupt, dup) need no probe; the - extension tokens (rate, reorder, gemodel, dist, seed, limit) are - checked against ``tc capabilities`` by the caller. ``correl`` has no - token of its own in the capabilities list, so it gates on "rate" as - the netem-extension build marker. - - :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) - - :returns: (keyword sequence, required extension tokens) - """ - - def values_of(key): - # tolerate the legacy bare-value shape ({"packet_loss": 10}) - values = filters.get(key) - if isinstance(values, (list, tuple)): - return list(values) - return [values] if values else [] - - parts = [] - ext_tokens = set() - - delay = values_of("delay") - if delay: - parts.append(f"delay {int(delay[0])}") - if len(delay) > 1 and int(delay[1]): - parts.append(f"jitter {int(delay[1])}") - - gemodel = values_of("gemodel") - if gemodel: - segment = f"loss gemodel {int(gemodel[0])}" - if len(gemodel) > 1: - segment += f" {int(gemodel[1])}" - if len(gemodel) > 2: - segment += f" {int(gemodel[2])}" - parts.append(segment) - ext_tokens.add("gemodel") - else: - loss = values_of("packet_loss") - if loss and int(loss[0]): - parts.append(f"loss {int(loss[0])}") - if len(loss) > 1 and int(loss[1]): - parts.append(f"correl {int(loss[1])}") - ext_tokens.add("rate") # correl: extension build marker - - duplicate = values_of("duplicate") - if duplicate and int(duplicate[0]): - parts.append(f"dup {int(duplicate[0])}") - if len(duplicate) > 1 and int(duplicate[1]): - parts.append(f"correl {int(duplicate[1])}") - ext_tokens.add("rate") # correl: extension build marker - - corrupt = values_of("corrupt") - if corrupt and int(corrupt[0]): - parts.append(f"corrupt {int(corrupt[0])}") - - reorder = values_of("reorder") - if reorder: - segment = f"reorder {int(reorder[0])}" - if len(reorder) > 1 and int(reorder[1]): - segment += f" correl {int(reorder[1])}" - if len(reorder) > 2 and int(reorder[2]): - segment += f" gap {int(reorder[2])}" - parts.append(segment) - ext_tokens.add("reorder") - - rate = values_of("rate") - if rate and str(rate[0]).strip(): - parts.append(f"rate {str(rate[0]).strip()}") - ext_tokens.add("rate") - - limit = values_of("limit") - if limit and int(limit[0]): - parts.append(f"limit {int(limit[0])}") - ext_tokens.add("limit") - - if delay and len(delay) > 2 and str(delay[2]).strip().lower() not in ("", "uniform"): - # "uniform" is the kernel default — emitting nothing keeps the - # command compatible with the original netem surface. - parts.append(f"distribution {str(delay[2]).strip().lower()}") - ext_tokens.add("dist") - - seed = values_of("seed") - if seed and int(seed[0]): - # 0 is the "disabled" convention everywhere else — treat it so - # here too (the controller's inactive-filter pass drops it). - parts.append(f"seed {int(seed[0])}") - ext_tokens.add("seed") - - return parts, ext_tokens - - async def _ubridge_apply_netem(self, host_ifc, filters): - """ - Translate the NIO's impairment filters into one tc netem qdisc on the - veth host end. The netem-expressible types (delay, packet_loss, - corrupt and the netem extensions rate/reorder/gemodel/duplicate/ - seed/limit) merge into a single qdisc — its egress covers the traffic - entering this container, and with both link endpoints applying - theirs, every direction of the link is impaired exactly once (the - same net effect as the relay, where both directions cross the single - filtered bridge). - - The kernel's netem replace MERGES optional attributes (rate, - correlation, reorder, corrupt, gemodel, distribution: absent attr = - previous value kept), so re-applying a filter set with a parameter - *removed* would silently keep the old value. A ``tc reset`` before - every ``netem set`` makes the re-apply a true reconcile (reset also - drops clsact and its bpf_drop filters — the caller re-applies them - right after in the same flow). An empty filter set is the reset - alone. - - :param host_ifc: veth host-end interface name - :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) - """ - - parts, ext_tokens = self._netem_command_parts(filters) - if parts and ext_tokens: - # Extension keywords need the netem-extension uBridge. Plain - # delay/loss/corrupt/dup never probes — an old uBridge serves - # the original surface untouched (mirrors bpf_drop's guard). - # Checked before the reset so a failed update leaves the - # previously applied qdisc in place. - caps = await self._ubridge_tc_capabilities() - missing = ext_tokens - set(caps.get("netem", "").split(",")) - if missing: - raise DockerError( - "Packet filter(s) using {} need a uBridge with the netem extensions " - "(tc capabilities reports '{}'); upgrade uBridge on this compute".format( - ", ".join(sorted(missing)), caps.get("netem") or "no tc support" - ) - ) - try: - await self._ubridge_send(f'tc reset "{host_ifc}"') - except UbridgeError as e: - # "No such file or directory" = no qdisc was attached: already - # clean. Everything else is a real failure and must surface. - if "No such file" not in str(e): - raise - if not parts: - return - await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) - - async def _ubridge_tc_capabilities(self): - """ - uBridge's tc-module feature report (``tc capabilities``), probed once - per uBridge process and cached — the reply shape is - ``netem=;ebpf=0|1;cbpf=0|1[;ebpf_modes=]`` (parsed - by parse_tc_capabilities). uBridge builds without the tc module - answer "Unknown module" and old builds without the command answer - "Unknown command": both mean no kernel filter support, cached as an - empty dict. - """ - - if self._ubridge_tc_caps is None: - caps = {} - if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - try: - reply = await self._ubridge_hypervisor.send("tc capabilities") - except UbridgeError: - reply = [] - caps = parse_tc_capabilities(reply[0] if reply else "") - self._ubridge_tc_caps = caps - return self._ubridge_tc_caps - - async def _ubridge_apply_bpf_drops(self, host_ifc, filters): - """ - Translate the NIO's ``bpf`` filter (one expression per line, any line - matching drops the packet) into cls_bpf classifiers on the veth host - end's clsact egress: ``tc bpf_drop`` compiles each line with libpcap - (same compiler as the relay's bpf filter) and drops matches with a - gact TC_ACT_SHOT. Reconcile is always flush + re-add — same-prio - resend replaces the filter node in uBridge, and the flush drops - lines the user removed. - - :param host_ifc: veth host-end interface name - :param filters: NIO filters dictionary - """ - - bpf = filters.get("bpf") - lines = [line.strip() for line in (bpf[0].split("\n") if bpf else []) if line.strip()] - if not lines and self._ubridge_tc_caps is None: - # Nothing to add and nothing was ever installed (no tc module - # probed yet) — skip touching an old uBridge entirely. - return - caps = await self._ubridge_tc_capabilities() - if not lines: - if caps: - await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') - return - if caps.get("cbpf") != "1": - raise DockerError( - "Packet filter 'bpf' on a kernel-datapath link needs a uBridge with cBPF support " - "(tc capabilities reports no cbpf); upgrade uBridge on this compute or keep the " - "link on the relay datapath" - ) - await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') - for offset, line in enumerate(lines): - try: - await self._ubridge_send( - 'tc bpf_drop add "{ifc}" {prio} "{expr}"'.format(ifc=host_ifc, prio=10 + offset, expr=line) - ) - except UbridgeError as e: - if "Cannot compile filter" not in str(e): - raise - # Mirror the relay path: a line that no longer compiles on - # this uBridge's libpcap is skipped with a warning instead - # of breaking the link (the controller already validated - # the syntax with tcpdump at create/update time). - message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {line}" - log.warning(message) - self.project.emit("log.warning", {"message": message}) - - async def _ubridge_apply_ebpf_drops(self, host_ifc, filters): - """ - Translate the NIO's stateful filters into uBridge's eBPF impairment - classifier on the veth host end's clsact egress (prio 1, below - bpf_drop's 10-99): ``frequency_drop`` becomes the exact every-Nth - mode (``tc nth_drop``; -1 = drop everything maps to every 1st), the - kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``), - and the kernel-only ``window_drop`` type the time-window mode - (``tc window_drop``: single outage, or recurring flaps with an - optional per-cycle jitter). The program evaluates the modes in the - fixed order nth → quota → window. A mode is only sent when - ``tc capabilities`` declares its token (``ebpf_modes``); builds - predating the field keep their shipped modes (everything but - window). - - Reconcile is a plain re-set on every apply: the preceding netem - apply resets the interface (which detaches the program and drops - uBridge's per-interface registry entry), so a mode set re-loads the - program from scratch, and a mode left absent is turned off - explicitly (idempotent) so a removed filter stops dropping. For - window_drop this also means the schedule restarts on every apply: - the first window opens Start-ms from the moment of this call. - - Semantic difference vs the relay, documented server-side: the relay - counts packets of BOTH directions through its single filtered - bridge, while each veth end's classifier counts only the traffic - entering that container — with both endpoints applying theirs, each - direction drops every Nth independently (their window schedules - start ~simultaneously, so a round trip survives only when both - directions are outside their windows). - - :param host_ifc: veth host-end interface name - :param filters: NIO filters dictionary - """ - - def values_of(key): - values = filters.get(key) - if isinstance(values, (list, tuple)): - return list(values) - return [values] if values else [] - - frequency = values_of("frequency_drop") - quota = values_of("quota") - window = values_of("window_drop") - requested = {filter_type: values_of(filter_type) for filter_type in FILTER_EBPF_MODES} - if not any(requested.values()) and self._ubridge_tc_caps is None: - # Nothing to set and nothing was ever installed (no probe yet) — - # skip touching an old uBridge entirely. - return - caps = await self._ubridge_tc_capabilities() - # A mode is usable iff ebpf=1 and the build declares its token - # (ebpf_modes); builds predating the field keep their shipped modes. - modes = usable_ebpf_modes(caps) - if not any(requested.values()): - # Only modes this build declares can ever have been installed — - # sending "off" for an undeclared mode would hit an unknown - # command; "off" is idempotent in uBridge. - for filter_type, mode in FILTER_EBPF_MODES.items(): - if mode in modes: - await self._ubridge_send(f'tc {mode}_drop "{host_ifc}" off') - return - unsupported = sorted(f for f, values in requested.items() if values and FILTER_EBPF_MODES[f] not in modes) - if unsupported: - if not modes: - raise DockerError( - "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " - "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " - "the uBridge binary); upgrade uBridge on this compute or keep the link on the " - "relay datapath".format(", ".join(unsupported)) - ) - raise DockerError( - "Packet filter(s) {} on a kernel-datapath link need a newer uBridge (tc capabilities " - "reports ebpf_modes={}); upgrade uBridge on this compute or keep the link on the " - "relay datapath".format(", ".join(unsupported), ",".join(modes)) - ) - if frequency: - # relay semantics: -1 = drop everything, N = every Nth packet - nth = 1 if int(frequency[0]) == -1 else int(frequency[0]) - await self._ubridge_send(f'tc nth_drop "{host_ifc}" {nth}') - elif "nth" in modes: - await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') - if quota: - await self._ubridge_send( - 'tc quota_drop "{ifc}" {bytes} {pct}'.format(ifc=host_ifc, bytes=int(quota[0]), pct=int(quota[1])) - ) - elif "quota" in modes: - await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') - if window: - # [start, outage, chance, period?, jitter?] — start is relative - # to this apply (the schedule restarts on every reconcile). - params = " ".join(str(int(v)) for v in window) - await self._ubridge_send(f'tc window_drop "{host_ifc}" {params}') - elif "window" in modes: - await self._ubridge_send(f'tc window_drop "{host_ifc}" off') - - async def _remove_kernel_markers(self, host_ifc): - """ - Tear down every kernel marker anchored to *host_ifc*: the veth survives - link deletion (it is the adapter interface, not the link), so the - AF_PACKET sockets must be closed explicitly — otherwise a deleted - link's markers would keep sniffing and signaling. Mirrors the relay - datapath where deleting the uBridge bridge silently drops its filters. - """ - - from gns3server.compute.marker.marker_manager import MarkerManager - - manager = MarkerManager.instance() - markers_dir = self.project.markers_working_directory() - for (name, link_id), anchor in list(self._marker_filter_bridges.items()): - if anchor != host_ifc: - continue - self._marker_filter_bridges.pop((name, link_id)) - self._marker_specs.pop((name, link_id), None) - await self._ubridge_delete_marker_filter(host_ifc, name) - try: - os.remove(os.path.join(markers_dir, f"{self._id}_{link_id}_{name}.pcap")) - except FileNotFoundError: - pass - except OSError as e: - log.warning("Could not remove marker pcap for '%s' on link %s: %s", name, link_id, e) - manager.unregister(self._id, name) - - async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): - """ - Detach a kernel-datapath adapter port from its per-link kernel bridge. - Both endpoints run this concurrently: deleting a bridge that still has - the peer's port enslaved fails with EBUSY, so the last endpoint to - remove its port wins the deletion and the loser's failure is expected - and suppressed. - """ - - host_ifc = self._kernel_veths.get((adapter_number, port_number)) - if host_ifc is not None: - await self._remove_kernel_markers(host_ifc) - # The veth survives link deletion, so an orphaned netem qdisc - # would keep impairing whatever attaches to the adapter next - # (kernel link or relay). Best-effort: ENOENT just means no - # qdisc was attached. - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tc reset "{host_ifc}"') - if self.status == "started": - await self._set_adapter_carrier(adapter_number, False, port_number) - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delete "{nio.bridge}"') diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 98feb66c1..cf48f4ab7 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1821,31 +1821,10 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): adapter_number=adapter_number, port_number=port_number, name=self.name ) ) - # Both endpoints run this concurrently and either may create the - # bridge first: treat an existing bridge as success instead of a - # create race failure. - try: - await self._ubridge_send(f'brctl create "{nio.bridge}"') - except UbridgeError: - # Raises again if the bridge genuinely does not exist. - await self._ubridge_send(f'brctl show "{nio.bridge}"') - await self._ubridge_send(f'link set "{nio.bridge}" up') - await self._ubridge_send(f'brctl addif "{nio.bridge}" "{host_ifc}"') - if nio.capturing: - # Restore a capture that was active before a node restart - # (mirrors the relay path's start_capture in _connect_nio). - await self._ubridge_send(f'capture start_kernel {host_ifc} "{nio.pcap_output_file}"') - # Markers carried by the NIO attach to the veth host end (AF_PACKET - # taps) — the anchor is the interface, not a relay bridge. - await self._ubridge_apply_markers(host_ifc, nio) - # Impairment filters become one tc netem qdisc on the veth host - # end (restored here on node restart, like the capture above); - # bpf expressions become cls_bpf match-drop classifiers and - # frequency_drop/quota/window_drop become the eBPF stateful - # classifier. - await self._ubridge_apply_netem(host_ifc, nio.filters) - await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) - await self._ubridge_apply_ebpf_drops(host_ifc, nio.filters) + # The mechanics (bridge create/addif, capture, markers, filters) + # live in KernelDatapathMixin, shared with every other node type + # whose adapters own a host-side anchor. + await self._kernel_attach(host_ifc, nio) return # Relay NIO. On a veth-backed adapter (the unified Docker interface) diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py new file mode 100644 index 000000000..1567b61fa --- /dev/null +++ b/gns3server/compute/kernel_datapath.py @@ -0,0 +1,627 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Kernel-datapath machinery shared by every node type whose adapters own a +host-side interface: the tc packet-filter translation (netem, classic-BPF +match-drop, eBPF stateful drops), the uBridge tc-capability probe, the kernel +marker primitives, the link attach/detach flows and the adapter carrier. + +An **anchor** is the host-side interface an adapter owns in the root +namespace — the veth host end of a Docker adapter, the persistent TAP of a +QEMU adapter, and whatever a future node type creates. Links attach *to* the +anchor (per-link kernel bridge port, or a uBridge relay endpoint via +AF_PACKET); links never create or destroy it. Every datapath-specific command +in this module takes an anchor name, which is why a TAP and a veth host end +are interchangeable here. + +A node type using this mixin provides: + +* ``_kernel_host_ifc(adapter_number, port_number)`` — its anchor for a port, + or None when the adapter has none (a vendor container without a veth); +* ``_kernel_anchors()`` — every anchor it currently owns, used to recognise + the polymorphic marker anchor; +* optionally ``_relay_carrier_bridge(adapter_number, port_number)`` — the + uBridge bridge carrying the adapter's TAP relay, for node types whose relay + datapath still rides a uBridge-owned TAP instead of an anchor; +* optionally ``_kernel_error(message)`` — the module's own error type, so + failures surface as ``DockerError`` / ``QemuError`` rather than ``NodeError``. +""" + +import os +import contextlib + +from gns3server.utils.tc_capabilities import FILTER_EBPF_MODES, parse_tc_capabilities, usable_ebpf_modes +from gns3server.compute.ubridge.ubridge_error import UbridgeError +from gns3server.compute.error import NodeError + +from .nios.nio_bridge import NIOBridge + +import logging + +log = logging.getLogger(__name__) + + +class KernelDatapathMixin: + """ + The datapath-agnostic half of a kernel-datapath node: consumed by + DockerVM and QEMUVm so each of them stays focused on its own node + lifecycle. + """ + + # ------------------------------------------------------------------ + # Hooks a node type implements + # ------------------------------------------------------------------ + + def _kernel_host_ifc(self, adapter_number, port_number=0): + """ + Returns the anchor interface of an adapter port, or None when the + adapter has no host-side interface (a vendor container wired through + a unix socket). + """ + + raise NotImplementedError + + def _kernel_anchors(self): + """ + Returns the set of anchor interface names this node currently owns. + """ + + raise NotImplementedError + + def _relay_carrier_bridge(self, adapter_number, port_number=0): + """ + Returns the uBridge relay bridge whose TAP carrier replicates this + adapter's connection state, or None when the adapter's relay does not + ride a uBridge-owned TAP (every anchor-backed adapter: its carrier is + the anchor's own admin state). + """ + + return None + + def _kernel_error(self, message): + """ + Returns the node module's error for a kernel-datapath failure + (DockerError, QemuError, ...). + """ + + return NodeError(message) + + def _kernel_marker_anchor(self, anchor): + """ + Whether *anchor* names one of this node's anchors. Marker anchors are + polymorphic: a relay bridge name on the relay datapath, the anchor + interface on the kernel datapath. Anchor names cannot collide with + uBridge bridge names. + """ + + return anchor in self._kernel_anchors() + + async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): + """Replicate an adapter's connection state on its host interface. + + Anchor-backed adapters toggle the anchor's admin state (the peer — + container end or VM TAP reader — loses carrier when it is down); + adapters whose relay rides a uBridge TAP toggle that TAP's carrier + from the uBridge fd holder. + """ + + host_ifc = self._kernel_host_ifc(adapter_number, port_number) + if host_ifc is not None: + state = "up" if connected else "down" + await self._ubridge_send(f'link set "{host_ifc}" {state}') + return + + bridge_name = self._relay_carrier_bridge(adapter_number, port_number) + if bridge_name is None: + return + state = "on" if connected else "off" + await self._ubridge_send(f"bridge set_nio_tap_carrier {bridge_name} {state}") + + # ------------------------------------------------------------------ + # Link attach / detach + # ------------------------------------------------------------------ + + async def _kernel_attach(self, anchor, nio): + """ + Attach an anchor to the per-link kernel bridge carried by *nio* + (NIOBridge): create the bridge, bring it and the anchor up, enslave + the anchor, then install the capture, markers and impairment filters + the NIO carries. Called on link creation and on node start (a link + that outlived a node restart is re-attached from its NIO). + + Both link endpoints run this concurrently and either may create the + bridge first: an existing bridge is success instead of a create race + failure. + """ + + try: + await self._ubridge_send(f'brctl create "{nio.bridge}"') + except UbridgeError: + # Raises again if the bridge genuinely does not exist. + await self._ubridge_send(f'brctl show "{nio.bridge}"') + await self._ubridge_send(f'link set "{nio.bridge}" up') + await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"') + if nio.capturing: + # Restore a capture that was active before a node restart + # (mirrors the relay path's start_capture). + await self._ubridge_send(f'capture start_kernel {anchor} "{nio.pcap_output_file}"') + # Markers carried by the NIO attach to the anchor (AF_PACKET taps) — + # the anchor is the interface, not a relay bridge. + await self._ubridge_apply_markers(anchor, nio) + # Impairment filters become one tc netem qdisc on the anchor + # (restored here on node restart, like the capture above); bpf + # expressions become cls_bpf match-drop classifiers and + # frequency_drop/quota/window_drop become the eBPF stateful + # classifier. + await self._ubridge_apply_netem(anchor, nio.filters) + await self._ubridge_apply_bpf_drops(anchor, nio.filters) + await self._ubridge_apply_ebpf_drops(anchor, nio.filters) + + async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): + """ + Detach an anchor from its per-link kernel bridge. Both endpoints run + this concurrently: deleting a bridge that still has the peer's port + enslaved fails with EBUSY, so the last endpoint to remove its port + wins the deletion and the loser's failure is expected and suppressed. + """ + + host_ifc = self._kernel_host_ifc(adapter_number, port_number) + if host_ifc is not None: + await self._remove_kernel_markers(host_ifc) + # The anchor survives link deletion, so an orphaned netem qdisc + # would keep impairing whatever attaches to the adapter next + # (kernel link or relay). Best-effort: ENOENT just means no + # qdisc was attached. + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tc reset "{host_ifc}"') + if self.status == "started": + await self._set_adapter_carrier(adapter_number, False, port_number) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delete "{nio.bridge}"') + + async def _remove_kernel_bridges(self): + """ + Delete every per-link kernel bridge this node still holds. Deleting a + node (or closing a project) never runs the link-teardown path, so a + bridge whose ports just disappeared would stay behind as an empty + orphan. Both link endpoints run this — the peer's still-enslaved port + makes the delete fail with EBUSY (suppressed) and the last one wins, + the same contract as link deletion. A node restart re-creates the + bridge from the NIO in the attach path, so nothing is lost by + deleting it. + """ + + if not self.ubridge: + return + for adapter in self._ethernet_adapters: + for nio in adapter.ports.values(): + if isinstance(nio, NIOBridge): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delete "{nio.bridge}"') + + # ------------------------------------------------------------------ + # Markers + # ------------------------------------------------------------------ + + async def _ubridge_add_marker_filter( + self, bridge_name, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None + ): + """ + Kernel-datapath variant of the `mark` filter attach: ``marker + add_kernel`` binds an AF_PACKET socket to the anchor. Keyword pairs + and semantics (match → MARK signal + pcap append) are identical to + the relay ``bridge add_packet_filter … mark`` command. + """ + + if self._kernel_marker_anchor(bridge_name): + self._validate_marker_name(name) + cmd = 'marker add_kernel {name} {ifc} "{bpf}"'.format(name=name, ifc=bridge_name, bpf=bpf) + if tag is not None: + cmd += f" tag {tag}" + if link_id: + cmd += f" link {link_id}" + if direction is not None: + cmd += f" dir {direction}" + linktype = self._marker_linktype(data_link_type) + if linktype is not None: + cmd += f" linktype {linktype}" + cmd += ' pcap "{path}"'.format(path=pcap_path) + await self._ubridge_send(cmd) + return + await super()._ubridge_add_marker_filter( + bridge_name, + name, + bpf, + pcap_path, + tag=tag, + link_id=link_id, + direction=direction, + data_link_type=data_link_type, + ) + + async def _ubridge_delete_marker_filter(self, bridge_name, name): + """ + Kernel-datapath variant of the marker removal: ``marker delete_kernel`` + (idempotent in uBridge — a no-op when the marker is already gone). + """ + + if self._kernel_marker_anchor(bridge_name): + if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): + return + try: + await self._ubridge_send(f"marker delete_kernel {bridge_name} {name}") + except UbridgeError as e: + log.warning("Could not remove kernel marker '%s' from %s: %s", name, bridge_name, e) + return + await super()._ubridge_delete_marker_filter(bridge_name, name) + + async def _ubridge_enable_marker_filter(self, anchor, name, state): + """ + Kernel-datapath variant of the marker on/off toggle: ``marker + enable_kernel`` — installed but silent when off, pcap preserved. + """ + + if self._kernel_marker_anchor(anchor): + await self._ubridge_send(f"marker enable_kernel {anchor} {name} {state}") + return + await super()._ubridge_enable_marker_filter(anchor, name, state) + + async def _remove_kernel_markers(self, host_ifc): + """ + Tear down every kernel marker anchored to *host_ifc*: the anchor + survives link deletion (it is the adapter interface, not the link), + so the AF_PACKET sockets must be closed explicitly — otherwise a + deleted link's markers would keep sniffing and signaling. Mirrors the + relay datapath where deleting the uBridge bridge silently drops its + filters. + """ + + from gns3server.compute.marker.marker_manager import MarkerManager + + manager = MarkerManager.instance() + markers_dir = self.project.markers_working_directory() + for (name, link_id), anchor in list(self._marker_filter_bridges.items()): + if anchor != host_ifc: + continue + self._marker_filter_bridges.pop((name, link_id)) + self._marker_specs.pop((name, link_id), None) + await self._ubridge_delete_marker_filter(host_ifc, name) + try: + os.remove(os.path.join(markers_dir, f"{self._id}_{link_id}_{name}.pcap")) + except FileNotFoundError: + pass + except OSError as e: + log.warning("Could not remove marker pcap for '%s' on link %s: %s", name, link_id, e) + manager.unregister(self._id, name) + + # ------------------------------------------------------------------ + # Packet filters (tc netem / cls_bpf / eBPF stateful classifier) + # ------------------------------------------------------------------ + + @staticmethod + def _netem_command_parts(filters): + """ + Build the ``tc netem set`` keyword sequence (in the frozen grammar + order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, + limit, distribution, seed) plus the set of capability tokens the + sequence needs beyond the original netem surface. Original-surface + keywords (delay, jitter, loss, corrupt, dup) need no probe; the + extension tokens (rate, reorder, gemodel, dist, seed, limit) are + checked against ``tc capabilities`` by the caller. ``correl`` has no + token of its own in the capabilities list, so it gates on "rate" as + the netem-extension build marker. + + :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) + + :returns: (keyword sequence, required extension tokens) + """ + + def values_of(key): + # tolerate the legacy bare-value shape ({"packet_loss": 10}) + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + parts = [] + ext_tokens = set() + + delay = values_of("delay") + if delay: + parts.append(f"delay {int(delay[0])}") + if len(delay) > 1 and int(delay[1]): + parts.append(f"jitter {int(delay[1])}") + + gemodel = values_of("gemodel") + if gemodel: + segment = f"loss gemodel {int(gemodel[0])}" + if len(gemodel) > 1: + segment += f" {int(gemodel[1])}" + if len(gemodel) > 2: + segment += f" {int(gemodel[2])}" + parts.append(segment) + ext_tokens.add("gemodel") + else: + loss = values_of("packet_loss") + if loss and int(loss[0]): + parts.append(f"loss {int(loss[0])}") + if len(loss) > 1 and int(loss[1]): + parts.append(f"correl {int(loss[1])}") + ext_tokens.add("rate") # correl: extension build marker + + duplicate = values_of("duplicate") + if duplicate and int(duplicate[0]): + parts.append(f"dup {int(duplicate[0])}") + if len(duplicate) > 1 and int(duplicate[1]): + parts.append(f"correl {int(duplicate[1])}") + ext_tokens.add("rate") # correl: extension build marker + + corrupt = values_of("corrupt") + if corrupt and int(corrupt[0]): + parts.append(f"corrupt {int(corrupt[0])}") + + reorder = values_of("reorder") + if reorder: + segment = f"reorder {int(reorder[0])}" + if len(reorder) > 1 and int(reorder[1]): + segment += f" correl {int(reorder[1])}" + if len(reorder) > 2 and int(reorder[2]): + segment += f" gap {int(reorder[2])}" + parts.append(segment) + ext_tokens.add("reorder") + + rate = values_of("rate") + if rate and str(rate[0]).strip(): + parts.append(f"rate {str(rate[0]).strip()}") + ext_tokens.add("rate") + + limit = values_of("limit") + if limit and int(limit[0]): + parts.append(f"limit {int(limit[0])}") + ext_tokens.add("limit") + + if delay and len(delay) > 2 and str(delay[2]).strip().lower() not in ("", "uniform"): + # "uniform" is the kernel default — emitting nothing keeps the + # command compatible with the original netem surface. + parts.append(f"distribution {str(delay[2]).strip().lower()}") + ext_tokens.add("dist") + + seed = values_of("seed") + if seed and int(seed[0]): + # 0 is the "disabled" convention everywhere else — treat it so + # here too (the controller's inactive-filter pass drops it). + parts.append(f"seed {int(seed[0])}") + ext_tokens.add("seed") + + return parts, ext_tokens + + async def _ubridge_apply_netem(self, host_ifc, filters): + """ + Translate the NIO's impairment filters into one tc netem qdisc on the + anchor. The netem-expressible types (delay, packet_loss, corrupt and + the netem extensions rate/reorder/gemodel/duplicate/seed/limit) merge + into a single qdisc — its egress covers the traffic entering this + node, and with both link endpoints applying theirs, every direction + of the link is impaired exactly once (the same net effect as the + relay, where both directions cross the single filtered bridge). + + The kernel's netem replace MERGES optional attributes (rate, + correlation, reorder, corrupt, gemodel, distribution: absent attr = + previous value kept), so re-applying a filter set with a parameter + *removed* would silently keep the old value. A ``tc reset`` before + every ``netem set`` makes the re-apply a true reconcile (reset also + drops clsact and its bpf_drop filters — the caller re-applies them + right after in the same flow). An empty filter set is the reset + alone. + + :param host_ifc: anchor interface name + :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) + """ + + parts, ext_tokens = self._netem_command_parts(filters) + if parts and ext_tokens: + # Extension keywords need the netem-extension uBridge. Plain + # delay/loss/corrupt/dup never probes — an old uBridge serves + # the original surface untouched (mirrors bpf_drop's guard). + # Checked before the reset so a failed update leaves the + # previously applied qdisc in place. + caps = await self._ubridge_tc_capabilities() + missing = ext_tokens - set(caps.get("netem", "").split(",")) + if missing: + raise self._kernel_error( + "Packet filter(s) using {} need a uBridge with the netem extensions " + "(tc capabilities reports '{}'); upgrade uBridge on this compute".format( + ", ".join(sorted(missing)), caps.get("netem") or "no tc support" + ) + ) + try: + await self._ubridge_send(f'tc reset "{host_ifc}"') + except UbridgeError as e: + # "No such file or directory" = no qdisc was attached: already + # clean. Everything else is a real failure and must surface. + if "No such file" not in str(e): + raise + if not parts: + return + await self._ubridge_send('tc netem set "{ifc}" {params}'.format(ifc=host_ifc, params=" ".join(parts))) + + async def _ubridge_tc_capabilities(self): + """ + uBridge's tc-module feature report (``tc capabilities``), probed once + per uBridge process and cached — the reply shape is + ``netem=;ebpf=0|1;cbpf=0|1[;ebpf_modes=]`` (parsed + by parse_tc_capabilities). uBridge builds without the tc module + answer "Unknown module" and old builds without the command answer + "Unknown command": both mean no kernel filter support, cached as an + empty dict. + """ + + if self._ubridge_tc_caps is None: + caps = {} + if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): + try: + reply = await self._ubridge_hypervisor.send("tc capabilities") + except UbridgeError: + reply = [] + caps = parse_tc_capabilities(reply[0] if reply else "") + self._ubridge_tc_caps = caps + return self._ubridge_tc_caps + + async def _ubridge_apply_bpf_drops(self, host_ifc, filters): + """ + Translate the NIO's ``bpf`` filter (one expression per line, any line + matching drops the packet) into cls_bpf classifiers on the anchor's + clsact egress: ``tc bpf_drop`` compiles each line with libpcap (same + compiler as the relay's bpf filter) and drops matches with a gact + TC_ACT_SHOT. Reconcile is always flush + re-add — same-prio resend + replaces the filter node in uBridge, and the flush drops lines the + user removed. + + :param host_ifc: anchor interface name + :param filters: NIO filters dictionary + """ + + bpf = filters.get("bpf") + lines = [line.strip() for line in (bpf[0].split("\n") if bpf else []) if line.strip()] + if not lines and self._ubridge_tc_caps is None: + # Nothing to add and nothing was ever installed (no tc module + # probed yet) — skip touching an old uBridge entirely. + return + caps = await self._ubridge_tc_capabilities() + if not lines: + if caps: + await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') + return + if caps.get("cbpf") != "1": + raise self._kernel_error( + "Packet filter 'bpf' on a kernel-datapath link needs a uBridge with cBPF support " + "(tc capabilities reports no cbpf); upgrade uBridge on this compute or keep the " + "link on the relay datapath" + ) + await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') + for offset, line in enumerate(lines): + try: + await self._ubridge_send( + 'tc bpf_drop add "{ifc}" {prio} "{expr}"'.format(ifc=host_ifc, prio=10 + offset, expr=line) + ) + except UbridgeError as e: + if "Cannot compile filter" not in str(e): + raise + # Mirror the relay path: a line that no longer compiles on + # this uBridge's libpcap is skipped with a warning instead + # of breaking the link (the controller already validated + # the syntax with tcpdump at create/update time). + message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {line}" + log.warning(message) + self.project.emit("log.warning", {"message": message}) + + async def _ubridge_apply_ebpf_drops(self, host_ifc, filters): + """ + Translate the NIO's stateful filters into uBridge's eBPF impairment + classifier on the anchor's clsact egress (prio 1, below bpf_drop's + 10-99): ``frequency_drop`` becomes the exact every-Nth mode + (``tc nth_drop``; -1 = drop everything maps to every 1st), the + kernel-only ``quota`` type the byte-cap mode (``tc quota_drop``), + and the kernel-only ``window_drop`` type the time-window mode + (``tc window_drop``: single outage, or recurring flaps with an + optional per-cycle jitter). The program evaluates the modes in the + fixed order nth → quota → window. A mode is only sent when + ``tc capabilities`` declares its token (``ebpf_modes``); builds + predating the field keep their shipped modes (everything but + window). + + Reconcile is a plain re-set on every apply: the preceding netem + apply resets the interface (which detaches the program and drops + uBridge's per-interface registry entry), so a mode set re-loads the + program from scratch, and a mode left absent is turned off + explicitly (idempotent) so a removed filter stops dropping. For + window_drop this also means the schedule restarts on every apply: + the first window opens Start-ms from the moment of this call. + + Semantic difference vs the relay, documented server-side: the relay + counts packets of BOTH directions through its single filtered + bridge, while each anchor's classifier counts only the traffic + entering that node — with both endpoints applying theirs, each + direction drops every Nth independently (their window schedules + start ~simultaneously, so a round trip survives only when both + directions are outside their windows). + + :param host_ifc: anchor interface name + :param filters: NIO filters dictionary + """ + + def values_of(key): + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + frequency = values_of("frequency_drop") + quota = values_of("quota") + window = values_of("window_drop") + requested = {filter_type: values_of(filter_type) for filter_type in FILTER_EBPF_MODES} + if not any(requested.values()) and self._ubridge_tc_caps is None: + # Nothing to set and nothing was ever installed (no probe yet) — + # skip touching an old uBridge entirely. + return + caps = await self._ubridge_tc_capabilities() + # A mode is usable iff ebpf=1 and the build declares its token + # (ebpf_modes); builds predating the field keep their shipped modes. + modes = usable_ebpf_modes(caps) + if not any(requested.values()): + # Only modes this build declares can ever have been installed — + # sending "off" for an undeclared mode would hit an unknown + # command; "off" is idempotent in uBridge. + for filter_type, mode in FILTER_EBPF_MODES.items(): + if mode in modes: + await self._ubridge_send(f'tc {mode}_drop "{host_ifc}" off') + return + unsupported = sorted(f for f, values in requested.items() if values and FILTER_EBPF_MODES[f] not in modes) + if unsupported: + if not modes: + raise self._kernel_error( + "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " + "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " + "the uBridge binary); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(unsupported)) + ) + raise self._kernel_error( + "Packet filter(s) {} on a kernel-datapath link need a newer uBridge (tc capabilities " + "reports ebpf_modes={}); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(unsupported), ",".join(modes)) + ) + if frequency: + # relay semantics: -1 = drop everything, N = every Nth packet + nth = 1 if int(frequency[0]) == -1 else int(frequency[0]) + await self._ubridge_send(f'tc nth_drop "{host_ifc}" {nth}') + elif "nth" in modes: + await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') + if quota: + await self._ubridge_send( + 'tc quota_drop "{ifc}" {bytes} {pct}'.format(ifc=host_ifc, bytes=int(quota[0]), pct=int(quota[1])) + ) + elif "quota" in modes: + await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') + if window: + # [start, outage, chance, period?, jitter?] — start is relative + # to this apply (the schedule restarts on every reconcile). + params = " ".join(str(int(v)) for v in window) + await self._ubridge_send(f'tc window_drop "{host_ifc}" {params}') + elif "window" in modes: + await self._ubridge_send(f'tc window_drop "{host_ifc}" off') diff --git a/pyproject.toml b/pyproject.toml index 6450b621e..696d785a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,7 +159,6 @@ extend-immutable-calls = [ "gns3server/api/routes/controller/users.py" = ["S105", "S106"] "gns3server/compute/base_node.py" = ["C901", "RUF019"] "gns3server/compute/docker/__init__.py" = ["B006", "C901"] -"gns3server/compute/docker/docker_kernel_datapath.py" = ["C901"] "gns3server/compute/docker/docker_vm.py" = ["B006", "C901", "E731", "RUF003", "RUF006", "S108"] "gns3server/compute/docker/iol_docker_vm.py" = ["RUF002", "S108"] "gns3server/compute/docker/vendor_docker_vm.py" = ["RUF002", "S108"] @@ -168,6 +167,7 @@ extend-immutable-calls = [ "gns3server/compute/dynamips/nodes/router.py" = ["F522", "RUF006"] "gns3server/compute/iou/iou_vm.py" = ["C901", "S108", "S324"] "gns3server/compute/iou/utils/iou_export.py" = ["C901", "RUF003"] +"gns3server/compute/kernel_datapath.py" = ["C901"] "gns3server/compute/project.py" = ["DTZ006", "S324"] "gns3server/compute/qemu/qemu_vm.py" = ["ASYNC220", "C901", "RUF003", "S603"] "gns3server/compute/ubridge/ubridge_hypervisor.py" = ["C901"] diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 058288889..f890b5fce 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -749,7 +749,7 @@ async def test_apply_bpf_drops_compile_failure_warns(vm): raise UbridgeError("209-Cannot compile filter 'icmp': can't parse filter expression: syntax error") vm._ubridge_send = AsyncioMagicMock(side_effect=send) - with patch("gns3server.compute.docker.docker_kernel_datapath.log.warning") as mock_log: + with patch("gns3server.compute.kernel_datapath.log.warning") as mock_log: await vm._ubridge_apply_bpf_drops(host_ifc, {"bpf": ["icmp"]}) # must not raise assert mock_log.called From 7d2d8c6747ad95350a766f01861214f8142c07a6 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 18:31:13 +0800 Subject: [PATCH 023/101] feat(qemu): anchor QEMU adapters on a persistent TAP QEMU had no host-side interface at all: every adapter was a local UDP tunnel into uBridge (-netdev socket plus two UDP ports per adapter), so a link could only ever be a userspace relay. Each adapter now owns a persistent TAP (gq{node_id[:8]}e{adapter}p{port}), created by uBridge before QEMU is launched and handed to this user, and the netdev is that TAP -- the anchor role Docker's veth host end plays. Links attach to it: * kernel link (NIOBridge): brctl create/up/addif enslaves the TAP into the per-link bridge and capture, markers and tc filters are installed on it, verbatim through the shared KernelDatapathMixin -- a TAP and a veth host end are interchangeable there (an egress qdisc on either impairs traffic entering the node, and PACKET_OUTGOING on either is the node receiving); * relay link: bridge add_nio_ethernet plus add_nio_udp -- AF_PACKET, never the TAP NIO, which would take the fd QEMU itself holds. The netdev type is fixed at launch, so this is all-or-nothing per VM: on the TAP datapath every link type attaches to (or detaches from) a running VM, including switching a link between the two datapaths. Stop un-persists the TAPs (QEMU's open fd keeps each device alive until it exits) and removes the per-link bridges; start sweeps leftovers first. A uBridge without the tap module keeps the previous socket-netdev datapath, so an unupgraded install is unaffected -- such a node simply cannot carry kernel links. --- gns3server/compute/docker/docker_vm.py | 13 +- gns3server/compute/kernel_datapath.py | 58 +++ gns3server/compute/qemu/qemu_vm.py | 284 +++++++++++--- .../compute/qemu/test_qemu_kernel_datapath.py | 348 ++++++++++++++++++ 4 files changed, 650 insertions(+), 53 deletions(-) create mode 100644 tests/compute/qemu/test_qemu_kernel_datapath.py diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index cf48f4ab7..25948a344 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1896,16 +1896,9 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): if self.status == "started": host_ifc = self._kernel_veths.get((adapter_number, port_number)) if host_ifc is not None: - # Incremental marker reconcile on the veth anchor — - # the relay-datapath equivalent of the branch below. - await self._ubridge_apply_markers(host_ifc, nio) - # The netem apply resets the interface first (the - # kernel merges optional netem attrs on replace), so - # everything anchored on clsact must re-apply after - # it: bpf drops flush + re-add, eBPF modes re-set. - await self._ubridge_apply_netem(host_ifc, nio.filters) - await self._ubridge_apply_bpf_drops(host_ifc, nio.filters) - await self._ubridge_apply_ebpf_drops(host_ifc, nio.filters) + # Incremental marker + filter reconcile on the anchor + # — the relay-datapath equivalent of the branch below. + await self._kernel_update(host_ifc, nio) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) return diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index 1567b61fa..a993ac1fa 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -171,6 +171,64 @@ class KernelDatapathMixin: await self._ubridge_apply_bpf_drops(anchor, nio.filters) await self._ubridge_apply_ebpf_drops(anchor, nio.filters) + async def _relay_attach(self, anchor, bridge_name, nio): + """ + Attach an anchor to a uBridge relay bridge (NIOUDP): the bridge turns + the anchor into an AF_PACKET endpoint and relays it to the link's UDP + peer. Called on link creation and on node start, and after a link was + re-created — the relay analogue of _kernel_attach. + """ + + await self._ubridge_send(f"bridge create {bridge_name}") + # libpcap cannot open a packet socket on an admin-down interface (its + # netlink promiscuous-mode transaction returns ENOENT), and an anchor + # is born down (carrier off until a link attaches) — bring it up for + # the relay attach. The carrier pass in the caller refines the state + # afterwards (a suspended NIO sets it back down). + await self._ubridge_send(f'link set "{anchor}" up') + await self._ubridge_send(f'bridge add_nio_ethernet {bridge_name} "{anchor}"') + await self._ubridge_send( + "bridge add_nio_udp {bridge_name} {lport} {rhost} {rport}".format( + bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport + ) + ) + if nio.capturing: + await self._ubridge_send( + 'bridge start_capture {bridge_name} "{pcap_file}"'.format( + bridge_name=bridge_name, pcap_file=nio.pcap_output_file + ) + ) + await self._ubridge_send(f"bridge start {bridge_name}") + await self._ubridge_apply_filters(bridge_name, nio.filters) + await self._ubridge_apply_markers(bridge_name, nio) + + async def _relay_detach(self, bridge_name): + """ + Drop a relay bridge: its AF_PACKET endpoint on the anchor goes with + it, which is what makes switching an adapter from the relay datapath + to a kernel link safe (otherwise the anchor would both be a bridge + port and a relay endpoint, duplicating one direction). + """ + + if self.ubridge: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f"bridge delete {bridge_name}") + + async def _kernel_update(self, anchor, nio): + """ + Re-apply everything a kernel link carries on an already-attached + anchor (filters and markers changed): no re-enslaving, no bridge + create. + """ + + await self._ubridge_apply_markers(anchor, nio) + # The netem apply resets the interface first (the kernel merges + # optional netem attrs on replace), so everything anchored on clsact + # must re-apply after it: bpf drops flush + re-add, eBPF modes re-set. + await self._ubridge_apply_netem(anchor, nio.filters) + await self._ubridge_apply_bpf_drops(anchor, nio.filters) + await self._ubridge_apply_ebpf_drops(anchor, nio.filters) + async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): """ Detach an anchor from its per-link kernel bridge. Both endpoints run diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 48e25ab7c..e8c73774e 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -48,6 +48,8 @@ from ...utils.images import md5sum from ..adapters.ethernet_adapter import EthernetAdapter from ..base_node import BaseNode from ..error import ImageMissingError, NodeError +from ..kernel_datapath import KernelDatapathMixin +from ..nios.nio_bridge import NIOBridge from ..nios.nio_tap import NIOTAP from ..nios.nio_udp import NIOUDP from .qemu_error import QemuError @@ -61,7 +63,7 @@ FORBIDDEN_OPTIONS = {"-blockdev", "-drive", "-hda", "-hdb", "-hdc", "-hdd", "-fs FORBIDDEN_OPTIONS |= {"-" + opt for opt in FORBIDDEN_OPTIONS if opt.startswith("-") and not opt.startswith("--")} -class QemuVM(BaseNode): +class QemuVM(KernelDatapathMixin, BaseNode): module_name = "qemu" """ @@ -118,7 +120,12 @@ class QemuVM(BaseNode): self._stdout_file = "" self._qemu_img_stdout_file = "" self._execute_lock = asyncio.Lock() - self._local_udp_tunnels = {} + self._local_udp_tunnels = {} # legacy datapath only (see _tap_datapath) + self._kernel_taps = {} + self._ubridge_tc_caps = None + # Whether this VM runs on the TAP datapath (uBridge created the + # adapter TAPs) or on the legacy socket-netdev one. Decided at start. + self._tap_datapath = False self._guest_cid = None self._command_line_changed = False self._qemu_version = None @@ -1118,6 +1125,14 @@ class QemuVM(BaseNode): if self._tpm: await self._start_swtpm() + # uBridge owns the adapter TAPs and must be up before they are + # created — and before QEMU opens them. Unlike a container, whose + # veths are created after it starts, the TAP has to pre-exist the + # process that reads it. + self._ubridge_tc_caps = None + await self._start_ubridge() + await self._prepare_tap_datapath() + command = await self._build_command() command_string = " ".join(shlex.quote(s) for s in command) try: @@ -1137,6 +1152,9 @@ class QemuVM(BaseNode): except (OSError, subprocess.SubprocessError, UnicodeEncodeError) as e: stdout = self.read_stdout() log.error(f"Could not start QEMU {self.qemu_path}: {e}\n{stdout}") + # uBridge was started (and the adapter TAPs created) before + # the launch: don't leave either behind on a failed start. + await self._stop_ubridge() raise QemuError(f"Could not start QEMU {self.qemu_path}: {e}\n{stdout}") await self._set_process_priority() @@ -1145,14 +1163,12 @@ class QemuVM(BaseNode): if "-enable-kvm" in command_string or "-enable-hax" in command_string: self._hw_virtualization = True - await self._start_ubridge() set_link_commands = [] for adapter_number, adapter in enumerate(self._ethernet_adapters): nio = adapter.get_nio(0) if nio: - await self.add_ubridge_udp_connection( - f"QEMU-{self._id}-{adapter_number}", self._local_udp_tunnels[adapter_number][1], nio - ) + await self._connect_nio(adapter_number, nio) + await self._set_adapter_carrier(adapter_number, not nio.suspend) if nio.suspend and self._replicate_network_connection_state: set_link_commands.append(f"set_link gns3-{adapter_number} off") elif self._replicate_network_connection_state: @@ -1472,6 +1488,156 @@ class QemuVM(BaseNode): else: log.debug(f"QEMU VM is not paused to be resumed, current status is {vm_status}") + # ------------------------------------------------------------------ + # Kernel datapath: the adapter anchor is a persistent TAP + # ------------------------------------------------------------------ + + def _tap_name(self, adapter_number, port_number=0): + """ + Deterministic anchor TAP name for an adapter port. The ``gq`` prefix + keeps it out of the ``gns3`` bridge/TAP name space (and apart from + Docker's ``gv``/``gc`` veth names); 8 hex chars of the node id plus + adapter/port keep it unique and within IFNAMSIZ (15). + """ + + return f"gq{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" + + def _kernel_host_ifc(self, adapter_number, port_number=0): + """ + The persistent TAP this adapter owns, or None before the VM started + (or on the legacy socket-netdev datapath). + """ + + return self._kernel_taps.get((adapter_number, port_number)) + + def _kernel_anchors(self): + return set(self._kernel_taps.values()) + + def _kernel_error(self, message): + return QemuError(message) + + async def _prepare_tap_datapath(self): + """ + Probe uBridge's tap module and, when it answers, create the TAP every + adapter owns — before QEMU is launched, so the netdev is already + there. A build without the module (or without CAP_NET_ADMIN) keeps the + legacy socket-netdev datapath: the local UDP tunnels to a uBridge + relay this node used before the TAP work. Kernel links require the TAP + datapath. + """ + + self._tap_datapath = False + self._kernel_taps.clear() + if not self.ubridge: + return + + probe = f"gq{self._id.replace('-', '')[:8]}prob" + try: + await self._ubridge_send(f'tap create "{probe}"') + except UbridgeError as e: + log.warning("QEMU VM '%s': uBridge cannot create persistent TAPs (%s)", self._name, e) + self.project.emit( + "log.warning", + { + "message": f"QEMU VM '{self._name}': uBridge does not support persistent TAPs ({e}); " + "this node runs on the legacy relay datapath and cannot carry kernel links" + }, + ) + return + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{probe}"') + + self._tap_datapath = True + await self._create_taps() + + async def _create_taps(self): + """ + Create the persistent TAP every adapter owns. Each adapter is born as + a TAP whatever its NIO type — the anchor role Docker's veth host end + plays — so a link attaches to (or detaches from) a running VM without + touching its interfaces. uBridge creates the device and hands + ownership to this user, so the unprivileged QEMU process can open it; + the TAP starts DOWN (carrier off until a link attaches). A persistent + TAP outlives its creator, so a leftover from a previous run (crash, + kill) is swept first. + """ + + for adapter_number, adapter in enumerate(self._ethernet_adapters): + tap = self._tap_name(adapter_number) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._ubridge_send(f'tap create "{tap}"') + try: + await self._ubridge_send(f"tap set_owner {tap} {os.getuid()}") + except UbridgeError as e: + # Only root could open the TAP now: the VM would fail to + # attach its netdev, which is worth surfacing here rather + # than as a launch failure. + log.warning("QEMU VM '%s': could not hand TAP %s to uid %s: %s", self._name, tap, os.getuid(), e) + await self._ubridge_send(f'link set "{tap}" down') + self._kernel_taps[(adapter_number, 0)] = tap + + async def _remove_taps(self): + """ + Delete the adapter TAPs together with the per-link kernel bridges + this VM still holds: a stopped VM has no links attached, so those + bridges are orphans whose ports just disappeared. + """ + + if self.ubridge: + for tap in self._kernel_taps.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._remove_kernel_bridges() + self._kernel_taps.clear() + + async def _stop_ubridge(self): + """ + Stops uBridge, removing the adapter TAPs first: they live in the root + namespace and outlive it (the tap module needs the control channel to + delete them), and QEMU holding the other end keeps each device alive + until it exits — unpersisted here, they disappear with the process. + The next start spawns a fresh uBridge, so the tc capabilities must be + probed again. + """ + + if self.ubridge: + await self._remove_taps() + self._ubridge_tc_caps = None + await super()._stop_ubridge() + + async def _connect_nio(self, adapter_number, nio, port_number=0): + """ + Attach a link's NIO to the adapter's TAP anchor: a kernel link + enslaves the TAP into the per-link kernel bridge, a relay link + attaches it to the uBridge relay bridge as an AF_PACKET endpoint. + """ + + bridge_name = f"QEMU-{self._id}-{adapter_number}" + if not self._tap_datapath: + if isinstance(nio, NIOBridge): + raise QemuError( + "Adapter {adapter_number} of QEMU VM '{name}' runs on the legacy relay datapath " + "(this uBridge cannot create persistent TAPs) and cannot carry a kernel link".format( + adapter_number=adapter_number, name=self._name + ) + ) + await self.add_ubridge_udp_connection(bridge_name, self._local_udp_tunnels[adapter_number][1], nio) + return + + anchor = self._kernel_host_ifc(adapter_number, port_number) + if anchor is None: + raise QemuError( + "Adapter {adapter_number} port {port_number} of QEMU VM '{name}' has no TAP interface; " + "restart the node to attach a link to it".format( + adapter_number=adapter_number, port_number=port_number, name=self._name + ) + ) + if isinstance(nio, NIOBridge): + await self._kernel_attach(anchor, nio) + return + await self._relay_attach(anchor, bridge_name, nio) + async def adapter_add_nio_binding(self, adapter_number, nio): """ Adds an adapter NIO binding. @@ -1487,13 +1653,16 @@ class QemuVM(BaseNode): if self.is_running(): try: - await self.add_ubridge_udp_connection( - f"QEMU-{self._id}-{adapter_number}", self._local_udp_tunnels[adapter_number][1], nio - ) - if self._replicate_network_connection_state: - await self._control_vm(f"set_link gns3-{adapter_number} on") + await self._connect_nio(adapter_number, nio) except (IndexError, KeyError): - raise QemuError(f'Adapter {adapter_number} does not exist on QEMU VM "{self._name}"') + raise QemuError( + 'Adapter {adapter_number} does not exist on QEMU VM "{name}"'.format( + name=self._name, adapter_number=adapter_number + ) + ) + await self._set_adapter_carrier(adapter_number, not nio.suspend) + if self._replicate_network_connection_state: + await self._control_vm(f"set_link gns3-{adapter_number} on") adapter.add_nio(0, nio) log.debug(f'QEMU VM "{self._name}" [{self._id}]: {nio} added to adapter {adapter_number}') @@ -1508,9 +1677,21 @@ class QemuVM(BaseNode): if self.is_running(): try: - await self.update_ubridge_udp_connection( - f"QEMU-{self._id}-{adapter_number}", self._local_udp_tunnels[adapter_number][1], nio - ) + if self._tap_datapath and isinstance(nio, NIOBridge): + # Filters or markers changed on an attached kernel link: + # re-apply on the anchor, no re-enslaving. + anchor = self._kernel_host_ifc(adapter_number) + if anchor is not None: + await self._kernel_update(anchor, nio) + elif self._tap_datapath: + # Relay link on the TAP anchor: re-apply on its bridge. + await self._ubridge_apply_filters(f"QEMU-{self._id}-{adapter_number}", nio.filters) + await self._ubridge_apply_markers(f"QEMU-{self._id}-{adapter_number}", nio) + else: + await self.update_ubridge_udp_connection( + f"QEMU-{self._id}-{adapter_number}", self._local_udp_tunnels[adapter_number][1], nio + ) + await self._set_adapter_carrier(adapter_number, not nio.suspend) if self._replicate_network_connection_state: if nio.suspend: await self._control_vm(f"set_link gns3-{adapter_number} off") @@ -1533,13 +1714,15 @@ class QemuVM(BaseNode): except IndexError: raise QemuError(f'Adapter {adapter_number} does not exist on QEMU VM "{self._name}"') + nio = adapter.get_nio(0) await self.stop_capture(adapter_number) if self.is_running(): if self._replicate_network_connection_state: await self._control_vm(f"set_link gns3-{adapter_number} off") - await self._ubridge_send("bridge delete {name}".format(name=f"QEMU-{self._id}-{adapter_number}")) - - nio = adapter.get_nio(0) + if isinstance(nio, NIOBridge): + await self._remove_kernel_nio(nio, adapter_number) + else: + await self._relay_detach(f"QEMU-{self._id}-{adapter_number}") if isinstance(nio, NIOUDP): self.manager.port_manager.release_udp_port(nio.lport, self._project) adapter.remove_nio(0) @@ -1582,11 +1765,16 @@ class QemuVM(BaseNode): nio.start_packet_capture(output_file) if self.ubridge: - await self._ubridge_send( - 'bridge start_capture {name} "{output_file}"'.format( - name=f"QEMU-{self._id}-{adapter_number}", output_file=output_file + anchor = self._kernel_host_ifc(adapter_number) + if isinstance(nio, NIOBridge) and anchor is not None: + # Kernel link: capture on the adapter's TAP anchor (AF_PACKET). + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + else: + await self._ubridge_send( + 'bridge start_capture {name} "{output_file}"'.format( + name=f"QEMU-{self._id}-{adapter_number}", output_file=output_file + ) ) - ) log.debug(f"QEMU VM '{self.name}' [{self.id}]: starting packet capture on adapter {adapter_number}") @@ -1603,7 +1791,10 @@ class QemuVM(BaseNode): nio.stop_packet_capture() if self.ubridge: - await self._ubridge_send("bridge stop_capture {name}".format(name=f"QEMU-{self._id}-{adapter_number}")) + if isinstance(nio, NIOBridge) and self._kernel_host_ifc(adapter_number) is not None: + await self._ubridge_send("capture stop_kernel") + else: + await self._ubridge_send("bridge stop_capture {name}".format(name=f"QEMU-{self._id}-{adapter_number}")) log.debug(f"QEMU VM '{self.name}' [{self.id}]: stopping packet capture on adapter {adapter_number}") @@ -2387,10 +2578,7 @@ class QemuVM(BaseNode): for adapter_number, adapter in enumerate(self._ethernet_adapters): mac = int_to_macaddress(macaddress_to_int(self._mac_address) + adapter_number) - # use a local UDP tunnel to connect to uBridge instead - if adapter_number not in self._local_udp_tunnels: - self._local_udp_tunnels[adapter_number] = self._create_local_udp_tunnel() - nio = self._local_udp_tunnels[adapter_number][0] + nio = adapter.get_nio(0) custom_adapter = self._get_custom_adapter_settings(adapter_number) adapter_type = custom_adapter.get("adapter_type", self._adapter_type) @@ -2415,23 +2603,33 @@ class QemuVM(BaseNode): addr = pci_device_id % 32 device_string = f"{device_string},bus=pci-bridge{bridge_id},addr=0x{addr:02x}" pci_device_id += 1 - if nio: - network_options.extend(["-device", f"{device_string},netdev=gns3-{adapter_number}"]) - if isinstance(nio, NIOUDP): - network_options.extend( - [ - "-netdev", - "socket,id=gns3-{},udp={}:{},localaddr={}:{}".format( - adapter_number, nio.rhost, nio.rport, "127.0.0.1", nio.lport - ), - ] - ) - elif isinstance(nio, NIOTAP): - network_options.extend( - ["-netdev", f"tap,id=gns3-{adapter_number},ifname={nio.tap_device},script=no,downscript=no"] - ) + # Every adapter is backed by its own TAP (created before launch, + # see _create_taps): the netdev is a kernel-side interface the + # link attaches to, not a UDP socket into uBridge. A legacy + # NIOTAP NIO still names its own device. + if isinstance(nio, NIOTAP): + tap = nio.tap_device else: - network_options.extend(["-device", device_string]) + tap = self._kernel_host_ifc(adapter_number) or self._tap_name(adapter_number) + network_options.extend(["-device", f"{device_string},netdev=gns3-{adapter_number}"]) + if self._tap_datapath or isinstance(nio, NIOTAP): + network_options.extend( + ["-netdev", f"tap,id=gns3-{adapter_number},ifname={tap},script=no,downscript=no"] + ) + else: + # Legacy datapath (uBridge without the tap module): QEMU talks + # UDP to a local tunnel, and uBridge relays it to the peer. + if adapter_number not in self._local_udp_tunnels: + self._local_udp_tunnels[adapter_number] = self._create_local_udp_tunnel() + tunnel_nio = self._local_udp_tunnels[adapter_number][0] + network_options.extend( + [ + "-netdev", + "socket,id=gns3-{},udp={}:{},localaddr={}:{}".format( + adapter_number, tunnel_nio.rhost, tunnel_nio.rport, "127.0.0.1", tunnel_nio.lport + ), + ] + ) return network_options diff --git a/tests/compute/qemu/test_qemu_kernel_datapath.py b/tests/compute/qemu/test_qemu_kernel_datapath.py new file mode 100644 index 000000000..a568e4327 --- /dev/null +++ b/tests/compute/qemu/test_qemu_kernel_datapath.py @@ -0,0 +1,348 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Kernel-datapath wiring for QEMU: the persistent TAP every adapter owns, its +netdev, kernel-link enslavement, relay attach on the anchor, capture, and the +legacy socket-netdev fallback for a uBridge without the tap module. +""" + +import os +import stat +import sys + +import pytest +import pytest_asyncio + +from unittest.mock import MagicMock, patch + +from tests.utils import asyncio_patch, AsyncioMagicMock + +from gns3server.compute.qemu import Qemu +from gns3server.compute.qemu.qemu_vm import QemuVM +from gns3server.compute.qemu.qemu_error import QemuError +from gns3server.compute.nios.nio_bridge import NIOBridge +from gns3server.compute.ubridge.ubridge_error import UbridgeError + + +BRIDGE = "gns3a1b2c3d4e5f" +TAP0 = "gq00010203e0p0" # the vm fixture's node id, adapter 0 + + +@pytest_asyncio.fixture +async def manager(port_manager): + + m = Qemu.instance() + m.port_manager = port_manager + return m + + +@pytest.fixture +def fake_qemu_binary(monkeypatch, tmpdir): + + monkeypatch.setenv("PATH", str(tmpdir)) + bin_path = os.path.join(os.environ["PATH"], "qemu-system-x86_64") + with open(bin_path, "w+") as f: + f.write("1") + os.chmod(bin_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + return bin_path + + +@pytest_asyncio.fixture(scope="function") +async def vm(compute_project, manager, fake_qemu_binary): + + vm = QemuVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", compute_project, manager, qemu_path=fake_qemu_binary) + vm._process_priority = "normal" + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_hypervisor.is_running.return_value = True + vm._ubridge_send = AsyncioMagicMock() + return vm + + +def _running(vm): + """Make is_running() true without a real process.""" + + vm._process = MagicMock(returncode=None) + + +# --------------------------------------------------------------------------- +# TAP lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_creates_adapter_taps(vm): + """ + The tap module is probed once per start; on success every adapter gets its + persistent TAP, owned by this user (so unprivileged QEMU can open it) and + born down (carrier off until a link attaches). A leftover from a previous + run is swept first. + """ + + await vm._prepare_tap_datapath() + + assert vm._tap_datapath is True + assert vm._kernel_taps == {(0, 0): TAP0} + vm._ubridge_send.assert_any_call('tap create "gq00010203prob"') + vm._ubridge_send.assert_any_call('tap delete "gq00010203prob"') + vm._ubridge_send.assert_any_call(f'tap delete "{TAP0}"') + vm._ubridge_send.assert_any_call(f'tap create "{TAP0}"') + vm._ubridge_send.assert_any_call(f"tap set_owner {TAP0} {os.getuid()}") + vm._ubridge_send.assert_any_call(f'link set "{TAP0}" down') + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_falls_back_without_the_tap_module(vm): + """ + A uBridge build without the tap module keeps the legacy socket-netdev + datapath — the node still starts, it just cannot carry kernel links. + """ + + async def send(command): + if command.startswith("tap create"): + raise UbridgeError("202-Unknown command 'create'") + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + with patch.object(vm.project, "emit") as emit: + await vm._prepare_tap_datapath() + + assert vm._tap_datapath is False + assert vm._kernel_taps == {} + assert emit.called + + +@pytest.mark.asyncio +async def test_tap_name_fits_ifnamsiz(vm): + """ + The anchor name has to fit IFNAMSIZ (15) even for a late adapter. + """ + + vm.adapters = 32 + assert len(vm._tap_name(31)) <= 15 + + +@pytest.mark.asyncio +async def test_network_options_use_the_tap_anchor(vm): + """ + On the TAP datapath the netdev is the adapter's anchor, not a UDP socket. + """ + + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + + options = await vm._network_options() + + assert f"tap,id=gns3-0,ifname={TAP0},script=no,downscript=no" in options + assert not any(option.startswith("socket,") for option in options) + assert not vm._local_udp_tunnels + + +@pytest.mark.asyncio +async def test_network_options_keep_the_legacy_socket_netdev(vm): + """ + Without the tap module nothing changes: the local UDP tunnel into uBridge + is still the adapter's netdev. + """ + + vm._tap_datapath = False + + options = await vm._network_options() + + assert any(option.startswith("socket,id=gns3-0,udp=") for option in options) + assert vm._local_udp_tunnels + + +# --------------------------------------------------------------------------- +# Link attach / detach +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_connect_nio_kernel_link_enslaves_the_tap(vm): + """ + A kernel link enslaves the adapter's TAP into the per-link kernel bridge, + with the bridge brought up first (a bridge that is down keeps its ports + DISABLED and forwards nothing, silently). + """ + + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + await vm._connect_nio(0, nio) + + vm._ubridge_send.assert_any_call(f'brctl create "{BRIDGE}"') + vm._ubridge_send.assert_any_call(f'link set "{BRIDGE}" up') + vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP0}"') + + +@pytest.mark.asyncio +async def test_connect_nio_relay_uses_the_tap_as_af_packet_endpoint(vm): + """ + A relay link attaches the TAP to the uBridge relay bridge via AF_PACKET + (not the TAP NIO: that one takes the fd QEMU holds). + """ + + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + + await vm._connect_nio(0, nio) + + bridge = f"QEMU-{vm.id}-0" + vm._ubridge_send.assert_any_call(f"bridge create {bridge}") + vm._ubridge_send.assert_any_call(f'link set "{TAP0}" up') + vm._ubridge_send.assert_any_call(f'bridge add_nio_ethernet {bridge} "{TAP0}"') + vm._ubridge_send.assert_any_call(f"bridge add_nio_udp {bridge} 20000 127.0.0.1 20001") + vm._ubridge_send.assert_any_call(f"bridge start {bridge}") + + +@pytest.mark.asyncio +async def test_connect_nio_without_anchor_raises(vm): + """ + A kernel link on an adapter whose TAP does not exist (a VM started before + the TAP datapath) fails with an actionable message. + """ + + vm._tap_datapath = True + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + with pytest.raises(QemuError, match="no TAP interface"): + await vm._connect_nio(0, nio) + + +@pytest.mark.asyncio +async def test_legacy_datapath_rejects_kernel_link(vm): + """ + The legacy datapath has no anchor to enslave: a kernel link there is an + error, not a silent fallback to the relay. + """ + + vm._tap_datapath = False + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + with pytest.raises(QemuError, match="legacy relay datapath"): + await vm._connect_nio(0, nio) + + +@pytest.mark.asyncio +async def test_add_nio_binding_on_a_running_vm_attaches_and_brings_the_carrier_up(vm): + _running(vm) + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + await vm.adapter_add_nio_binding(0, nio) + + vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP0}"') + vm._ubridge_send.assert_any_call(f'link set "{TAP0}" up') + + +@pytest.mark.asyncio +async def test_update_nio_binding_reapplies_filters_on_the_anchor(vm): + """ + A filter update on an attached kernel link reconciles on the anchor + without re-enslaving it (no second addif). + """ + + _running(vm) + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"delay": [10]} + + await vm.adapter_update_nio_binding(0, nio) + + vm._ubridge_send.assert_any_call(f'tc reset "{TAP0}"') + vm._ubridge_send.assert_any_call(f'tc netem set "{TAP0}" delay 10') + assert not any(f'brctl addif' in str(c) for c in vm._ubridge_send.call_args_list) + + +@pytest.mark.asyncio +async def test_remove_nio_binding_detaches_the_anchor(vm): + """ + Removing a kernel link resets the anchor (no impairment survives on the + adapter) and detaches it from the bridge. + """ + + _running(vm) + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + with asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM.stop_capture"): + await vm.adapter_remove_nio_binding(0) + + vm._ubridge_send.assert_any_call(f'tc reset "{TAP0}"') + vm._ubridge_send.assert_any_call(f'brctl delif "{BRIDGE}" "{TAP0}"') + vm._ubridge_send.assert_any_call(f'brctl delete "{BRIDGE}"') + + +@pytest.mark.asyncio +async def test_stop_ubridge_removes_taps_and_orphan_bridges(vm): + """ + A stopped VM has no links attached, so the TAPs and any per-link bridge + it still holds are removed while the uBridge control channel is up. + """ + + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + with asyncio_patch("gns3server.compute.base_node.BaseNode._stop_ubridge") as stop: + await vm._stop_ubridge() + + vm._ubridge_send.assert_any_call(f'tap delete "{TAP0}"') + vm._ubridge_send.assert_any_call(f'brctl delete "{BRIDGE}"') + assert stop.called + assert vm._kernel_taps == {} + assert vm._ubridge_tc_caps is None + + +# --------------------------------------------------------------------------- +# Capture +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_capture_on_a_kernel_link_uses_the_anchor(vm): + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.start_capture(0, "/tmp/test.pcap") + vm._ubridge_send.assert_any_call(f'capture start_kernel {TAP0} "/tmp/test.pcap"') + + await vm.stop_capture(0) + vm._ubridge_send.assert_any_call("capture stop_kernel") + + +@pytest.mark.asyncio +async def test_capture_on_a_relay_link_uses_the_relay_bridge(vm): + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.start_capture(0, "/tmp/test.pcap") + vm._ubridge_send.assert_any_call(f'bridge start_capture QEMU-{vm.id}-0 "/tmp/test.pcap"') + + await vm.stop_capture(0) + vm._ubridge_send.assert_any_call(f"bridge stop_capture QEMU-{vm.id}-0") From 187201a08ca9726a4ff97ce1929fd2ab47928609 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 18:56:57 +0800 Subject: [PATCH 024/101] feat(controller): offer QEMU the kernel datapath, gated per compute Kernel eligibility becomes "both endpoints on one compute, each able to anchor": Docker always (adapters are born as veth pairs), QEMU when that compute's uBridge can create the persistent TAPs its adapters anchor on. The question is asked per compute, because the link needs both ends anchored -- so a mixed or half-upgraded deployment degrades to the relay instead of failing an attach. probe_tap_support() creates and deletes one throwaway persistent TAP through a scratch uBridge, cached by binary identity like the tc report; /capabilities exposes `ubridge_tap` (absent = unknown, which keeps links on the relay). The compute route for a QEMU adapter NIO took UDPNIO only, so a kernel link was rejected with 422: its body and response now accept Union[UDPNIO, BridgeNIO] exactly like Docker's. Stale "Docker-to-Docker" wording in the config, link and MCP descriptions now talks about anchors, and docs/features/qemu-kernel-datapath.md documents the datapath -- verified end-to-end on a real instance: 32/32 kernel-datapath checks and 29/29 on the relay variant. --- docs/features/docker-kernel-datapath.md | 7 +- docs/features/qemu-kernel-datapath.md | 160 ++++++++++++++++++ gns3server/agent/mcp/__init__.py | 7 +- gns3server/api/routes/compute/capabilities.py | 7 +- gns3server/api/routes/compute/qemu_nodes.py | 15 +- gns3server/compute/ubridge/tc_probe.py | 71 ++++++++ gns3server/controller/compute.py | 1 + gns3server/controller/udp_link.py | 47 +++-- gns3server/schemas/config.py | 5 +- gns3server/schemas/controller/capabilities.py | 5 + gns3server/schemas/controller/links.py | 5 +- tests/api/routes/compute/test_capabilities.py | 25 +++ tests/compute/ubridge/test_tc_probe.py | 32 +++- tests/controller/test_compute.py | 2 +- tests/controller/test_kernel_datapath_link.py | 95 ++++++++++- 15 files changed, 444 insertions(+), 40 deletions(-) create mode 100644 docs/features/qemu-kernel-datapath.md diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index fad77439a..0be8facef 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -364,8 +364,11 @@ bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capt enable_kernel_datapath = True ``` -Intended to grow into the global datapath switch as QEMU / IOU / Dynamips -migrate to kernel bridges (QEMU: tap enslaved per-link bridge, zero relay). +Intended to grow into the global datapath switch as IOU / Dynamips migrate to +kernel bridges. **QEMU has landed** on the same pattern (persistent TAP anchor +enslaved into the per-link bridge, zero relay) — see +`docs/features/qemu-kernel-datapath.md`; the shared half of this document's +compute machinery now lives in `gns3server/compute/kernel_datapath.py`. ## Verification diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md new file mode 100644 index 000000000..045b43dad --- /dev/null +++ b/docs/features/qemu-kernel-datapath.md @@ -0,0 +1,160 @@ + + +# QEMU kernel datapath + +QEMU adapters anchor on a **persistent TAP**, and links between two QEMU +nodes — or a QEMU node and a Docker container — on the same compute are wired +in the kernel (per-link Linux bridge) instead of the uBridge UDP relay. The +compute side is the same machinery Docker already uses: `KernelDatapathMixin` +(`gns3server/compute/kernel_datapath.py`) takes an *anchor interface* and does +not care whether it is a veth host end or a TAP. + +## Architecture + +Kernel link between two QEMU nodes on one compute: + +``` + QEMU A (fd) root namespace QEMU B (fd) + ┌──────────┐ -netdev ┌──────────────────┐ -netdev ┌──────────┐ + │ e1000 │◄───tap──────┤ gq{idA}e0p0 ────┼──► gns3{link_id_prefix} + └──────────┘ │ (bridge port) │ (bridge port) │ + │ per-link kernel │ gq{idB}e0p0 │ + │ bridge = "cable"│ │ + └──────────────────┘ +``` + +Relay link on the same anchor (cross-compute, mixed node types, or +`enable_kernel_datapath` off): + +``` + ┌──────────┐ tap ┌────────────────────────────┐ UDP ┌──────────┐ + │ QEMU A │◄────────►│ uBridge relay bridge │◄───────►│ peer │ + │ (fd) │ gq…e0p0 │ add_nio_ethernet ↔ add_ │ tunnel │ (any) │ + └──────────┘ │ nio_udp │ └──────────┘ + └────────────────────────────┘ +``` + +The relay attaches the TAP with `bridge add_nio_ethernet` (AF_PACKET, which +does not take the device over) — **never** `bridge add_nio_tap`, which would +claim the fd QEMU itself holds. + +## Why a TAP works like a veth host end + +| Event | Docker anchor (veth host end) | QEMU anchor (TAP) | +|---|---|---| +| Frames the node sends | arrive on the veth as `PACKET_HOST` | written to the tap fd, arrive as RX (`PACKET_HOST`) | +| Frames the peer sends | injected by uBridge, seen as `PACKET_OUTGOING` | injected by uBridge, seen as `PACKET_OUTGOING` | +| `tc` egress qdisc impairs | traffic entering the container | traffic entering the VM | +| Capture/marker anchor | the interface | the same interface | + +So every filter, capture and marker primitive carries over verbatim, and +`delay 100` on both anchors still measures ≈200 ms round trip while each +direction is impaired exactly once. + +## Datapath selection + +`UDPLink._kernel_datapath_eligible` needs both endpoints on one compute and +both able to anchor: + +* Docker always can (adapters are born as veth pairs); +* QEMU can when that compute's uBridge reports the `tap` module — asked per + compute through `/capabilities` (`ubridge_tap`, probed by creating and + deleting one throwaway TAP, cached by binary identity), because an old + uBridge leaves QEMU on the legacy socket-netdev datapath and cannot anchor + a kernel link. An unknown/failed probe keeps the link on the relay. + +Because the `-netdev` type is fixed when QEMU is launched, this is +all-or-nothing per VM: on the TAP datapath *every* link can attach to (or +detach from) a running VM, including switching a link between the two +datapaths (delete + re-create, or a project reopen). + +## Adapter lifecycle + +* **Birth (node start, before QEMU is launched)** — `_prepare_tap_datapath` + probes the tap module, then `_create_taps` sweeps any leftover (a persistent + TAP outlives a crash) and creates one TAP per adapter: + `tap create gq{node_id[:8]}e{adapter}p{port}` → `tap set_owner ` + (so the unprivileged QEMU process can open it) → `link set down` + (carrier off until a link attaches). QEMU then opens it as + `-netdev tap,id=gns3-{adapter},ifname=,script=no,downscript=no`. +* **Life (running)** — the TAP is never created or destroyed again. Link + create/delete/switch, suspend, capture and markers only change *what is + attached to it*. +* **Death (node stop)** — `_remove_taps` un-persists the TAPs (`tap delete`; + QEMU's open fd keeps each device alive until it exits) and deletes the + per-link kernel bridges the node still holds. Deleting a node never runs the + link-teardown path, so those bridges would otherwise stay behind as empty + orphans; a restart rebuilds them from the NIO. + +## Link operations + +| Operation | Kernel-datapath action | +|---|---| +| Create / attach | `brctl create` (EEXIST tolerated via `brctl show`) + `link set up` + `brctl addif ` + carrier up + markers + `tc netem set` (filters) | +| Delete / detach | marker teardown → `tc reset ` → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The TAP survives — the adapter keeps it for the next link | +| Reset (`POST /links/{id}/reset`) | delete + create (re-evaluates eligibility) | +| Suspend (`PUT /links/{id}` `{"suspend": true}`) | tap admin-down (writing to a down TAP fd fails with EIO, so the link is genuinely dead) + QMP `set_link gns3-N off` so the guest notices; resume restores both, and the netem qdisc survives the flap | + +## Capture, markers, filters + +Identical to Docker's kernel links, on the TAP: `capture start_kernel ` +(driven by the NIO type, not by a link flag), `marker add_kernel …` +(only the capture node's end), and one `tc netem set` per anchor so each +direction is impaired once. `frequency_drop`/`quota`/`window_drop` run as the +eBPF classifier on the anchor's clsact egress. + +## Legacy fallback + +A uBridge build without the `tap` module (or without `CAP_NET_ADMIN`) keeps +the previous datapath untouched: local UDP tunnels into uBridge, QEMU launched +with `-netdev socket`, and the node logs a warning that it cannot carry kernel +links. Such a node is never offered a kernel link, so nothing fails. + +## Verified + +End-to-end through a real server (isolated instance, two QEMU nodes with a +stub binary, the test playing the guest by holding the TAP fds and pushing +frames): **32/32 checks** on the kernel datapath and **29/29** on the relay +variant (`enable_kernel_datapath = false`). Kernel run highlights: + +* anchors created persistent and owned by the server user, QEMU's netdev is + the TAP, no local UDP tunnel left; `link_list` reports `kernel_datapath: true` + and one per-link bridge holds both taps; +* frames cross in both directions (~0.1 ms); `delay 100` ⇒ netem on both + anchors and a measured **100.3 ms** one-way; +* suspend ⇒ anchor DOWN, no traffic, resume ⇒ 100.3 ms again (filter kept); +* capture ⇒ a 252-byte pcap written by `capture start_kernel`; +* link delete ⇒ bridge gone, qdisc clean, anchor alive; re-create ⇒ 0.1 ms + with no residual impairment; node stop ⇒ no taps, no bridges left. + +Relay run the same script with the flag off: same lifecycle, filter measured +at 109.8 ms (applied by uBridge's bridge — no netem on the anchor, as +expected), no kernel bridge holding the tap. + +## Configuration + +```ini +[Server] +# Wire eligible links (both endpoints on one compute, both able to anchor) +# through kernel interfaces instead of the uBridge UDP relay. +enable_kernel_datapath = True +``` + +## Known caveat + +Anchors are host-side netdevs, so the kernel gives them an IPv6 link-local +address and emits MLD/DAD from them; those frames flood into the emulated +segment. The hardening requirement (anchors must be pure L2, no L3 identity) +is specified for uBridge in `docs/design/ubridge-l2-anchor-spec.md` — it +applies to QEMU TAPs and Docker veths alike. + +## Roadmap + +IOU (its fabric would need a kernel-side port in `iol_bridge`), Dynamips +(hypervisor-created taps, enslavable as they are), and the Ethernet switch / +cloud paths (their anchors are ubridge-owned by design) are still on the +relay; cross-compute kernel links need VXLAN/GENEVE encapsulation, which is +a separate project (it would serve Docker links the same way). diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index 1c5398233..232eb0193 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -801,9 +801,10 @@ async def link_update( Supported kwargs: - suspend: boolean - Suspend or resume the link - filters: dict - Packet filters (values MUST be positional arrays). Types - marked (K) run on kernel-datapath links only (Docker-to-Docker on one - compute) and are rejected with 409 on relay links; which types a given - link accepts is authoritatively answered by link_available_filters: + marked (K) run on kernel-datapath links only (both endpoints on one + compute with an anchor of their own) and are rejected with 409 on relay + links; which types a given link accepts is authoritatively answered by + link_available_filters: * frequency_drop: [N] - Drop every Nth packet (N: -1 to 32767; -1 = drop everything). Exact on kernel links (eBPF counter) * packet_loss: [chance, correl?] - Loss percentage (0-100); correl (K) diff --git a/gns3server/api/routes/compute/capabilities.py b/gns3server/api/routes/compute/capabilities.py index 055d0fb05..f2a51d702 100644 --- a/gns3server/api/routes/compute/capabilities.py +++ b/gns3server/api/routes/compute/capabilities.py @@ -25,7 +25,7 @@ from fastapi import APIRouter, Request from gns3server import schemas from gns3server.compute import MODULES -from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities +from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support from gns3server.utils.path import get_default_project_directory from gns3server.utils.tc_capabilities import usable_ebpf_modes from gns3server.version import __version__ @@ -52,6 +52,10 @@ async def get_capabilities(request: Request) -> dict: "cbpf": caps.get("cbpf") == "1", } + # Persistent TAP support: the anchor QEMU adapters need for the kernel + # datapath. None = unknown (probe failed or uBridge too old). + ubridge_tap = await probe_tap_support() + # record the controller hostname or IP address if request.client: request.app.state.controller_host = request.client.host @@ -64,4 +68,5 @@ async def get_capabilities(request: Request) -> dict: "disk_size": psutil.disk_usage(get_default_project_directory()).total, "node_types": node_types, "ubridge_tc": ubridge_tc, + "ubridge_tap": ubridge_tap, } diff --git a/gns3server/api/routes/compute/qemu_nodes.py b/gns3server/api/routes/compute/qemu_nodes.py index 17bcb3ca2..ca858d245 100644 --- a/gns3server/api/routes/compute/qemu_nodes.py +++ b/gns3server/api/routes/compute/qemu_nodes.py @@ -269,18 +269,19 @@ async def resume_qemu_node(node: QemuVM = Depends(dep_node)) -> None: @router.post( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def create_qemu_node_nio( *, adapter_number: int, port_number: int = Path(..., ge=0, le=0), - nio_data: schemas.UDPNIO, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], node: QemuVM = Depends(dep_node), -) -> schemas.UDPNIO: +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ - Add a NIO (Network Input/Output) to the node. + Add a NIO (Network Input/Output) to the node: a UDP relay NIO, or a + kernel-datapath bridge NIO the node enslaves its adapter TAP into. The port number on the Qemu node is always 0. """ @@ -292,16 +293,16 @@ async def create_qemu_node_nio( @router.put( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def update_qemu_node_nio( *, adapter_number: int, port_number: int = Path(..., ge=0, le=0), - nio_data: schemas.UDPNIO, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], node: QemuVM = Depends(dep_node), -) -> schemas.UDPNIO: +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Update a NIO (Network Input/Output) on the node. The port number on the Qemu node is always 0. diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py index 33f2962b1..f6ba7a3ec 100644 --- a/gns3server/compute/ubridge/tc_probe.py +++ b/gns3server/compute/ubridge/tc_probe.py @@ -49,6 +49,8 @@ log = logging.getLogger(__name__) # binary identity -> parsed report (None = probed and unusable) _cache = {} +# binary identity -> whether the tap module works (None = unknown) +_tap_cache = {} _lock = asyncio.Lock() @@ -112,3 +114,72 @@ async def _probe(path, config, timeout): with contextlib.suppress(Exception): await hypervisor.stop() shutil.rmtree(working_dir, ignore_errors=True) + + +async def probe_tap_support(timeout: float = 15.0): + """ + Whether this host's uBridge can create persistent TAPs (the ``tap`` + module), probed the same way as the tc report — a throwaway uBridge, the + answer cached by binary identity. QEMU adapters anchor on such a TAP, so + this is what makes a QEMU node kernel-datapath eligible; an old build + answers "Unknown command", reported as None (unknown) and cached as such. + """ + + config = Config.instance() + path = shutil.which(config.settings.Server.ubridge_path) + if not path: + return None + try: + stat = os.stat(path) + key = (path, stat.st_mtime_ns, stat.st_size) + except OSError: + return None + + async with _lock: + if key in _tap_cache: + return _tap_cache[key] + supported = await _probe_tap(path, config, timeout) + _tap_cache[key] = supported + return supported + + +async def _probe_tap(path, config, timeout): + """ + Spawn a throwaway uBridge and try to create (then delete) one persistent + TAP. The probe device is named after a uuid so a concurrent probe on the + same host cannot collide; a delete that fails leaves it behind, which is + worth a warning but not a failure of the probe. + """ + + working_dir = tempfile.mkdtemp(prefix="gns3-tap-probe-") + hypervisor = Hypervisor( + None, + path, + working_dir, + config.settings.Server.ubridge_control_transport, + config.settings.Server.host, + str(uuid.uuid4()), + ) + name = f"gns3tap{uuid.uuid4().hex[:6]}" + try: + + async def ask(): + await hypervisor.start() + await hypervisor.connect() + await hypervisor.send(f"tap create {name}") + try: + await hypervisor.send(f"tap delete {name}") + except UbridgeError as e: + log.warning("Probe TAP %s could not be deleted: %s", name, e) + return True + + try: + return await asyncio.wait_for(ask(), timeout=timeout) + except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e: + # Missing module, old build, or no CAP_NET_ADMIN: unknown. + log.debug("uBridge tap-module probe failed: %s", e) + return None + finally: + with contextlib.suppress(Exception): + await hypervisor.stop() + shutil.rmtree(working_dir, ignore_errors=True) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index ef2593f66..3c470b1bb 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -100,6 +100,7 @@ class Compute: "disk_size": 0, "node_types": [], "ubridge_tc": None, + "ubridge_tap": None, } self.name = name # Cache of interfaces on remote host diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index e2e9602d7..9eca6924a 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -163,30 +163,47 @@ class UDPLink(Link): def _kernel_datapath_eligible(self, node1, node2): """ - Whether this link can be wired on the kernel datapath (veth pairs - enslaved into a per-link Linux bridge) instead of the uBridge UDP - relay. Every GNS3 filter type has a kernel equivalent — netem for - delay/packet_loss/corrupt and the netem extensions, cls_bpf for bpf, - the eBPF stateful classifier for frequency_drop — so filters no - longer disqualify the kernel path. Capture and markers are served by - uBridge's AF_PACKET modules on the veth host end. + Whether this link can be wired on the kernel datapath (the adapters' + host-side anchors enslaved into a per-link Linux bridge) instead of + the uBridge UDP relay. Every GNS3 filter type has a kernel equivalent + — netem for delay/packet_loss/corrupt and the netem extensions, + cls_bpf for bpf, the eBPF stateful classifier for frequency_drop — so + filters no longer disqualify the kernel path. Capture and markers are + served by uBridge's AF_PACKET modules on the anchor. - Docker adapters are born as veth pairs (unified interface), so the - datapath is a runtime decision — links attach to running containers - too: the compute side enslaves the veth host end (brctl) or attaches - the relay to it (add_nio_ethernet) without touching the container's - interfaces. + Every adapter owns an anchor, so the datapath is a runtime decision — + links attach to running nodes too: the compute side enslaves the + anchor (brctl) or attaches the relay to it (add_nio_ethernet) without + touching the interfaces the node itself uses. Both endpoints must be + on the same compute (there is no kernel link across hosts) and both + must be able to anchor: Docker adapters are born as veth pairs, + QEMU adapters as persistent TAPs, which need uBridge's tap module — + asked of that compute, since an old uBridge leaves QEMU on the relay + datapath and cannot carry a kernel link at all. """ if not Config.instance().settings.Server.enable_kernel_datapath: return False - if node1.node_type != "docker" or node2.node_type != "docker": - return False if node1.compute.id != node2.compute.id: return False if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): return False - return True + return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2) + + @staticmethod + def _kernel_endpoint_ready(node): + """ + Whether a node can anchor a kernel link on its compute. + """ + + if node.node_type == "docker": + return True + if node.node_type != "qemu": + return False + capabilities = node.compute.capabilities or {} + # Strict True: an unreported capability (old uBridge, failed probe) + # keeps the link on the relay, where it always works. + return capabilities.get("ubridge_tap") is True async def _prepare(self): """ diff --git a/gns3server/schemas/config.py b/gns3server/schemas/config.py index fc83f39fd..e17084a0e 100644 --- a/gns3server/schemas/config.py +++ b/gns3server/schemas/config.py @@ -185,8 +185,9 @@ class ServerSettings(BaseModel): allow_raw_images: bool = Field(True, description="Allow raw images to be uploaded to the server") enable_kernel_datapath: bool = Field( True, - description="Wire eligible Docker-to-Docker links (same compute, no filters/markers) through " - "kernel veth/bridge interfaces instead of the uBridge UDP relay", + description="Wire eligible links (both endpoints on one compute, both able to anchor: Docker veth, " + "or QEMU when the compute's uBridge supports persistent TAPs) through kernel interfaces instead " + "of the uBridge UDP relay", ) auto_discover_images: bool = Field(True, description="Automatically discover images in the images directory") image_sync_interval: int = Field(900, ge=10, description="Seconds between automatic image inventory scans") diff --git a/gns3server/schemas/controller/capabilities.py b/gns3server/schemas/controller/capabilities.py index bd4b8832a..72786f755 100644 --- a/gns3server/schemas/controller/capabilities.py +++ b/gns3server/schemas/controller/capabilities.py @@ -50,3 +50,8 @@ class Capabilities(BaseModel): ubridge_tc: Optional[UbridgeTcCapabilities] = Field( None, description="uBridge tc-module capabilities; absent when the probe failed or uBridge has no tc module" ) + ubridge_tap: Optional[bool] = Field( + None, + description="Whether uBridge can create the persistent TAPs a QEMU adapter anchors on " + "(needed to wire QEMU links on the kernel datapath); absent when the probe failed", + ) diff --git a/gns3server/schemas/controller/links.py b/gns3server/schemas/controller/links.py index dccc33063..47c397855 100644 --- a/gns3server/schemas/controller/links.py +++ b/gns3server/schemas/controller/links.py @@ -83,8 +83,9 @@ class Link(LinkBase): link_type: Optional[LinkType] = None kernel_datapath: Optional[bool] = Field( None, - description="Read only property. True when this link is wired through kernel veth/bridge interfaces " - "(currently Docker-to-Docker on one compute) instead of the uBridge UDP relay", + description="Read only property. True when this link is wired through kernel interfaces " + "(veth or TAP anchors enslaved to a per-link Linux bridge, both endpoints on one compute) " + "instead of the uBridge UDP relay", ) capturing: Optional[bool] = Field(None, description="Read only property. True if a capture running on the link") capture_file_name: Optional[str] = Field( diff --git a/tests/api/routes/compute/test_capabilities.py b/tests/api/routes/compute/test_capabilities.py index f3216daea..4934ae6f2 100644 --- a/tests/api/routes/compute/test_capabilities.py +++ b/tests/api/routes/compute/test_capabilities.py @@ -28,6 +28,12 @@ from gns3server.version import __version__ pytestmark = pytest.mark.asyncio +async def _no_tap_probe(): + """The tap probe spawns uBridge — stub it out for payload assertions.""" + + return None + + class TestCapabilitiesRoutes: async def test_get(self, app: FastAPI, compute_client: AsyncClient, windows_platform, monkeypatch) -> None: @@ -35,6 +41,7 @@ class TestCapabilitiesRoutes: return None monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -59,6 +66,7 @@ class TestCapabilitiesRoutes: "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, "ubridge_tc": None, + "ubridge_tap": None, } async def test_get_on_gns3vm(self, app: FastAPI, compute_client: AsyncClient, on_gns3vm, monkeypatch) -> None: @@ -67,6 +75,7 @@ class TestCapabilitiesRoutes: return None monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -91,6 +100,7 @@ class TestCapabilitiesRoutes: "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, "ubridge_tc": None, + "ubridge_tap": None, } async def test_get_reports_ubridge_tc(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: @@ -99,6 +109,7 @@ class TestCapabilitiesRoutes: return {"netem": "delay,rate", "ebpf": "1", "cbpf": "0", "ebpf_modes": "nth,quota,window,flow"} monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", fake_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_tc"] == { @@ -107,3 +118,17 @@ class TestCapabilitiesRoutes: "ebpf_modes": ["nth", "quota", "window", "flow"], "cbpf": False, } + + async def test_get_reports_ubridge_tap(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: + """ + The tap capability is what makes a QEMU node kernel-eligible, so it + travels with the other compute capabilities. + """ + + async def fake_tap_probe(): + return True + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", fake_tap_probe) + response = await compute_client.get(app.url_path_for("compute:get_capabilities")) + assert response.status_code == status.HTTP_200_OK + assert response.json()["ubridge_tap"] is True diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py index 6bb5178cd..3a7d2a640 100644 --- a/tests/compute/ubridge/test_tc_probe.py +++ b/tests/compute/ubridge/test_tc_probe.py @@ -22,7 +22,7 @@ payload: spawn a throwaway uBridge, ask once, cache by binary identity. import pytest from gns3server.compute.ubridge import tc_probe -from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities +from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support from gns3server.compute.ubridge.ubridge_error import UbridgeError pytestmark = pytest.mark.asyncio @@ -31,6 +31,8 @@ pytestmark = pytest.mark.asyncio class FakeHypervisor: """Records spawns; answers `tc capabilities` with a canned reply.""" + last = None + reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] error = None spawned = 0 @@ -38,6 +40,7 @@ class FakeHypervisor: def __init__(self, project, path, working_dir, transport, host, node_id): self.commands = [] + FakeHypervisor.last = self FakeHypervisor.spawned += 1 async def start(self): @@ -66,6 +69,8 @@ def probe_env(monkeypatch, tmp_path): FakeHypervisor.error = None FakeHypervisor.reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] tc_probe._cache.clear() + tc_probe._tap_cache.clear() + FakeHypervisor.last = None # a real file on disk: the cache is keyed on its (path, mtime, size) binary = tmp_path / "ubridge" binary.write_bytes(b"") @@ -113,3 +118,28 @@ async def test_probe_empty_reply_means_unknown(): FakeHypervisor.reply = [] assert await probe_tc_capabilities() is None + + +async def test_probe_tap_support_creates_and_deletes_a_probe_tap(): + + assert await probe_tap_support() is True + assert FakeHypervisor.spawned == 1 + commands = FakeHypervisor.last.commands + assert commands[0].startswith("tap create ") + assert commands[1].startswith("tap delete ") + + +async def test_probe_tap_support_unknown_on_an_old_build(): + + FakeHypervisor.error = "202-Unknown command 'create'" + assert await probe_tap_support() is None + # cached like the tc report: no respawn on every /capabilities hit + assert await probe_tap_support() is None + assert FakeHypervisor.spawned == 1 + + +async def test_probe_tap_support_missing_binary(monkeypatch): + + monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) + assert await probe_tap_support() is None + assert FakeHypervisor.spawned == 0 diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index a6f41e2d3..1fce70eb1 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -381,7 +381,7 @@ async def test_json(compute): "disk_usage_percent": 0, "connected": True, "last_error": None, - "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None}, + "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None}, } assert compute.asdict(topology_dump=True) == { "compute_id": "my_compute_id", diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 1ce85ac30..e714994dd 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -90,18 +90,101 @@ async def test_kernel_datapath_eligible(project): assert link._kernel_datapath_eligible(node1, node2) is True -@pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_for_other_node_types(project): - +def _tap_capable_compute(tap_support=True, compute_id="compute-1"): compute = MagicMock() - compute.id = "compute-1" - node1 = _node(project, compute, "docker1") - node2 = _node(project, compute, "qemu1", node_type="qemu") + compute.id = compute_id + compute.capabilities = {"ubridge_tap": tap_support} + return compute + + +async def _link(project, node1, node2): link = UDPLink(project) await link.add_node(node1, 0, 0, batch=True, dump=False) await link.add_node(node2, 0, 0, batch=True, dump=False) + return link + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_other_node_types(project): + """ + A node type with no anchor of its own (IOU, VPCS, cloud, ...) keeps the + relay however capable the compute is — and a QEMU node on a compute whose + uBridge cannot create persistent TAPs does too, because it runs on the + legacy relay datapath there. + """ + + compute = _tap_capable_compute(tap_support=None) + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "qemu1", node_type="qemu") + link = await _link(project, node1, node2) assert link._kernel_datapath_eligible(node1, node2) is False + compute = _tap_capable_compute() + node1 = _node(project, compute, "iou1", node_type="iou") + node2 = _node(project, compute, "docker1") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_for_qemu_with_tap_support(project): + """ + QEMU adapters anchor on a persistent TAP, so a compute whose uBridge has + the tap module gets kernel links too — including the mixed docker-to-qemu + case (both anchors end up in the same per-link kernel bridge). + """ + + compute = _tap_capable_compute() + node1 = _node(project, compute, "qemu1", node_type="qemu") + node2 = _node(project, compute, "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True + + node2 = _node(project, compute, "docker1") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True + + +@pytest.mark.asyncio +async def test_kernel_datapath_capability_is_asked_per_compute(project): + """ + A QEMU node on a compute without the tap module keeps the link on the + relay even when its peer sits on a capable compute — the link needs both + endpoints anchored, so the capability is a two-end AND. + """ + + capable = _tap_capable_compute(compute_id="compute-1") + plain = _tap_capable_compute(tap_support=None, compute_id="compute-2") + node1 = _node(project, capable, "qemu1", node_type="qemu") + node2 = _node(project, plain, "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + # same compute, but the compute cannot anchor: still the relay + node2 = _node(project, _tap_capable_compute(tap_support=None), "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_prepare_qemu_kernel_link_emits_bridge_nios(project): + """ + The prepared link data is the same NIOBridge spec Docker links use — no + UDP ports, no peer resolution — so nothing else in the controller has to + know which node type the endpoint is. + """ + + compute = _tap_capable_compute() + node1 = _node(project, compute, "qemu1", node_type="qemu") + node2 = _node(project, compute, "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + + await link._prepare() + + assert link.kernel_datapath is True + assert [d["type"] for d in link._link_data] == ["nio_bridge", "nio_bridge"] + assert link._link_data[0]["bridge"] == link._link_data[1]["bridge"] + @pytest.mark.asyncio async def test_kernel_datapath_not_eligible_across_computes(project): From a1fcd29bce150485fe5b69cea24adbbbfc011ad3 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 19:00:03 +0800 Subject: [PATCH 025/101] fix(qemu): build a command line for a headless node _console_options() fell through the "none" branch and returned None, so _build_command() blew up on `command.extend(None)`: a QEMU node with console_type "none" -- offered by the GUI, and what API/MCP-driven labs use to skip the console entirely -- failed to start with a 500 and a traceback. Return an empty list, exactly like _aux_options() already does, and cover it with a test that builds the command line for a headless node. Verified on a live instance: such a node now starts (204) with `console: null` and no -serial device in the QEMU command line. --- gns3server/compute/qemu/qemu_vm.py | 1 + tests/compute/qemu/test_qemu_vm.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index e8c73774e..9309e25b5 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -2042,6 +2042,7 @@ class QemuVM(KernelDatapathMixin, BaseNode): return self._spice_with_agent_options(self.console) elif self._console_type != "none": raise QemuError(f"Console type {self._console_type} is unknown") + return [] def _aux_options(self): diff --git a/tests/compute/qemu/test_qemu_vm.py b/tests/compute/qemu/test_qemu_vm.py index 7b39bc4ae..1f4953c27 100644 --- a/tests/compute/qemu/test_qemu_vm.py +++ b/tests/compute/qemu/test_qemu_vm.py @@ -624,6 +624,22 @@ async def test_build_command(vm, fake_qemu_binary): ] +@pytest.mark.asyncio +async def test_build_command_console_none(vm): + """ + A headless QEMU node (console_type none) builds a command line without a + serial device: the console options for "none" used to fall through and + return None, which made node start fail with a TypeError. + """ + + vm.console_type = "none" + os.environ["DISPLAY"] = "0:0" + with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()): + cmd = await vm._build_command() + assert "-serial" not in cmd + assert "-netdev" in cmd + + @pytest.mark.asyncio async def test_build_command_manual_uuid(vm): """ From 81ad9b837f55c068c946d1b8ac6fce515e817e3b Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 19:41:32 +0800 Subject: [PATCH 026/101] docs(spec): record the delivered L2-only anchors and their residual MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uBridge delivered `link l2only` (e2e3155) and the brctl delete refusal (83a84fb); verified from the server side -- anchors and per-link bridges report addrgenmode none, carry no addresses, and an idle anchor is silent once the one-shot membership burst has settled. That burst is the one thing the command cannot remove, so it is now recorded where the acceptance criteria live, with the measurement: enslaving a port makes the kernel announce the bridge's multicast memberships (90 B MLDv2 at +0.00 s, 54 B IGMPv3 at +0.09 s, 90 B MLDv2 at +0.73 s, then silence; no addresses involved, and IPv4 has no per-device IGMP switch). Decision: accepted and documented, not suppressed -- §E.2 measures idle silence after a settle window instead. Pins the other delivered behaviour with a test: a bridge now survives a single-sided node stop, so brctl create answers EEXIST on the common path rather than only in a create race -- the attach must not short-circuit there, because the restarted side has a brand-new anchor whose filters and markers still have to be re-applied. --- docs/design/ubridge-l2-anchor-spec.md | 53 +++++++++++++++---- docs/features/qemu-kernel-datapath.md | 12 ++++- .../docker/test_docker_kernel_datapath.py | 31 +++++++++++ 3 files changed, 84 insertions(+), 12 deletions(-) diff --git a/docs/design/ubridge-l2-anchor-spec.md b/docs/design/ubridge-l2-anchor-spec.md index 6fef6d106..8c6afad71 100644 --- a/docs/design/ubridge-l2-anchor-spec.md +++ b/docs/design/ubridge-l2-anchor-spec.md @@ -3,11 +3,17 @@ SPDX-License-Identifier: CC-BY-SA-4.0 See LICENSE file for licensing information. --> -> Frozen requirements spec for the **uBridge** project. Status: **not yet -> delivered.** Requested by gns3-server branch `feat/qemu-kernel-datapath` -> (QEMU persistent-TAP anchors) and, retroactively, by the shipped Docker -> kernel-datapath stack: today every host-side anchor device carries an IPv6 -> identity, so emulated links carry the *host's* IPv6 chatter. +> Frozen requirements spec for the **uBridge** project. Status: **delivered** +> (uBridge `e2e3155` — `link l2only` plus the creators that apply it) and +> verified from the server side: anchors and per-link bridges report +> `addrgenmode none`, `ip -4/-6 addr` are empty, and an idle anchor is silent +> once the one-shot membership burst of §0 has settled. +> +> **Residual, accepted (measured after delivery):** enslaving a port still +> makes the *bridge device* announce its multicast memberships — one IGMPv3 +> plus one or two MLDv2 reports in the first second — which floods into the +> emulated segment. It is outside this command's reach by design (no addresses +> are involved, and IPv4 has no per-device IGMP off switch); see §0. > > Scope guard: no wire format, no command grammar change to existing > commands — this adds one command and one internal step in four creators. @@ -53,6 +59,32 @@ invariant we keep is *no addresses at all* (measured: `ip -4 addr` is empty on every anchor and per-link bridge on the deployment above, with NetworkManager reporting them `unmanaged`). IPv6 is the only protocol that lights itself up. +### The residual this command does not remove (measured after delivery) + +With `l2only` in place (anchors and bridges report `addrgenmode none`, no +addresses), **enslaving a port still makes the bridge device speak once** — +the kernel announces the multicast memberships the enslavement brings to the +bridge: + +``` ++0.00 s 90 B IPv6 MLDv2 report dst 33:33:00:00:00:16 (ICMPv6 143) ++0.09 s 54 B IPv4 IGMPv3 report dst 01:00:5e:00:00:16 0.0.0.0 → 224.0.0.22 ++0.73 s 90 B IPv6 MLDv2 report dst 33:33:00:00:00:16 +``` + +(Scratch tap pair + per-link bridge on the test host, unprivileged; source MAC += the bridge's. A 10 s window contains exactly those three frames, then silence. +The same burst appears through the server: one 90 B IPv6 frame inside a 2 s +window right after a link is created.) + +It is outside `l2only`'s reach by design: no addresses are involved (the report +sources are `::` and `0.0.0.0`), and IPv4 has no per-device "no IGMP" knob to +set. **Decision: accepted and documented, not suppressed** — it is standard L2 +control traffic, one-shot per link creation or port change, and it stops. The +acceptance test in §E.2 therefore measures after a settle window. Suppressing +it (e.g. `disable_ipv6` on the bridge before enslaving, for the MLD half) would +be a new, separate requirement and has not been asked for. + ### Goal Every host-side device uBridge creates for the data plane is **pure L2**: no @@ -147,11 +179,12 @@ data plane hardens it, and the acceptance checks in §E apply to it. 1. **No addresses.** After each creator: `ip -6 addr show dev ` is empty (in particular no `fe80::`) and `ip -4 addr show dev ` is empty; the same for the bridge created by `brctl create`. -2. **Idle silence.** With the device UP and in its normal role (TAP with an open - fd / veth with its peer up / bridge with two attached ports), nothing appears - on it for 5 s — measured from the peer end, or via - `capture start_kernel ` while idle and by asserting the pcap has - zero packets. Baseline today: 6 frames / 2 s. +2. **Idle silence, after the settle window.** Let the last port enslavement + settle (≥ 2 s — that burst is expected, see §0), then with the device UP and + in its normal role (TAP with an open fd / veth with its peer up / bridge with + two attached ports) nothing appears on it for 5 s — measured from the peer + end, or via `capture start_kernel ` while idle with the pcap + asserted empty. Baseline before this work: 6 frames / 2 s, continuous. 3. **Idempotency & revert.** `on` twice → `100` both times, state unchanged; `off` restores the kernel default (link-local returns after the device is cycled down/up). diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 045b43dad..1797ca4c6 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -148,8 +148,16 @@ enable_kernel_datapath = True Anchors are host-side netdevs, so the kernel gives them an IPv6 link-local address and emits MLD/DAD from them; those frames flood into the emulated segment. The hardening requirement (anchors must be pure L2, no L3 identity) -is specified for uBridge in `docs/design/ubridge-l2-anchor-spec.md` — it -applies to QEMU TAPs and Docker veths alike. +is specified for uBridge in `docs/design/ubridge-l2-anchor-spec.md` and is +**delivered**: anchors and per-link bridges come up with `addrgenmode none`, +no addresses, and an idle anchor is silent (asserted in the e2e after a 2.5 s +settle). + +One residual is accepted and documented there: enslaving a port makes the +*kernel* announce the bridge's multicast memberships once — an IGMPv3 plus one +or two MLDv2 reports in the first second — which floods into the segment like +any L2 control frame. It stops; it is outside the L2-only command's reach (no +addresses involved, IPv4 has no per-device IGMP switch). ## Roadmap diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index f890b5fce..f68ddb11f 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -291,6 +291,37 @@ async def test_connect_nio_kernel_tolerates_bridge_create_race(vm): vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{host_ifc}"') +@pytest.mark.asyncio +async def test_connect_nio_kernel_eexist_still_reapplies_everything(vm): + """ + EEXIST is a *common* path now: a bridge survives a single-sided node + restart (the peer's still-enslaved port makes brctl delete answer EBUSY, + which is suppressed), so the restarted side finds its bridge already + there. The create-or-verify step must not short-circuit the rest of the + attach — the restarted side has a brand-new anchor and needs its capture, + markers and filters re-applied on it. + """ + + async def send(command): + if "brctl create" in command: + raise UbridgeError("Could not create bridge gns3a1b2c3d4e5f: File exists (EEXIST)") + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + nio.filters = {"delay": [10]} + + await vm._connect_nio(0, nio) + + vm._ubridge_send.assert_any_call(f'brctl show "{BRIDGE}"') + vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{host_ifc}"') + # ... and the port-level configuration of the fresh anchor goes on top of + # it, unconditionally (never inside a "the bridge was created" branch). + vm._ubridge_send.assert_any_call(f'tc reset "{host_ifc}"') + vm._ubridge_send.assert_any_call(f'tc netem set "{host_ifc}" delay 10') + + @pytest.mark.asyncio async def test_connect_nio_kernel_create_failure_propagates(vm): From 9a14a91293099b7cd3139f8e0bca886f5d972179 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 00:59:37 +0800 Subject: [PATCH 027/101] feat(iou): anchor every Ethernet bay/unit on a persistent TAP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IOU's only wire is the IOL fabric (Unix datagrams, 8-byte headers, NETMAP), so it has no host netdev to enslave — the kernel datapath gives each Ethernet bay/unit a persistent TAP anchor the uBridge IOL bridge binds the port to (iol_bridge add_nio_tap), which links then enslave into their per-link kernel bridge exactly like Docker's veth and QEMU's TAP. * IOUVM joins KernelDatapathMixin: one TAP per Ethernet bay/unit (gi{id8}e{bay}p{unit}), created at start after a capability probe and a stale sweep, born down and address-free; serial bays never anchor. * _networking dispatches per NIO type: a relay link keeps iol_bridge add_nio_udp, a kernel link binds the anchor then runs _kernel_attach. * Stop order is QEMU's reverse, because uBridge holds the anchor fds: iol_bridge delete (releases every port NIO) -> tap delete (a held TAP answers EBADFD) -> brctl delete -> hypervisor stop. Removing a single link releases its fd the same way (delete_nio_tap). * The IOU-specific marker reconcile override is gone: marker primitives are polymorphic on the anchor (marker *_kernel on a TAP, iol_bridge commands on an IOL location), so the base reconcile serves both. Capture on a kernel link anchors on the TAP (capture start_kernel); filters are tc on the anchor, never combined with the userspace iol_bridge filters a relay link uses. --- gns3server/compute/base_node.py | 3 +- gns3server/compute/iou/iou_vm.py | 438 ++++++++++++------ tests/compute/iou/test_iou_kernel_datapath.py | 413 +++++++++++++++++ 3 files changed, 723 insertions(+), 131 deletions(-) create mode 100644 tests/compute/iou/test_iou_kernel_datapath.py diff --git a/gns3server/compute/base_node.py b/gns3server/compute/base_node.py index 18f4cfdc3..56076e83c 100644 --- a/gns3server/compute/base_node.py +++ b/gns3server/compute/base_node.py @@ -1449,7 +1449,8 @@ class BaseNode: The anchor is resolved from the (name, link_id)→bridge map populated at apply time; entries are iterated so a node that hosts the same marker name on several links (e.g. IOU with one IOL-BRIDGE per node) toggles every - copy. IOU overrides this for its ``iol_bridge`` command shape. + copy. IOU's ``iol_bridge`` command shape lives in its + ``_ubridge_enable_marker_filter`` override. :param name: marker filter name :param enabled: True = on (signal+pcap), False = off (paused tap) diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index a4b6ddf80..7c61c7d18 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -36,6 +36,7 @@ import sys import gns3server.utils.asyncio import gns3server.utils.images +from gns3server.compute.ubridge.tc_probe import probe_iol_tap_support from gns3server.compute.ubridge.ubridge_error import UbridgeError from gns3server.utils.asyncio import locking from gns3server.utils.asyncio.ssh_server import AsyncioSSHServer @@ -46,6 +47,8 @@ from gns3server.utils.hostname import is_ios_hostname_valid from ..adapters.ethernet_adapter import EthernetAdapter from ..adapters.serial_adapter import SerialAdapter from ..base_node import BaseNode +from ..kernel_datapath import KernelDatapathMixin +from ..nios.nio_bridge import NIOBridge from ..nios.nio_udp import NIOUDP from .iou_error import IOUError from .utils.iou_export import nvram_export @@ -133,7 +136,7 @@ class IOUL1KeepaliveProtocol(asyncio.DatagramProtocol): log.debug('IOU "%s": could not send an L1 keepalive: %s', self._vm.name, e) -class IOUVM(BaseNode): +class IOUVM(KernelDatapathMixin, BaseNode): module_name = "iou" # Class-level caches shared across all IOU VM instances using the same image. @@ -189,6 +192,13 @@ class IOUVM(BaseNode): self._application_id = application_id self._l1_keepalives = False + # Kernel datapath (see KernelDatapathMixin): one persistent TAP per + # Ethernet bay/unit, opened and held by uBridge's iol_bridge while a + # kernel link attaches the port (iol_bridge add_nio_tap). + self._kernel_taps = {} + self._ubridge_tc_caps = None + self._tap_datapath = False + def _nvram_changed(self, path): """ Called when the NVRAM file has changed @@ -675,6 +685,7 @@ class IOUVM(BaseNode): await self._check_iou_license() await self._start_ubridge() + await self._prepare_tap_datapath() self._create_netmap_config() if self.use_default_iou_values: # make sure we have the default nvram amount to correctly push the configs @@ -784,10 +795,14 @@ class IOUVM(BaseNode): @locking async def _networking(self): """ - Configures the IOL bridge in uBridge. + Configures the IOL bridge in uBridge: every port's NIO is wired to + it — a relay link (NIOUDP) as an ``iol_bridge`` UDP NIO, a kernel + link (NIOBridge) by binding the port's anchor TAP to the bridge + (``iol_bridge add_nio_tap``) and enslaving that TAP into the link's + per-link kernel bridge. """ - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" + bridge_name = self._iol_bridge_name() try: # delete any previous bridge if it exists await self._ubridge_send(f"iol_bridge delete {bridge_name}") @@ -806,20 +821,175 @@ class IOUVM(BaseNode): ) if nio.capturing: await self._ubridge_send( - 'iol_bridge start_capture {name} "{output_file}" {data_link_type}'.format( + 'iol_bridge start_capture {name} {bay} {unit} "{output_file}" {data_link_type}'.format( name=bridge_name, + bay=bay_id, + unit=unit_id, output_file=nio.pcap_output_file, data_link_type=re.sub(r"^DLT_", "", nio.pcap_data_link_type), ) ) - await self._ubridge_apply_filters(bay_id, unit_id, nio.filters) - await self._ubridge_apply_markers(bay_id, unit_id, nio) + location = self._iol_location(bay_id, unit_id) + await self._ubridge_apply_filters(location, nio.filters) + await self._ubridge_apply_markers(location, nio) + elif nio and isinstance(nio, NIOBridge): + await self._attach_kernel_nio(bay_id, unit_id, nio) unit_id += 1 bay_id += 1 await self._ubridge_send(f"iol_bridge start {bridge_name}") + # ------------------------------------------------------------------ + # Kernel datapath: the port anchor is a persistent TAP held by uBridge + # ------------------------------------------------------------------ + + def _iol_bridge_name(self): + """ + Name of this node's IOL bridge in uBridge (the fake IOL instance + the NETMAP maps every bay/unit to). + """ + + return f"IOL-BRIDGE-{self.application_id + 512}" + + def _iol_location(self, adapter_number, port_number): + """ + uBridge's location token for one IOL port: ``iol_bridge`` commands + take the bridge name plus the bay/unit coordinates positionally. + """ + + return f"{self._iol_bridge_name()} {adapter_number} {port_number}" + + def _tap_name(self, adapter_number, port_number): + """ + Deterministic anchor TAP name for an Ethernet bay/unit. The ``gi`` + prefix keeps it out of the ``gns3`` bridge/TAP name space (and apart + from Docker's ``gv``/``gc`` and QEMU's ``gq``); 8 hex chars of the + node id plus bay/unit keep it unique and within IFNAMSIZ (15). + """ + + return f"gi{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" + + def _kernel_host_ifc(self, adapter_number, port_number=0): + """ + The persistent TAP this Ethernet bay/unit owns, or None on serial + bays (never anchored) and before the node started. + """ + + return self._kernel_taps.get((adapter_number, port_number)) + + def _kernel_anchors(self): + """ + Every anchor TAP this node currently owns. + """ + + return set(self._kernel_taps.values()) + + def _kernel_error(self, message): + return IOUError(message) + + async def _prepare_tap_datapath(self): + """ + Probe uBridge for the commands the kernel datapath needs (the tap + module plus ``iol_bridge add_nio_tap``, both covered by the compute + capability probe, cached by binary identity) and, when it answers, + create the persistent TAP every Ethernet bay/unit owns — the anchor + role QEMU's TAPs and Docker's veth host ends play. A uBridge + without them keeps this node on the relay datapath: every link + rides ``iol_bridge add_nio_udp``, exactly as before. + """ + + self._tap_datapath = False + self._kernel_taps.clear() + if not self.ubridge: + return + + if await probe_iol_tap_support() is not True: + message = ( + f'IOU "{self._name}": uBridge cannot bind an IOL port to a persistent TAP ' + "(iol_bridge add_nio_tap); this node runs on the relay datapath and cannot " + "carry kernel links" + ) + log.warning(message) + self.project.emit("log.warning", {"message": message}) + return + + self._tap_datapath = True + await self._create_taps() + + async def _create_taps(self): + """ + Create the persistent TAP every Ethernet bay/unit owns (four units + per bay, the IOU adapter model). uBridge holds a device's fd only + while a kernel link binds the port to it (``iol_bridge + add_nio_tap``); until then the TAP sits DOWN with no holder, and it + survives link churn untouched — links attach to it, never recreate + it. A persistent TAP outlives its creator, so a leftover from a + previous run (crash, kill) is swept first. + """ + + for adapter_number, adapter in enumerate(self._ethernet_adapters): + for port_number in adapter.ports.keys(): + tap = self._tap_name(adapter_number, port_number) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._ubridge_send(f'tap create "{tap}"') + await self._ubridge_send(f'link set "{tap}" down') + self._kernel_taps[(adapter_number, port_number)] = tap + + async def _attach_kernel_nio(self, adapter_number, port_number, nio): + """ + Wire a kernel link's NIO (NIOBridge) on a running node: bind the + port's anchor TAP to the IOL bridge (uBridge opens and holds the + fd, relaying the IOL fabric onto the device), then enslave the TAP + into the per-link kernel bridge — the shared mixin flow, with the + anchor playing the role Docker's veth and QEMU's TAP play. + """ + + anchor = self._kernel_host_ifc(adapter_number, port_number) + if anchor is None: + raise self._kernel_error( + "Bay {bay}/{unit} of IOU '{name}' has no TAP anchor to carry a kernel link " + "(serial ports stay on the relay datapath, and a node whose uBridge lacks " + "iol_bridge add_nio_tap runs relay-only)".format( + bay=adapter_number, unit=port_number, name=self._name + ) + ) + await self._ubridge_send( + 'iol_bridge add_nio_tap {name} {iol_id} {bay} {unit} "{tap}"'.format( + name=self._iol_bridge_name(), + iol_id=self.application_id, + bay=adapter_number, + unit=port_number, + tap=anchor, + ) + ) + await self._kernel_attach(anchor, nio) + await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) + + async def _stop_ubridge(self): + """ + Stops uBridge, releasing the kernel datapath's host state first — + in the reverse order of QEMU's stop, because here uBridge itself + holds the anchor TAPs open (``iol_bridge add_nio_tap``): the IOL + bridge must be deleted, closing every port's TAP fd, before + ``tap delete`` can succeed (a held device answers EBADFD), and the + per-link kernel bridges need the control channel as well. The next + start spawns a fresh uBridge, so the tc capabilities must be probed + again. + """ + + if self.ubridge: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f"iol_bridge delete {self._iol_bridge_name()}") + for tap in self._kernel_taps.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._remove_kernel_bridges() + self._kernel_taps.clear() + self._ubridge_tc_caps = None + await super()._stop_ubridge() + def _termination_callback(self, process_name, returncode): """ Called when the process has stopped. @@ -1179,12 +1349,23 @@ class IOUVM(BaseNode): log.debug(f'IOU "{self._name}" [{self._id}]: {nio} added to {adapter_number}/{port_number}') if self.ubridge: - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" + if isinstance(nio, NIOBridge): + await self._attach_kernel_nio(adapter_number, port_number, nio) + return await self._ubridge_send( - f"iol_bridge add_nio_udp {bridge_name} {self.application_id} {adapter_number} {port_number} {nio.lport} {nio.rhost} {nio.rport}" + "iol_bridge add_nio_udp {name} {iol_id} {bay} {unit} {lport} {rhost} {rport}".format( + name=self._iol_bridge_name(), + iol_id=self.application_id, + bay=adapter_number, + unit=port_number, + lport=nio.lport, + rhost=nio.rhost, + rport=nio.rport, + ) ) - await self._ubridge_apply_filters(adapter_number, port_number, nio.filters) - await self._ubridge_apply_markers(adapter_number, port_number, nio) + location = self._iol_location(adapter_number, port_number) + await self._ubridge_apply_filters(location, nio.filters) + await self._ubridge_apply_markers(location, nio) async def adapter_update_nio_binding(self, adapter_number, port_number, nio): """ @@ -1196,131 +1377,96 @@ class IOUVM(BaseNode): """ if self.ubridge: - await self._ubridge_apply_filters(adapter_number, port_number, nio.filters) - await self._ubridge_apply_markers(adapter_number, port_number, nio) + if isinstance(nio, NIOBridge): + # Filters or markers changed on an attached kernel link, or + # its suspend state did: re-apply on the anchor, no + # re-binding of the IOL port and no re-enslaving. + anchor = self._kernel_host_ifc(adapter_number, port_number) + if anchor is not None: + await self._kernel_update(anchor, nio) + await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) + return + location = self._iol_location(adapter_number, port_number) + await self._ubridge_apply_filters(location, nio.filters) + await self._ubridge_apply_markers(location, nio) - async def _ubridge_apply_filters(self, adapter_number, port_number, filters): + async def _ubridge_apply_filters(self, location, filters): """ - Apply filter like rate limiting + Apply relay packet filters on one IOL port (``iol_bridge`` + userspace filters). Kernel links never come here — their filters + run as tc on the port's anchor (see _attach_kernel_nio / + _kernel_update). - :param adapter_number: adapter number - :param port_number: port number + :param location: the port's IOL location ("{bridge} {bay} {unit}") :param filters: Array of filter dictionnary """ - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" - location = f"{bridge_name} {adapter_number} {port_number}" + await self._ubridge_send("iol_bridge reset_packet_filters " + location) for filter in self._build_filter_list(filters): cmd = f"iol_bridge add_packet_filter {location} {filter}" await self._ubridge_send(cmd) - async def _ubridge_apply_markers(self, adapter_number, port_number, nio): + async def _ubridge_add_marker_filter( + self, location, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None + ): """ - Reconcile traffic-insight markers on the IOL bridge (diff desired - ``nio.markers`` against installed ``_marker_specs``): delete removed, - rebuild changed, toggle on/off-only changes, add new, skip unchanged. - - IOU uses ``iol_bridge`` (not ``bridge``) and the ``add_packet_filter`` - command carries extra ``{bay} {unit}`` positional arguments between the - bridge name and the filter name — this override mirrors the pattern in - ``_ubridge_apply_filters`` above. - - :param adapter_number: bay id - :param port_number: unit id - :param nio: NIO instance carrying ``nio.markers`` + IOU's `mark` filter attach. The marker anchor is polymorphic: on the + relay datapath it is the port's IOL location, and ``iol_bridge + add_packet_filter`` carries extra ``{bay} {unit}`` positionals between + the bridge name and the filter name; on the kernel datapath it is the + port's TAP, and the mixin's ``marker add_kernel`` applies. """ - from gns3server.compute.marker.marker_manager import MarkerManager - markers = nio.markers if hasattr(nio, "markers") else {} - manager = MarkerManager.instance() - markers_dir = self.project.markers_working_directory() - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" - location = f"{bridge_name} {adapter_number} {port_number}" - desired = {(name, spec.get("link_id", "")): spec for name, spec in markers.items()} + if self._kernel_marker_anchor(location): + await super()._ubridge_add_marker_filter( + location, + name, + bpf, + pcap_path, + tag=tag, + link_id=link_id, + direction=direction, + data_link_type=data_link_type, + ) + return + self._validate_marker_name(name) + # iol_bridge add_packet_filter {br} {bay} {unit} {name} mark "{bpf}" [tag {id}] pcap "{path}" + cmd = 'iol_bridge add_packet_filter {loc} {name} mark "{bpf}"'.format(loc=location, name=name, bpf=bpf) + if tag is not None: + cmd += f" tag {tag}" + if link_id: + cmd += f" link {link_id}" + if direction is not None: + cmd += f" dir {direction}" + linktype = self._marker_linktype(data_link_type) + if linktype is not None: + cmd += f" linktype {linktype}" + cmd += ' pcap "{path}"'.format(path=pcap_path) + await self._ubridge_send(cmd) - # 1. Remove installed markers that are no longer desired. - # Scope to THIS port's IOL location — the map is node-wide and also - # holds markers on this IOU's other ports, which must not be deleted - # when reconciling a single NIO (see base_node for the same guard). - for key in list(self._marker_filter_bridges): - if self._marker_filter_bridges[key] != location: - continue - if key not in desired: - mname, link_id = key - installed_location = self._marker_filter_bridges.pop(key) - self._marker_specs.pop(key, None) - await self._ubridge_delete_marker_filter(installed_location, mname) - try: - os.remove(os.path.join(markers_dir, f"{self._id}_{link_id}_{mname}.pcap")) - except FileNotFoundError: - pass - except OSError as e: - log.warning("Could not remove marker pcap for '%s' on link %s: %s", mname, link_id, e) - manager.unregister(self._id, mname) + async def _ubridge_enable_marker_filter(self, location, name, state): + """ + IOU's on/off toggle: ``iol_bridge enable_packet_filter`` on the + relay datapath (the port's IOL location), the mixin's + ``marker enable_kernel`` on the kernel datapath's anchor. + """ - # 2. Add / reconcile desired markers. - rebuild_fields = ("bpf", "tag", "direction", "data_link_type") - for (name, link_id), spec in desired.items(): - bpf = spec.get("bpf", "") - tag = spec.get("tag") - enabled = spec.get("enabled", True) - if (name, link_id) in self._marker_filter_bridges: - installed_spec = self._marker_specs.get((name, link_id)) - if installed_spec is None: - continue # installed (legacy, no spec) — skip to avoid dup - if any(installed_spec.get(f) != spec.get(f) for f in rebuild_fields): - installed_location = self._marker_filter_bridges.get((name, link_id)) - await self._ubridge_delete_marker_filter(installed_location, name) - elif installed_spec.get("enabled", True) != enabled: - await self._ubridge_set_marker_filter_state(name, enabled) - self._marker_specs[(name, link_id)] = spec - continue - else: - continue - pcap_path = os.path.join(markers_dir, f"{self._id}_{link_id}_{name}.pcap") - # iol_bridge add_packet_filter {br} {bay} {unit} {name} mark "{bpf}" [tag {id}] pcap "{path}" - cmd = f'iol_bridge add_packet_filter {location} {name} mark "{bpf}"' - if tag is not None: - cmd += f" tag {tag}" - if link_id: - cmd += f" link {link_id}" - direction = spec.get("direction") - if direction is not None: - cmd += f" dir {direction}" - linktype = self._marker_linktype(spec.get("data_link_type")) - if linktype is not None: - cmd += f" linktype {linktype}" - cmd += f' pcap "{pcap_path}"' - try: - await self._ubridge_send(cmd) - except UbridgeError as e: - if "syntax error" in str(e).lower() or "compile filter" in str(e).lower(): - message = f"Warning: ignoring marker '{name}' due to BPF syntax error: {e}" - log.warning(message) - self.project.emit("log.warning", {"message": message}) - continue - raise - if not enabled: - try: - await self._ubridge_send(f"iol_bridge enable_packet_filter {location} {name} off") - except UbridgeError as e: - log.warning(f"Could not turn marker '{name}' off on {location}: {e}") - manager.register(str(self.project.id), self._id, name, link_id, tag) - self._marker_filter_bridges[name, link_id] = location - self._marker_specs[name, link_id] = spec - - async def _ubridge_set_marker_filter_state(self, name, enabled): - """IOU override: toggle every (name, link_id) entry via ``iol_bridge``.""" - - state = "on" if enabled else "off" - for (n, lid), location in list(self._marker_filter_bridges.items()): - if n == name: - await self._ubridge_send(f"iol_bridge enable_packet_filter {location} {name} {state}") + if self._kernel_marker_anchor(location): + await super()._ubridge_enable_marker_filter(location, name, state) + return + await self._ubridge_send(f"iol_bridge enable_packet_filter {location} {name} {state}") async def _ubridge_delete_marker_filter(self, location, name): - """IOU override: remove a single marker filter via ``iol_bridge`` - (location = ``{bridge} {bay} {unit}``), not a bridge-wide reset.""" + """ + IOU override: remove a single marker filter via ``iol_bridge`` on + the relay datapath (location = ``{bridge} {bay} {unit}``), via the + mixin's ``marker delete_kernel`` on a kernel anchor — both + fine-grained, not a bridge-wide reset. + """ + if self._kernel_marker_anchor(location): + await super()._ubridge_delete_marker_filter(location, name) + return if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): return try: @@ -1353,8 +1499,24 @@ class IOUVM(BaseNode): log.debug(f'IOU "{self._name}" [{self._id}]: {nio} removed from {adapter_number}/{port_number}') if self.ubridge: - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" - await self._ubridge_send(f"iol_bridge delete_nio_udp {bridge_name} {adapter_number} {port_number}") + if isinstance(nio, NIOBridge): + await self._remove_kernel_nio(nio, adapter_number, port_number) + # Release the anchor's fd while the control channel lives: + # uBridge holds it while the port is bound (add_nio_tap), + # and a held TAP answers tap delete with EBADFD. The + # persistent device itself survives for the next link. + with contextlib.suppress(UbridgeError): + await self._ubridge_send( + "iol_bridge delete_nio_tap {name} {bay} {unit}".format( + name=self._iol_bridge_name(), bay=adapter_number, unit=port_number + ) + ) + else: + await self._ubridge_send( + "iol_bridge delete_nio_udp {name} {bay} {unit}".format( + name=self._iol_bridge_name(), bay=adapter_number, unit=port_number + ) + ) return nio @@ -1674,16 +1836,26 @@ class IOUVM(BaseNode): ) if self.ubridge: - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" - await self._ubridge_send( - 'iol_bridge start_capture {name} {bay} {unit} "{output_file}" {data_link_type}'.format( - name=bridge_name, - bay=adapter_number, - unit=port_number, - output_file=output_file, - data_link_type=re.sub(r"^DLT_", "", data_link_type), + if isinstance(nio, NIOBridge): + # Kernel link: capture on the port's TAP anchor (AF_PACKET). + anchor = self._kernel_host_ifc(adapter_number, port_number) + if anchor is None: + raise self._kernel_error( + "Bay {bay}/{unit} of IOU '{name}' has no TAP anchor to capture on".format( + bay=adapter_number, unit=port_number, name=self._name + ) + ) + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + else: + await self._ubridge_send( + 'iol_bridge start_capture {name} {bay} {unit} "{output_file}" {data_link_type}'.format( + name=self._iol_bridge_name(), + bay=adapter_number, + unit=port_number, + output_file=output_file, + data_link_type=re.sub(r"^DLT_", "", data_link_type), + ) ) - ) async def stop_capture(self, adapter_number, port_number): """ @@ -1699,5 +1871,11 @@ class IOUVM(BaseNode): nio.stop_packet_capture() log.debug(f'IOU "{self._name}" [{self._id}]: stopping packet capture on {adapter_number}/{port_number}') if self.ubridge: - bridge_name = f"IOL-BRIDGE-{self.application_id + 512}" - await self._ubridge_send(f"iol_bridge stop_capture {bridge_name} {adapter_number} {port_number}") + if isinstance(nio, NIOBridge): + await self._ubridge_send("capture stop_kernel") + else: + await self._ubridge_send( + "iol_bridge stop_capture {name} {bay} {unit}".format( + name=self._iol_bridge_name(), bay=adapter_number, unit=port_number + ) + ) diff --git a/tests/compute/iou/test_iou_kernel_datapath.py b/tests/compute/iou/test_iou_kernel_datapath.py new file mode 100644 index 000000000..9a49853a7 --- /dev/null +++ b/tests/compute/iou/test_iou_kernel_datapath.py @@ -0,0 +1,413 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation as either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Kernel-datapath wiring for IOU: the persistent TAP every Ethernet bay/unit +owns, the IOL-fabric binding (iol_bridge add_nio_tap) a kernel link adds to +it, kernel-link enslavement, the relay fallback for a uBridge without the +command, and the reverse-of-QEMU stop order (uBridge holds the anchor fds). +""" + +import pytest +import pytest_asyncio + +from unittest.mock import MagicMock, patch + +from tests.utils import AsyncioMagicMock + +from gns3server.compute.iou import IOU +from gns3server.compute.iou.iou_vm import IOUVM +from gns3server.compute.iou.iou_error import IOUError +from gns3server.compute.nios.nio_bridge import NIOBridge + + +NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" +BRIDGE = "gns3a1b2c3d4e5f" +TAP00 = "gi00010203e0p0" # the vm fixture's node id, bay 0 unit 0 +IOL_BRIDGE = "IOL-BRIDGE-513" # application_id 1 + 512 + + +@pytest_asyncio.fixture +async def manager(port_manager): + + m = IOU.instance() + m.port_manager = port_manager + return m + + +@pytest_asyncio.fixture(scope="function") +async def vm(compute_project, manager): + + vm = IOUVM("test", NODE_ID, compute_project, manager, application_id=1) + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_hypervisor.is_running.return_value = True + vm._ubridge_send = AsyncioMagicMock() + return vm + + +def _capable(vm, monkeypatch): + """Make the iol-tap capability probe answer True.""" + + async def probe(timeout=15.0): + return True + + monkeypatch.setattr("gns3server.compute.iou.iou_vm.probe_iol_tap_support", probe) + return vm + + +# --------------------------------------------------------------------------- +# Anchor lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_creates_one_tap_per_ethernet_unit(vm, monkeypatch): + """ + On a capable uBridge every Ethernet bay/unit gets its persistent TAP — + four units per bay, the IOU adapter model — swept of leftovers first and + born down. Serial bays never anchor. + """ + + _capable(vm, monkeypatch) + await vm._prepare_tap_datapath() + + assert vm._tap_datapath is True + assert set(vm._kernel_taps) == {(bay, unit) for bay in range(2) for unit in range(4)} + assert vm._kernel_taps[(0, 0)] == TAP00 + vm._ubridge_send.assert_any_call(f'tap delete "{TAP00}"') + vm._ubridge_send.assert_any_call(f'tap create "{TAP00}"') + vm._ubridge_send.assert_any_call(f'link set "{TAP00}" down') + assert not any("set_owner" in str(c) for c in vm._ubridge_send.call_args_list) + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_falls_back_without_the_command(vm, monkeypatch): + """ + A uBridge without iol_bridge add_nio_tap keeps the node on the relay + datapath — it still starts, it just cannot carry kernel links. + """ + + async def probe(timeout=15.0): + return None + + monkeypatch.setattr("gns3server.compute.iou.iou_vm.probe_iol_tap_support", probe) + with patch.object(vm.project, "emit") as emit: + await vm._prepare_tap_datapath() + + assert vm._tap_datapath is False + assert vm._kernel_taps == {} + assert emit.called + + +def test_tap_name_fits_ifnamsiz(vm): + """ + The anchor name has to fit IFNAMSIZ (15) even for a late bay/unit. + """ + + assert len(vm._tap_name(15, 3)) <= 15 + + +def test_kernel_host_ifc_is_none_on_serial_bays(vm): + """ + Serial bays (past the Ethernet adapters) never anchor, so a kernel link + there fails with the serial-port message rather than wiring nonsense. + """ + + assert vm._kernel_host_ifc(2, 0) is None + + +# --------------------------------------------------------------------------- +# Link attach / detach +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_attach_kernel_nio_binds_the_anchor_and_enslaves_it(vm): + """ + A kernel link binds the port's anchor TAP to the IOL bridge (uBridge + holds the fd and relays the fabric onto it), then enslaves the TAP into + the per-link kernel bridge. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + await vm._attach_kernel_nio(0, 0, nio) + + vm._ubridge_send.assert_any_call(f'iol_bridge add_nio_tap {IOL_BRIDGE} 1 0 0 "{TAP00}"') + vm._ubridge_send.assert_any_call(f'brctl create "{BRIDGE}"') + vm._ubridge_send.assert_any_call(f'link set "{BRIDGE}" up') + vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') + vm._ubridge_send.assert_any_call(f'link set "{TAP00}" up') + + +@pytest.mark.asyncio +async def test_attach_kernel_nio_without_anchor_raises(vm): + """ + A kernel link on a serial bay (or a node that never anchored) is an + error naming the relay datapath, not a silent fallback. + """ + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + with pytest.raises(IOUError, match="no TAP anchor"): + await vm._attach_kernel_nio(2, 0, nio) + + +@pytest.mark.asyncio +async def test_add_nio_binding_relay_keeps_the_iol_udp_path(vm): + """ + A relay link is wired exactly as before: a UDP NIO on the IOL bridge + port, userspace filters at the port's IOL location. + """ + + vm._ubridge_hypervisor.is_running.return_value = True + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + nio.filters = {"delay": [10]} + + await vm.adapter_add_nio_binding(0, 0, nio) + + vm._ubridge_send.assert_any_call(f"iol_bridge add_nio_udp {IOL_BRIDGE} 1 0 0 20000 127.0.0.1 20001") + vm._ubridge_send.assert_any_call(f"iol_bridge reset_packet_filters {IOL_BRIDGE} 0 0") + vm._ubridge_send.assert_any_call(f"iol_bridge add_packet_filter {IOL_BRIDGE} 0 0 filter0 delay 10") + + +@pytest.mark.asyncio +async def test_update_nio_binding_kernel_reapplies_on_the_anchor(vm): + """ + A filter update on an attached kernel link reconciles on the anchor + (tc) without re-binding the IOL port or re-enslaving the TAP. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"delay": [10]} + + await vm.adapter_update_nio_binding(0, 0, nio) + + vm._ubridge_send.assert_any_call(f'tc reset "{TAP00}"') + vm._ubridge_send.assert_any_call(f'tc netem set "{TAP00}" delay 10') + assert not any("iol_bridge add_nio" in str(c) for c in vm._ubridge_send.call_args_list) + assert not any("brctl addif" in str(c) for c in vm._ubridge_send.call_args_list) + + +@pytest.mark.asyncio +async def test_update_nio_binding_kernel_suspend_flaps_the_anchor(vm): + """ + Suspending a kernel link admin-downs the anchor: the IOL relay's writes + fail EIO (dropped) and nothing comes back — a genuinely dead link. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.suspend = True + + await vm.adapter_update_nio_binding(0, 0, nio) + + vm._ubridge_send.assert_any_call(f'link set "{TAP00}" down') + + +@pytest.mark.asyncio +async def test_remove_nio_binding_kernel_releases_the_fd_and_detaches(vm): + """ + Removing a kernel link detaches the anchor from the per-link bridge, + resets its qdisc, and unbinds the port from the TAP — releasing uBridge's + fd while the control channel lives, so a later tap delete cannot hit + EBADFD. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.adapter_remove_nio_binding(0, 0) + + vm._ubridge_send.assert_any_call(f'tc reset "{TAP00}"') + vm._ubridge_send.assert_any_call(f'brctl delif "{BRIDGE}" "{TAP00}"') + vm._ubridge_send.assert_any_call(f'brctl delete "{BRIDGE}"') + vm._ubridge_send.assert_any_call(f"iol_bridge delete_nio_tap {IOL_BRIDGE} 0 0") + + +# --------------------------------------------------------------------------- +# Stop order (uBridge holds the anchor fds — the reverse of QEMU) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_stop_ubridge_releases_fds_before_deleting_taps(vm): + """ + The IOL bridge must be deleted (closing every port's TAP fd) before the + taps are unpersisted, and the per-link bridges still need the control + channel — all before the hypervisor itself stops. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + with patch("gns3server.compute.base_node.BaseNode._stop_ubridge", new=AsyncioMagicMock()) as stop: + await vm._stop_ubridge() + + commands = [str(c[0][0]) if c[0] else "" for c in vm._ubridge_send.call_args_list] + assert commands.index(f"iol_bridge delete {IOL_BRIDGE}") < commands.index(f'tap delete "{TAP00}"') + assert commands.index(f'tap delete "{TAP00}"') < commands.index(f'brctl delete "{BRIDGE}"') + assert commands.index(f'brctl delete "{BRIDGE}"') < len(commands) + assert stop.called + assert vm._kernel_taps == {} + assert vm._ubridge_tc_caps is None + + +# --------------------------------------------------------------------------- +# _networking (node start with links already attached) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_networking_wires_kernel_and_relay_ports(vm): + """ + On start, a kernel-linked port binds its anchor to the IOL bridge and + enslaves it; a relay-linked port keeps its UDP NIO. The IOL bridge is + recreated and started around both. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + kernel_nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, kernel_nio) + relay_nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + vm._ethernet_adapters[1].add_nio(1, relay_nio) + + await vm._networking() + + vm._ubridge_send.assert_any_call(f"iol_bridge delete {IOL_BRIDGE}") + vm._ubridge_send.assert_any_call(f"iol_bridge create {IOL_BRIDGE} 513") + vm._ubridge_send.assert_any_call(f'iol_bridge add_nio_tap {IOL_BRIDGE} 1 0 0 "{TAP00}"') + vm._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') + vm._ubridge_send.assert_any_call(f"iol_bridge add_nio_udp {IOL_BRIDGE} 1 1 1 20000 127.0.0.1 20001") + vm._ubridge_send.assert_any_call(f"iol_bridge start {IOL_BRIDGE}") + + +@pytest.mark.asyncio +async def test_networking_restores_relay_capture_with_port_coordinates(vm): + """ + A capture active across a restart is restored on the port's bay/unit — + the iol_bridge command takes them positionally (the old restore sent a + 3-arg command uBridge refused with 203). + """ + + relay_nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + relay_nio.start_packet_capture("/tmp/capture.pcap", "DLT_EN10MB") + vm._ethernet_adapters[0].add_nio(0, relay_nio) + + await vm._networking() + + vm._ubridge_send.assert_any_call(f'iol_bridge start_capture {IOL_BRIDGE} 0 0 "/tmp/capture.pcap" EN10MB') + + +# --------------------------------------------------------------------------- +# Capture +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_capture_on_a_kernel_link_uses_the_anchor(vm): + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.start_capture(0, 0, "/tmp/test.pcap") + vm._ubridge_send.assert_any_call(f'capture start_kernel {TAP00} "/tmp/test.pcap"') + + await vm.stop_capture(0, 0) + vm._ubridge_send.assert_any_call("capture stop_kernel") + + +@pytest.mark.asyncio +async def test_capture_on_a_relay_link_uses_the_iol_port(vm): + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.start_capture(0, 0, "/tmp/test.pcap") + vm._ubridge_send.assert_any_call(f'iol_bridge start_capture {IOL_BRIDGE} 0 0 "/tmp/test.pcap" EN10MB') + + await vm.stop_capture(0, 0) + vm._ubridge_send.assert_any_call(f"iol_bridge stop_capture {IOL_BRIDGE} 0 0") + + +# --------------------------------------------------------------------------- +# Markers (the polymorphic anchor: IOL location on relay, TAP on kernel) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_markers_on_a_kernel_link_attach_to_the_anchor(vm): + """ + The marker reconcile is datapath-agnostic: with the anchor as the + location, the attach/toggle/delete primitives translate to + marker *_kernel on the TAP. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + nio.markers = { + "m1": {"bpf": "vlan 100", "link_id": "lk1", "enabled": True, "data_link_type": "DLT_EN10MB"}, + } + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm._ubridge_apply_markers(TAP00, nio) + + vm._ubridge_send.assert_any_call( + f'marker add_kernel m1 {TAP00} "vlan 100" link lk1 pcap "{vm.project.markers_working_directory()}/' + f'{vm.id}_lk1_m1.pcap"' + ) + assert vm._marker_filter_bridges[("m1", "lk1")] == TAP00 + + await vm._ubridge_set_marker_filter_state("m1", False) + vm._ubridge_send.assert_any_call(f"marker enable_kernel {TAP00} m1 off") + + await vm.delete_marker_capture("m1", "lk1", nio) + vm._ubridge_send.assert_any_call(f"marker delete_kernel {TAP00} m1") + + +@pytest.mark.asyncio +async def test_markers_on_a_relay_link_keep_the_iol_shape(vm): + """ + On the relay datapath the same reconcile talks iol_bridge, with the + bay/unit positionals — unchanged behaviour. + """ + + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + nio.markers = { + "m1": {"bpf": "vlan 100", "link_id": "lk1", "enabled": True, "data_link_type": "DLT_EN10MB"}, + } + vm._ethernet_adapters[0].add_nio(0, nio) + location = f"{IOL_BRIDGE} 0 0" + + await vm._ubridge_apply_markers(location, nio) + + vm._ubridge_send.assert_any_call( + f'iol_bridge add_packet_filter {location} m1 mark "vlan 100" link lk1 pcap ' + f'"{vm.project.markers_working_directory()}/{vm.id}_lk1_m1.pcap"' + ) + assert vm._marker_filter_bridges[("m1", "lk1")] == location + + await vm._ubridge_set_marker_filter_state("m1", False) + vm._ubridge_send.assert_any_call(f"iol_bridge enable_packet_filter {location} m1 off") + + await vm.delete_marker_capture("m1", "lk1", nio) + vm._ubridge_send.assert_any_call(f"iol_bridge delete_packet_filter {location} m1") From 200c946eef39cfb248144830b2c3499f1558726d Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 00:59:41 +0800 Subject: [PATCH 028/101] feat(ubridge): probe iol_bridge's TAP-port support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whether an IOU port can terminate on a persistent TAP (iol_bridge add_nio_tap) is a capability of its own: the tap module and that command land independently, so a build may have one without the other. Probed on a scratch IOL bridge with a scratch TAP — one add/delete cycle, no iol_bridge start needed (attaching a NIO works on a stopped bridge) — cached by binary identity, with the scratch bridge id above the per-node id space the fabric locks (513..1024) so a probe cannot collide with a running node. Any failure, including an old build's "Unknown command", means unknown: IOU stays on the relay datapath. Reported through /capabilities as ubridge_iol_tap, next to ubridge_tap. --- gns3server/api/routes/compute/capabilities.py | 7 +- gns3server/compute/ubridge/tc_probe.py | 91 +++++++++++++++++++ gns3server/controller/compute.py | 1 + gns3server/schemas/controller/capabilities.py | 6 ++ tests/api/routes/compute/test_capabilities.py | 23 ++++- tests/compute/ubridge/test_tc_probe.py | 52 ++++++++++- tests/controller/test_compute.py | 2 +- 7 files changed, 178 insertions(+), 4 deletions(-) diff --git a/gns3server/api/routes/compute/capabilities.py b/gns3server/api/routes/compute/capabilities.py index f2a51d702..df5e9d0e6 100644 --- a/gns3server/api/routes/compute/capabilities.py +++ b/gns3server/api/routes/compute/capabilities.py @@ -25,7 +25,7 @@ from fastapi import APIRouter, Request from gns3server import schemas from gns3server.compute import MODULES -from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support +from gns3server.compute.ubridge.tc_probe import probe_iol_tap_support, probe_tc_capabilities, probe_tap_support from gns3server.utils.path import get_default_project_directory from gns3server.utils.tc_capabilities import usable_ebpf_modes from gns3server.version import __version__ @@ -56,6 +56,10 @@ async def get_capabilities(request: Request) -> dict: # datapath. None = unknown (probe failed or uBridge too old). ubridge_tap = await probe_tap_support() + # IOL-port TAP termination (iol_bridge add_nio_tap): the anchor IOU's + # Ethernet ports need for the kernel datapath. None = unknown. + ubridge_iol_tap = await probe_iol_tap_support() + # record the controller hostname or IP address if request.client: request.app.state.controller_host = request.client.host @@ -69,4 +73,5 @@ async def get_capabilities(request: Request) -> dict: "node_types": node_types, "ubridge_tc": ubridge_tc, "ubridge_tap": ubridge_tap, + "ubridge_iol_tap": ubridge_iol_tap, } diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py index f6ba7a3ec..23ed916fb 100644 --- a/gns3server/compute/ubridge/tc_probe.py +++ b/gns3server/compute/ubridge/tc_probe.py @@ -51,6 +51,8 @@ log = logging.getLogger(__name__) _cache = {} # binary identity -> whether the tap module works (None = unknown) _tap_cache = {} +# binary identity -> whether iol_bridge can bind a port to a TAP (None = unknown) +_iol_tap_cache = {} _lock = asyncio.Lock() @@ -183,3 +185,92 @@ async def _probe_tap(path, config, timeout): with contextlib.suppress(Exception): await hypervisor.stop() shutil.rmtree(working_dir, ignore_errors=True) + + +# Above the per-node IOL bridge ids: a real node's application id is 1..512, +# so its bridge locks /tmp/netio/513..1024. 1050 cannot collide with a +# node — only with another probe process, whose failure then answers "unknown" +# (the safe answer: IOU stays on the relay datapath). +_IOL_PROBE_BRIDGE_ID = 1050 + + +async def probe_iol_tap_support(timeout: float = 15.0): + """ + Whether this host's uBridge can terminate an IOL port on a persistent + TAP (``iol_bridge add_nio_tap``) — the anchor IOU's Ethernet ports need + for the kernel datapath. Probed on a scratch IOL bridge with a scratch + TAP (both cleaned up), cached by binary identity like the tap probe; + any failure (missing binary, old build answering "Unknown command", + lock contention with another server process) means unknown, reported as + None and cached as such. + """ + + config = Config.instance() + path = shutil.which(config.settings.Server.ubridge_path) + if not path: + return None + try: + stat = os.stat(path) + key = (path, stat.st_mtime_ns, stat.st_size) + except OSError: + return None + + async with _lock: + if key in _iol_tap_cache: + return _iol_tap_cache[key] + supported = await _probe_iol_tap(path, config, timeout) + _iol_tap_cache[key] = supported + return supported + + +async def _probe_iol_tap(path, config, timeout): + """ + Spawn a throwaway uBridge and run one add_nio_tap/delete_nio_tap cycle on + a scratch bridge and scratch TAP — attaching works on a stopped bridge, + so no iol_bridge start is needed. The scratch resources are cleaned up + best-effort even when the probe itself fails. + """ + + working_dir = tempfile.mkdtemp(prefix="gns3-iol-probe-") + hypervisor = Hypervisor( + None, + path, + working_dir, + config.settings.Server.ubridge_control_transport, + config.settings.Server.host, + str(uuid.uuid4()), + ) + bridge = f"gns3iolprobe{uuid.uuid4().hex[:4]}" + tap = f"gns3ita{uuid.uuid4().hex[:6]}" + try: + + async def ask(): + await hypervisor.start() + await hypervisor.connect() + await hypervisor.send(f"iol_bridge create {bridge} {_IOL_PROBE_BRIDGE_ID}") + try: + await hypervisor.send(f"tap create {tap}") + await hypervisor.send( + f"iol_bridge add_nio_tap {bridge} {_IOL_PROBE_BRIDGE_ID - 1} 0 0 {tap}" + ) + await hypervisor.send(f"iol_bridge delete_nio_tap {bridge} 0 0") + finally: + # iol_bridge delete releases every port NIO (the TAP fd), + # so the subsequent tap delete cannot hit EBADFD. + with contextlib.suppress(UbridgeError): + await hypervisor.send(f"iol_bridge delete {bridge}") + with contextlib.suppress(UbridgeError): + await hypervisor.send(f"tap delete {tap}") + return True + + try: + return await asyncio.wait_for(ask(), timeout=timeout) + except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e: + # Old build ("Unknown command"), missing module, or no + # CAP_NET_ADMIN: unknown. + log.debug("uBridge iol-tap probe failed: %s", e) + return None + finally: + with contextlib.suppress(Exception): + await hypervisor.stop() + shutil.rmtree(working_dir, ignore_errors=True) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 3c470b1bb..8162a5e9c 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -101,6 +101,7 @@ class Compute: "node_types": [], "ubridge_tc": None, "ubridge_tap": None, + "ubridge_iol_tap": None, } self.name = name # Cache of interfaces on remote host diff --git a/gns3server/schemas/controller/capabilities.py b/gns3server/schemas/controller/capabilities.py index 72786f755..106df4b0f 100644 --- a/gns3server/schemas/controller/capabilities.py +++ b/gns3server/schemas/controller/capabilities.py @@ -55,3 +55,9 @@ class Capabilities(BaseModel): description="Whether uBridge can create the persistent TAPs a QEMU adapter anchors on " "(needed to wire QEMU links on the kernel datapath); absent when the probe failed", ) + ubridge_iol_tap: Optional[bool] = Field( + None, + description="Whether uBridge can bind an IOL port to a persistent TAP (iol_bridge add_nio_tap, " + "the anchor IOU Ethernet ports need to wire IOU links on the kernel datapath); absent when the " + "probe failed", + ) diff --git a/tests/api/routes/compute/test_capabilities.py b/tests/api/routes/compute/test_capabilities.py index 4934ae6f2..b196de34c 100644 --- a/tests/api/routes/compute/test_capabilities.py +++ b/tests/api/routes/compute/test_capabilities.py @@ -29,7 +29,7 @@ pytestmark = pytest.mark.asyncio async def _no_tap_probe(): - """The tap probe spawns uBridge — stub it out for payload assertions.""" + """The probes spawn uBridge — stub them out for payload assertions.""" return None @@ -42,6 +42,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -67,6 +68,7 @@ class TestCapabilitiesRoutes: "disk_size": psutil.disk_usage(get_default_project_directory()).total, "ubridge_tc": None, "ubridge_tap": None, + "ubridge_iol_tap": None, } async def test_get_on_gns3vm(self, app: FastAPI, compute_client: AsyncClient, on_gns3vm, monkeypatch) -> None: @@ -76,6 +78,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -101,6 +104,7 @@ class TestCapabilitiesRoutes: "disk_size": psutil.disk_usage(get_default_project_directory()).total, "ubridge_tc": None, "ubridge_tap": None, + "ubridge_iol_tap": None, } async def test_get_reports_ubridge_tc(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: @@ -110,6 +114,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", fake_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_tc"] == { @@ -129,6 +134,22 @@ class TestCapabilitiesRoutes: return True monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", fake_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_tap"] is True + + async def test_get_reports_ubridge_iol_tap(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: + """ + The IOL-port TAP capability is what makes an IOU node kernel-eligible, + so it travels with the other compute capabilities. + """ + + async def fake_iol_tap_probe(): + return True + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", fake_iol_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) + response = await compute_client.get(app.url_path_for("compute:get_capabilities")) + assert response.status_code == status.HTTP_200_OK + assert response.json()["ubridge_iol_tap"] is True diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py index 3a7d2a640..96735ab8e 100644 --- a/tests/compute/ubridge/test_tc_probe.py +++ b/tests/compute/ubridge/test_tc_probe.py @@ -22,7 +22,7 @@ payload: spawn a throwaway uBridge, ask once, cache by binary identity. import pytest from gns3server.compute.ubridge import tc_probe -from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support +from gns3server.compute.ubridge.tc_probe import probe_iol_tap_support, probe_tc_capabilities, probe_tap_support from gns3server.compute.ubridge.ubridge_error import UbridgeError pytestmark = pytest.mark.asyncio @@ -35,6 +35,7 @@ class FakeHypervisor: reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] error = None + error_prefix = None # fail only commands starting with this (a build missing one command) spawned = 0 def __init__(self, project, path, working_dir, transport, host, node_id): @@ -54,6 +55,8 @@ class FakeHypervisor: self.commands.append(command) if self.error: raise UbridgeError(self.error) + if self.error_prefix and command.startswith(self.error_prefix): + raise UbridgeError(f"202-Unknown command") if command == "tc capabilities": return FakeHypervisor.reply return ["OK"] @@ -67,9 +70,11 @@ def probe_env(monkeypatch, tmp_path): FakeHypervisor.spawned = 0 FakeHypervisor.error = None + FakeHypervisor.error_prefix = None FakeHypervisor.reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] tc_probe._cache.clear() tc_probe._tap_cache.clear() + tc_probe._iol_tap_cache.clear() FakeHypervisor.last = None # a real file on disk: the cache is keyed on its (path, mtime, size) binary = tmp_path / "ubridge" @@ -143,3 +148,48 @@ async def test_probe_tap_support_missing_binary(monkeypatch): monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) assert await probe_tap_support() is None assert FakeHypervisor.spawned == 0 + + +async def test_probe_iol_tap_support_runs_one_cycle_on_a_scratch_bridge(): + """ + The probe walks the exact add_nio_tap/delete_nio_tap cycle a kernel link + uses, on a scratch bridge whose id sits above the per-node id space, and + cleans up both scratch resources even when the cycle itself fails. + """ + + assert await probe_iol_tap_support() is True + assert FakeHypervisor.spawned == 1 + commands = FakeHypervisor.last.commands + assert commands[0].startswith("iol_bridge create gns3iolprobe") + assert commands[0].endswith(" 1050") + assert commands[1].startswith("tap create gns3ita") + assert commands[2].startswith("iol_bridge add_nio_tap ") + assert commands[2].split()[-1].startswith("gns3ita") + assert commands[3].startswith("iol_bridge delete_nio_tap ") + # cleanup: bridge first (it releases the TAP fd), then the device + assert commands[4].startswith("iol_bridge delete ") + assert commands[5].startswith("tap delete ") + + +async def test_probe_iol_tap_support_unknown_on_an_old_build(): + """ + A uBridge without iol_bridge add_nio_tap answers "Unknown command": the + probe reports unknown (IOU stays on the relay datapath), cleans up its + scratch bridge and tap, and caches the failure. + """ + + FakeHypervisor.error_prefix = "iol_bridge add_nio_tap" + assert await probe_iol_tap_support() is None + commands = FakeHypervisor.last.commands + assert any(c.startswith("iol_bridge delete ") for c in commands) + assert any(c.startswith("tap delete ") for c in commands) + # cached like the other probes: no respawn on every /capabilities hit + assert await probe_iol_tap_support() is None + assert FakeHypervisor.spawned == 1 + + +async def test_probe_iol_tap_support_missing_binary(monkeypatch): + + monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) + assert await probe_iol_tap_support() is None + assert FakeHypervisor.spawned == 0 diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index 1fce70eb1..ee48a143e 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -381,7 +381,7 @@ async def test_json(compute): "disk_usage_percent": 0, "connected": True, "last_error": None, - "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None}, + "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None}, } assert compute.asdict(topology_dump=True) == { "compute_id": "my_compute_id", From 806afec0d81f59914db7545ee5d96551da32adf6 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 00:59:46 +0800 Subject: [PATCH 029/101] feat(controller): offer IOU the kernel datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eligibility gains an IOU branch gated on the compute's ubridge_iol_tap (strict True: an unreported capability keeps the link on the relay), and a port-level rule: a kernel link is an Ethernet segment, so any endpoint whose port is not Ethernet (IOU serial) disqualifies the link before it is dispatched. Without that, a serial link between two capable IOU nodes would be sent as nio_bridge and fail at the compute. The IOU NIO routes accept nio_bridge alongside the UDP/TAP/Ethernet NIOs — without the widening the controller's POST is 422'd before the node ever sees it, the same trap the QEMU routes hit. --- gns3server/api/routes/compute/iou_nodes.py | 15 ++-- gns3server/controller/udp_link.py | 23 +++++-- tests/controller/test_kernel_datapath_link.py | 69 ++++++++++++++++++- 3 files changed, 92 insertions(+), 15 deletions(-) diff --git a/gns3server/api/routes/compute/iou_nodes.py b/gns3server/api/routes/compute/iou_nodes.py index 50600a734..661b63bec 100644 --- a/gns3server/api/routes/compute/iou_nodes.py +++ b/gns3server/api/routes/compute/iou_nodes.py @@ -189,15 +189,15 @@ async def reload_iou_node(node: IOUVM = Depends(dep_node)) -> None: @router.post( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO], + response_model=Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def create_iou_node_nio( adapter_number: int, port_number: int, - nio_data: Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO], + nio_data: Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO, schemas.BridgeNIO], node: IOUVM = Depends(dep_node), -) -> Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO]: +) -> Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO, schemas.BridgeNIO]: """ Add a NIO (Network Input/Output) to the node. """ @@ -210,15 +210,15 @@ async def create_iou_node_nio( @router.put( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO], + response_model=Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def update_iou_node_nio( adapter_number: int, port_number: int, - nio_data: Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO], + nio_data: Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO, schemas.BridgeNIO], node: IOUVM = Depends(dep_node), -) -> Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO]: +) -> Union[schemas.EthernetNIO, schemas.TAPNIO, schemas.UDPNIO, schemas.BridgeNIO]: """ Update a NIO (Network Input/Output) on the node. """ @@ -227,7 +227,8 @@ async def update_iou_node_nio( nio.filters.clear() if isinstance(nio_data, schemas.UDPNIO) and nio_data.filters: nio.filters = nio_data.filters - # NIO type is a Union (Ethernet/TAP/UDP); only UDPNIO carries markers. + # NIO type is a Union (Ethernet/TAP/UDP/bridge); only UDPNIO and + # BridgeNIO carry markers. nio.markers = getattr(nio_data, "markers", None) or {} await node.adapter_update_nio_binding(adapter_number, port_number, nio) return nio.asdict() diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 9eca6924a..416bb8050 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -179,7 +179,12 @@ class UDPLink(Link): must be able to anchor: Docker adapters are born as veth pairs, QEMU adapters as persistent TAPs, which need uBridge's tap module — asked of that compute, since an old uBridge leaves QEMU on the relay - datapath and cannot carry a kernel link at all. + datapath and cannot carry a kernel link at all. IOU's Ethernet + bays anchor on persistent TAPs too, bound to its IOL fabric + (``iol_bridge add_nio_tap``) — a separate capability, since it can + be present or absent independently of the tap module. A kernel link + is an Ethernet segment, so non-Ethernet ports (IOU serial) stay on + the relay whatever the node's capabilities. """ if not Config.instance().settings.Server.enable_kernel_datapath: @@ -188,6 +193,10 @@ class UDPLink(Link): return False if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): return False + for side in self._nodes: + port = side.get("port") + if port is not None and port.link_type != "ethernet": + return False return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2) @staticmethod @@ -198,12 +207,14 @@ class UDPLink(Link): if node.node_type == "docker": return True - if node.node_type != "qemu": - return False capabilities = node.compute.capabilities or {} - # Strict True: an unreported capability (old uBridge, failed probe) - # keeps the link on the relay, where it always works. - return capabilities.get("ubridge_tap") is True + if node.node_type == "qemu": + # Strict True: an unreported capability (old uBridge, failed + # probe) keeps the link on the relay, where it always works. + return capabilities.get("ubridge_tap") is True + if node.node_type == "iou": + return capabilities.get("ubridge_iol_tap") is True + return False async def _prepare(self): """ diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index e714994dd..b61d5a644 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -107,8 +107,8 @@ async def _link(project, node1, node2): @pytest.mark.asyncio async def test_kernel_datapath_not_eligible_for_other_node_types(project): """ - A node type with no anchor of its own (IOU, VPCS, cloud, ...) keeps the - relay however capable the compute is — and a QEMU node on a compute whose + A node type with no anchor of its own (VPCS, cloud, ...) keeps the relay + however capable the compute is — and a QEMU node on a compute whose uBridge cannot create persistent TAPs does too, because it runs on the legacy relay datapath there. """ @@ -120,11 +120,76 @@ async def test_kernel_datapath_not_eligible_for_other_node_types(project): assert link._kernel_datapath_eligible(node1, node2) is False compute = _tap_capable_compute() + node1 = _node(project, compute, "vpcs1", node_type="vpcs") + node2 = _node(project, compute, "docker1") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_for_iou_with_iol_tap_support(project): + """ + IOU's Ethernet bays anchor on persistent TAPs bound to the IOL fabric + (iol_bridge add_nio_tap) — its own capability, asked per compute like + QEMU's, and mixable with docker/qemu endpoints on the same compute. + """ + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} + node1 = _node(project, compute, "iou1", node_type="iou") + node2 = _node(project, compute, "iou2", node_type="iou") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True + + for peer in (_node(project, compute, "docker1"), _node(project, compute, "qemu1", node_type="qemu")): + link = await _link(project, node1, peer) + assert link._kernel_datapath_eligible(node1, peer) is True + + +@pytest.mark.asyncio +async def test_iou_eligibility_is_its_own_capability(project): + """ + The tap module and the IOL-port TAP command land independently: a + compute reporting only one of them keeps IOU on the relay (an old + uBridge without add_nio_tap), and an unreported/failed probe does too. + """ + + compute = _tap_capable_compute() # ubridge_tap only, no ubridge_iol_tap node1 = _node(project, compute, "iou1", node_type="iou") node2 = _node(project, compute, "docker1") link = await _link(project, node1, node2) assert link._kernel_datapath_eligible(node1, node2) is False + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": None} + node1 = _node(project, compute, "iou1", node_type="iou") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_serial_ports(project): + """ + A kernel link is an Ethernet segment: an IOU serial port stays on the + relay whatever the compute's capabilities — dispatching it to the + kernel path would fail at the compute (no anchor on a serial bay). + """ + + from gns3server.controller.ports.serial_port import SerialPort + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} + + node1 = _node(project, compute, "iou1", node_type="iou") + node1._ports = [SerialPort("Serial0/0", 0, 0, 0)] + node2 = _node(project, compute, "iou2", node_type="iou") + node2._ports = [SerialPort("Serial0/0", 0, 0, 0)] + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + @pytest.mark.asyncio async def test_kernel_datapath_eligible_for_qemu_with_tap_support(project): From f23c3bb6e602cd526335ca1191619f809b6c8503 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 00:59:53 +0800 Subject: [PATCH 030/101] fix(iou): carry suspend from the NIO update body The IOU NIO update route never copied the body's suspend flag onto the NIO; the Docker and QEMU routes always did. Harmless on the relay, where suspend is emulated by the synthetic frequency_drop filter the controller injects, but fatal on a kernel link, whose suspend *is* the anchor's admin state: a suspended link kept forwarding traffic, and the resume looked like it worked because nothing had ever stopped. --- gns3server/api/routes/compute/iou_nodes.py | 5 +++- tests/api/routes/compute/test_iou_nodes.py | 34 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/gns3server/api/routes/compute/iou_nodes.py b/gns3server/api/routes/compute/iou_nodes.py index 661b63bec..4ae44a215 100644 --- a/gns3server/api/routes/compute/iou_nodes.py +++ b/gns3server/api/routes/compute/iou_nodes.py @@ -228,8 +228,11 @@ async def update_iou_node_nio( if isinstance(nio_data, schemas.UDPNIO) and nio_data.filters: nio.filters = nio_data.filters # NIO type is a Union (Ethernet/TAP/UDP/bridge); only UDPNIO and - # BridgeNIO carry markers. + # BridgeNIO carry markers and suspend. Suspend is what the compute turns + # into an admin-down anchor on a kernel link (native carrier), so it must + # reach the NIO like it does on the Docker/QEMU routes. nio.markers = getattr(nio_data, "markers", None) or {} + nio.suspend = getattr(nio_data, "suspend", None) or False await node.adapter_update_nio_binding(adapter_number, port_number, nio) return nio.asdict() diff --git a/tests/api/routes/compute/test_iou_nodes.py b/tests/api/routes/compute/test_iou_nodes.py index feebe2b87..1b815ded1 100644 --- a/tests/api/routes/compute/test_iou_nodes.py +++ b/tests/api/routes/compute/test_iou_nodes.py @@ -297,6 +297,40 @@ class TestIOUNodesRoutes: assert response.json()["type"] == "nio_udp" assert response.json()["filters"] == {} + async def test_iou_nio_update_carries_suspend_and_bridge( + self, app: FastAPI, compute_client: AsyncClient, vm: dict + ) -> None: + """ + A kernel-datapath NIO (nio_bridge) is accepted on the IOU routes, and + the update body's suspend flag reaches the NIO — the compute turns it + into an admin-down anchor, so a dropped suspend would leave a + "suspended" link forwarding traffic. + """ + + url = app.url_path_for( + "compute:create_iou_node_nio", + project_id=vm["project_id"], + node_id=vm["node_id"], + adapter_number="1", + port_number="0", + ) + params = {"type": "nio_bridge", "bridge": "gns3a1b2c3d4e5f"} + response = await compute_client.post(url, json=params) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["type"] == "nio_bridge" + + url = app.url_path_for( + "compute:update_iou_node_nio", + project_id=vm["project_id"], + node_id=vm["node_id"], + adapter_number="1", + port_number="0", + ) + params["suspend"] = True + response = await compute_client.put(url, json=params) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["suspend"] is True + async def test_iou_nio_create_ethernet( self, app: FastAPI, compute_client: AsyncClient, vm: dict, ethernet_device: str ) -> None: From 3b8172cbc83e9f3d29407b266cbebbc35cd29482 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 01:00:02 +0800 Subject: [PATCH 031/101] docs(iou): record the IOU kernel datapath and its spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs/features/iou-kernel-datapath.md: architecture (one irreducible userspace hop on the fabric leg, kernel link segment), datapath selection, the anchor lifecycle with the reverse-of-QEMU stop order, capture/markers/filters, relay fallback, and the verified numbers (an isolated server with two IOU nodes behind a fake ELF image, the test playing the IOL fabric: 64/64 kernel, 62/62 relay). * docs/design/ubridge-iol-tap-anchor-spec.md: delivered — implemented on the ubridge fork's feature/iol-tap-anchor branch (both commands, the listener hardening, plus releasing every port NIO on iol_bridge delete even when the bridge was not running), with the implementation notes that came back (208 vs the older 214 for a missing bridge; one perror line per frame dropped on a down anchor). * The Docker/QEMU feature docs point at it from their roadmaps. --- docs/design/ubridge-iol-tap-anchor-spec.md | 268 +++++++++++++++++++++ docs/features/docker-kernel-datapath.md | 8 +- docs/features/iou-kernel-datapath.md | 170 +++++++++++++ docs/features/qemu-kernel-datapath.md | 9 +- 4 files changed, 452 insertions(+), 3 deletions(-) create mode 100644 docs/design/ubridge-iol-tap-anchor-spec.md create mode 100644 docs/features/iou-kernel-datapath.md diff --git a/docs/design/ubridge-iol-tap-anchor-spec.md b/docs/design/ubridge-iol-tap-anchor-spec.md new file mode 100644 index 000000000..172a656b9 --- /dev/null +++ b/docs/design/ubridge-iol-tap-anchor-spec.md @@ -0,0 +1,268 @@ + + +> Frozen requirements spec for the **uBridge** project. Status: **delivered +> and verified from the server side** — implemented on the +> `feature/iol-tap-anchor` branch of the ubridge fork (both commands, the §B +> hardening, and one fix beyond the spec: `iol_bridge delete` now releases +> every port NIO even when the bridge was not running, so a stopped node's +> anchor fds never leak). The server side landed on +> `feat/iou-kernel-datapath` and passed a live e2e (64/64 kernel, 62/62 +> relay; see `docs/features/iou-kernel-datapath.md`). +> +> Notes from the implementation: the two new commands answer `208` for a +> missing bridge where the older `add_nio_udp` answered `214` (no client +> branches on the code), and a write to an admin-DOWN anchor logs one +> `perror` line per dropped frame. +> +> Scope guard: no wire format change, no change to existing commands or +> replies. This adds one NIO type to the existing `iol_bridge` module (two +> commands) plus one error-tolerance fix in the two listener threads. The +> `tap`, `brctl`, `link`, `tc`, `marker` and `capture` modules are reused +> as-is. + +# IOL bridge TAP anchor ports (`iol_bridge add_nio_tap`) + +## 0. Background & scope + +### What IOU's physical layer is + +An IOU instance has no host netdev. Its only wire is the Unix-datagram +fabric in `/tmp/netio/`: frames carry an 8-byte IOL header (dst id, +src id, dst port, src port, message type, channel), and the NETMAP file +maps every `bay/unit` of the instance to a second, fake instance — which +today is uBridge itself (`iol_bridge create `, +binding `/tmp/netio/`). Each port of that fake +instance terminates on a **destination NIO**, and the only destination +NIO type is UDP (`iol_bridge add_nio_udp`): one listener thread per port +relays fabric → NIO, and one bridge listener relays fabric-side reads → +NIO sends. + +### The problem + +With UDP as the only termination, every IOU link is a uBridge userspace +relay leg: + +- an IOU port cannot join a per-link kernel bridge, so IOU ↔ Docker/QEMU + traffic always crosses through a UDP relay even when both ends sit on + the same compute; +- kernel-class impairments (`tc netem`, the clsact eBPF classifiers) and + the `*_kernel` capture/marker commands cannot anchor on an IOU port — + only the userspace packet filters apply, whose delay implementation is + the known blocking-nanosleep defect that inflates RTT under load; +- suspend of a single link has no clean mapping. + +### What this change is — and what it cannot be + +Give the port a second destination NIO type: a **persistent TAP, +created by gns3-server with the existing `tap create` command and opened +(held) by uBridge**. uBridge's iol_bridge then plays, on that TAP, the +same role the QEMU process plays on its TAP: it is the device's +userspace endpoint. Everything the anchor model already delivers — +`brctl addif` enslavement, `tc` on the anchor, `capture start_kernel`, +`marker add_kernel`, `link set up/down` for carrier/suspend — is keyed +on the interface name and does not care who holds the fd. + +One userspace hop on the IOU leg is irreducible: the Unix fabric is +IOU's only physical layer, so frames always pass +fabric ↔ listener thread ↔ TAP fd. What kernelization buys is that the +**link segment itself** is the kernel: anchor → per-link bridge → peer +anchor, with kernel filters, markers and capture on the anchors, and +suspend mapped to anchor admin-down. That is the accepted design, not a +deficiency of this spec. + +### Goal + +`iol_bridge` ports can terminate on a pre-existing persistent TAP +anchor, with the same lifecycle semantics as `add_nio_udp` (attach +before or after `iol_bridge start`, replace a previous NIO, delete and +re-add), and the relay survives — without exiting — the anchor being +administratively DOWN, which is a steady state of this design (port +anchored but no link attached; link suspended). + +Explicitly **not** in scope: + +- creating or deleting the TAP device itself — the server owns the + anchor lifecycle through the existing `tap` module (`tap create`, + `tap delete`; the device is created hardened per the L2-only spec); +- serial bays — a TAP carries Ethernet frames; serial links stay on + `add_nio_udp`; +- an MTU command (`link set mtu `) — does not exist today and is + not required by default-MTU IOU traffic; noted as future work if + jumbo frames ever matter; +- any change to the NETMAP, the IOL header encoding, or the L1 + keepalive protocol. + +## A. New commands + +Belong to the existing `iol_bridge` module, next to their UDP +counterparts. `bay`/`unit` and the IOL header handling are identical to +`add_nio_udp`. + +### `iol_bridge add_nio_tap ` (6 args) + +| Arg | Description | +|-----|-------------| +| `` | An existing IOL bridge (`208` if not) | +| `` | The real instance's application id, encoded into the IOL header as with `add_nio_udp`; the existing "`iol_id` == bridge id" refusal in `create_iol_port_entry` applies unchanged | +| ``, `` | Port coordinates, same encoding and MAX_PORTS check as `add_nio_udp` | +| `` | A **pre-existing** network device (`208` if not, see below); ≤ IFNAMSIZ−1 chars (`204` otherwise) | + +Semantics: + +1. **Require an existing device.** Resolve `` with + `if_nametoindex` first and refuse with `208` when absent. This is the + same reasoning as the `tap` module's `tap_require_existing`: without + the check, `TUNSETIFF` silently *creates a transient* TAP that dies + with the fd — an anchor that vanishes on detach is worse than a + clear error. +2. Open the device with the existing `create_nio_tap()` (by-name + `TUNSETIFF`, `IFF_TAP | IFF_NO_PI`, default carrier). No owner or + persistence ioctl — the device is persistent and owned already. +3. Install it through the existing `create_iol_port_entry()`, which is + generic over `nio_t *`: it frees whatever NIO the port held before + (listener stop guarded on `tid`, delay lines torn down under + `iol_delay_lock`, capture/filters freed), pre-computes the IOL + header, and starts the port listener if the bridge is running. A + port may therefore swap `add_nio_udp` ↔ `add_nio_tap` repeatedly — + that is the datapath-switch path and must keep working. +4. If the fd cannot be opened (`create_nio_tap` fails), reply `206` and + free the nio as the UDP handler does. + +No `carrier` argument: administrative state is the server's lever, via +the existing `link set up|down`. + +### `iol_bridge delete_nio_tap ` (3 args) + +Same body as `cmd_delete_nio_udp`, including its thread-safety guards: +stop the port listener (cancel guarded on `tid != 0`, join), destroy +both delay lines under `iol_delay_lock`, free capture, filters and the +NIO (which closes the TAP fd). **The persistent device itself survives** +— the server deletes it with `tap delete` when the node stops, exactly +as it does for QEMU anchors. Deleting a port that holds no NIO is a +no-op `100` (mirrors `delete_nio_udp`). + +Replies: + +``` +iol_bridge add_nio_tap IOL-BRIDGE-513 1 0 0 gi0badfe9e0p0 +100-NIO TAP added to IOL bridge 'IOL-BRIDGE-513' +iol_bridge delete_nio_tap IOL-BRIDGE-513 0 0 +100-NIO TAP deleted from IOL bridge 'IOL-BRIDGE-513' +``` + +## B. Required hardening: the anchor can be administratively DOWN + +A TAP anchor is expected to sit admin-DOWN for long periods (port +anchored, no link attached yet; or the link suspended). The listeners +must treat that as normal: + +1. **`iol_bridge_listener`, NIO send path** (IOL → TAP, `nio->send` = a + `write` on the fd): a write to an admin-DOWN TAP fails with **EIO**. + The current tolerated set is `ECONNREFUSED | ENETDOWN | EINVAL`, + anything else is `exit(EXIT_FAILURE)` — which would take the whole + uBridge down the first time a suspended link carries traffic. Add + `EIO` to the tolerated set: drop the frame (optionally a debug log) + and continue. +2. **`iol_nio_listener`, NIO receive path** (TAP → IOL, `nio_recv` = a + `read` on the fd): while the device is DOWN the read blocks (no + carrier), which is fine — but a return of `0` must not be counted or + forwarded (it would fabricate header-only packets for the instance). + Treat `recv <= 0` as continue; treat `-1` with `EIO` as continue + alongside the existing `ECONNREFUSED | ENETDOWN`. + +No other listener behavior changes: the fabric-side `sendto` already +tolerates `ENOENT`/`ECONNREFUSED` for the window where the IOU process +has not created its endpoint yet, and that stays. + +## C. Implementation notes + +- The command handlers are thin: existence check + `create_nio_tap` + + `create_iol_port_entry` (add), and the `cmd_delete_nio_udp` body + (delete). The three copies of that teardown block in the module are a + pre-existing pattern — a shared helper is welcome but not required by + this spec. +- Do **not** route the TAP through `nio_tap_open`'s `/dev/net/tun` + path-name variant; the anchor is addressed by interface name. +- Statistics (`iol_bridge get_stats`) work unchanged: the counters hang + off the generic nio and both listeners already update them. +- Port filters, delay lines, capture (`iol_bridge add_packet_filter`, + `start_capture`, …) keep working on a TAP-terminated port; gns3-server + simply will not combine userspace filters with `tc` on the same port + (see the alignment section). + +## D. Reply contract & error codes + +| Code | Meaning | +|------|---------| +| `100` | OK — NIO added/deleted (delete of an empty port is also `100`) | +| `203` | Bad number of parameters | +| `204` | Invalid parameter (tap name longer than IFNAMSIZ−1) | +| `206` | Unable to create (bridge exists but the TAP fd cannot be opened) | +| `208` | Unknown object — bridge or **tap device** does not exist | + +Old builds answer `202-Unknown command` for both commands; gns3-server +uses that to fall back to the UDP datapath, mirroring the `tap`-probe +degradation. + +## E. Test requirements + +1. **Both-direction relay.** On a running IOL bridge with one + TAP-terminated port: a frame sent on the fabric for that bay/unit + appears on the TAP netdev (stripped of the IOL header); a frame + injected on the netdev reaches the fabric with the port's correct + 8-byte header. (A scratch Unix-datagram endpoint can play the + instance; no IOU binary is needed.) +2. **No transient device.** `add_nio_tap … nosuchtap0` → `208`, and no + interface is created (`ip -o link` unchanged). +3. **Unknowns.** Unknown bridge → `208`; name ≥ IFNAMSIZ → `204`; a + bridge-id == iol_id → the existing refusal. +4. **Delete keeps the device.** After `delete_nio_tap`, the port has no + NIO (stats/list reflect that), the TAP netdev still exists, and the + port accepts `add_nio_udp` (and re-`add_nio_tap`) afterwards. +5. **Swap semantics.** A port holding a UDP NIO that gets `add_nio_tap` + releases the UDP socket (old listener joined, port usable), and vice + versa — no fd or thread leak after several swaps. +6. **DOWN-anchor resilience.** With the TAP admin-DOWN, drive traffic + from the fabric side for ≥ 100 frames: uBridge stays alive, frames + are dropped, nothing is written to the netdev. After `link set up`, + traffic flows again. This is the regression test for §B.1. +7. **Idle DOWN port is quiet.** A DOWN anchored port produces no + header-only or zero-length fabric sends (§B.2). +8. **Kernel-bridge interop.** Enslave the anchor into a kernel bridge + (`brctl create` + `addif`) while the port relays: frames cross + bridge ↔ fabric through the port, and a peer port on the same bridge + receives them. This is the actual deployment shape. + +## gns3-server alignment (informational — not uBridge scope) + +- **Anchors**: one TAP per Ethernet bay/unit, `gi{node_id[:8]}e{bay}p{unit}`, + created persistent (hardened by `tap create` per the L2-only spec) at + node start, swept for stale leftovers first, `tap delete`d at node + stop — the exact lifecycle QEMU anchors use. All Ethernet ports are + anchored at start ("tap-everywhere"); serial bays never are. +- **Links**: kernel links enslave the anchor via the shared + `KernelDatapathMixin` (`brctl create`/`addif`, `tc`, markers, + `capture start_kernel`, suspend = anchor admin-down). Relay links keep + `add_nio_udp` and never touch the anchor. A port switching datapaths + is just an NIO swap (§A.3). +- **Filters are never double-applied**: on a kernel link only the anchor + `tc` path is used; the port's userspace filters carry relay links + only. +- **Capability gating**: the eligibility check needs to know the command + exists before creating a link. Probe by attempting the operation on a + scratch bridge + scratch TAP (create, `add_nio_tap`, delete, clean + up), cached by binary identity like `probe_tap_support`, reported + through `/capabilities`; an old build answering `202` keeps IOU on + the relay datapath. +- **L1 keepalives**: unchanged — an anchored-but-unconnected port holds + no NIO from the server's perspective, so the responder keeps ignoring + it and IOU sees the port down, which matches the anchor being DOWN. + +## Roadmap note + +`link set mtu ` does not exist today. Default-MTU IOU traffic +does not need it; if jumbo IOU frames ever matter it becomes a small, +separate requirement on the `link` module. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 0be8facef..42ebb763f 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -367,8 +367,12 @@ enable_kernel_datapath = True Intended to grow into the global datapath switch as IOU / Dynamips migrate to kernel bridges. **QEMU has landed** on the same pattern (persistent TAP anchor enslaved into the per-link bridge, zero relay) — see -`docs/features/qemu-kernel-datapath.md`; the shared half of this document's -compute machinery now lives in `gns3server/compute/kernel_datapath.py`. +`docs/features/qemu-kernel-datapath.md` — and **IOU after it** +(`docs/features/iou-kernel-datapath.md`; the fabric terminator in uBridge +gained a TAP-terminated port, `iol_bridge add_nio_tap`, specified in +`docs/design/ubridge-iol-tap-anchor-spec.md`). The shared half of this +document's compute machinery now lives in +`gns3server/compute/kernel_datapath.py`. ## Verification diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md new file mode 100644 index 000000000..b90efc673 --- /dev/null +++ b/docs/features/iou-kernel-datapath.md @@ -0,0 +1,170 @@ + + +# IOU kernel datapath + +IOU's only physical layer is the Unix-datagram IOL fabric (`/tmp/netio/`, +8-byte IOL headers, NETMAP), so it has no host netdev to enslave — a kernel +link instead gives each **Ethernet bay/unit** a persistent TAP anchor +(`gi{node_id[:8]}e{bay}p{unit}`), and uBridge's `iol_bridge` (the fake IOL +instance the NETMAP maps to) binds the port to that TAP with +`iol_bridge add_nio_tap`. Links then attach to the anchor exactly as they do +for Docker (veth) and QEMU (TAP): a per-link Linux bridge enslaves it, tc +impairments run on it, capture and markers anchor on it. The uBridge +capability is specified in `docs/design/ubridge-iol-tap-anchor-spec.md` and +delivered by the `feature/iol-tap-anchor` branch of the ubridge fork. + +Serial bays never anchor (a TAP carries Ethernet frames): serial links stay +on the relay datapath whatever the compute's capabilities. + +## Architecture + +``` + IOU process root namespace + ┌──────────┐ unix datagram ┌──────────────────────────────┐ + │ IOL │◄──── /tmp/netio ──►│ uBridge iol_bridge (listener)│ + │ fabric │ (8-byte hdr) │ │ holds the TAP fd │ + └──────────┘ │ ▼ │ + │ gi{id}e{bay}p{unit} ──────┼──► gns3{link_id} + │ (persistent TAP anchor) │ (per-link kernel + └──────────────────────────────┘ bridge port) +``` + +One userspace hop on the IOU leg is **irreducible** — the fabric is Unix +sockets, so frames always cross fabric → listener thread → TAP fd. What the +kernel datapath buys is the **link segment**: anchor → per-link bridge → +peer anchor crosses entirely in the kernel (including IOU ↔ Docker/QEMU +links), impairments run as kernel tc instead of the userspace delay line +(the known blocking-nanosleep RTT defect), and suspend maps to anchor +admin-down (writes to a down TAP fail EIO and are dropped by the delivered +§B hardening). Direction semantics on the anchor match QEMU's exactly: +uBridge holds the fd like the QEMU process does. + +## Datapath selection + +`UDPLink._kernel_datapath_eligible` extends the Docker/QEMU rules with: + +* the compute must report **`ubridge_iol_tap`** (its own capability: the tap + module and `iol_bridge add_nio_tap` land independently, and either can be + missing on an old build). Probed by running one add/delete cycle on a + scratch bridge + scratch TAP (`probe_iol_tap_support`, cached by binary + identity, scratch bridge id 1050 — above the per-node id space 513..1024 + the fabric locks); any failure means unknown → relay; +* **both linked ports must be Ethernet** — an IOU serial port disqualifies + the kernel path per-port, not per-node, so serial links between capable + IOU nodes silently stay on the relay instead of failing at the compute. + +The NIO schema on the compute NIO routes accepts `nio_bridge` alongside the +UDP/TAP/Ethernet NIOs (the same widening QEMU needed — without it the +controller's NIO POST is 422'd before the node ever sees it). + +## Anchor lifecycle — and why the stop order is QEMU's reverse + +* **Birth (node start)** — `_prepare_tap_datapath` probes the capability and + creates one persistent TAP per Ethernet bay/unit (4 units per bay, swept + for stale leftovers first, born DOWN). uBridge holds a device's fd only + while a kernel link binds the port; until then the TAP has no holder. +* **Life** — the TAP is never recreated. A kernel link adds + `iol_bridge add_nio_tap ` then + enslaves the anchor (`_kernel_attach`); a relay link keeps + `iol_bridge add_nio_udp` and never touches the anchor. Switching a port + between datapaths is just an NIO swap — the IOL port frees its previous + NIO (fd or socket) on attach. +* **Death (node stop)** — `_stop_ubridge` runs the cleanup **in the reverse + order of QEMU's**, because here uBridge itself holds the anchor fds: + `iol_bridge delete` (releases every port's TAP fd) → `tap delete` per + anchor (a held TAP answers EBADFD/207) → `brctl delete` for per-link + bridges still held → stop the hypervisor. Removing a single kernel link + does the same in miniature: `_remove_kernel_nio` (markers, tc reset, + delif, brctl delete) then `iol_bridge delete_nio_tap` — releasing the fd + while the control channel lives. + +## Link operations + +| Operation | Kernel-datapath action | +|---|---| +| Create / attach | `iol_bridge add_nio_tap` (bind port ↔ anchor, uBridge holds the fd) + `brctl create` (EEXIST tolerated) + `link set up` + `brctl addif` + carrier up + markers + `tc netem set` | +| Delete / detach | marker teardown → `tc reset ` → carrier off → `brctl delif`/`brctl delete` (last endpoint wins) → `iol_bridge delete_nio_tap` (release the fd; the device survives) | +| Update (filters/markers changed) | reconcile on the anchor only — no re-binding, no re-enslaving | +| Suspend | anchor admin-down (IOL relay writes fail EIO → dropped; reads block) + resume restores | + +## Capture, markers, filters + +Capture on a kernel link is `capture start_kernel ` (driven by the NIO +type); markers ride the base reconcile with the anchor as the location — +IOU's marker primitives are polymorphic (`marker add_kernel/enable_kernel/ +delete_kernel` on an anchor, `iol_bridge … mark` on an IOL location), which +let the old IOU-specific reconcile override be deleted outright. Filters are +one tc netem qdisc per anchor plus cls_bpf/eBPF drops — the port's +userspace `iol_bridge add_packet_filter` filters carry relay links only and +are never combined with tc on the same port. + +## Relay fallback + +A uBridge without `iol_bridge add_nio_tap` (old build) keeps the node on the +relay datapath unchanged: every link rides `iol_bridge add_nio_udp`, the +node starts fine, and the warning says it cannot carry kernel links. The +controller never offers such a node a kernel link (capability unreported). + +## Verified + +Unit level: anchor lifecycle and naming, kernel/relay binding on add, +update and remove, the stop order (iol_bridge delete before tap delete +before brctl delete before hypervisor stop), `_networking` on restart with +mixed kernel/relay ports, capture and marker command shapes on both +datapaths, the capability probe (cycle, old-build fallback, cleanup, +caching), capability plumbing, and controller eligibility (per-compute +capability, its independence from `ubridge_tap`, serial-port exclusion). + +End-to-end on a live server (isolated instance, two IOU nodes behind a fake +ELF image, the test playing the IOL fabric on `/tmp/netio/` and pushing +frames through it): **64/64 checks** on the kernel datapath and **62/62** on +the relay (`enable_kernel_datapath = false`). Kernel highlights: + +* 8 anchors per node (2 bays × 4 units), persistent, DOWN, address-free; + `link_list` reports `kernel_datapath: true` and one per-link bridge holds + both anchors with `brport/state = 3`; +* fabric → anchor → bridge → anchor → fabric in both directions at ~0.2 ms; + `delay 100` ⇒ netem on both anchors and a measured **100.5 ms** one-way; +* suspend ⇒ anchor DOWN, no traffic, resume ⇒ 100.4 ms with the qdisc intact; +* capture ⇒ a 328-byte pcap written by `capture start_kernel`; +* link delete ⇒ bridge gone, qdisc clean, anchor alive, re-create at 0.6 ms; +* a **serial** link between the same two capable nodes stays on the relay + (`kernel_datapath: false`) — per-port exclusion, not per-node; +* single-sided restart ⇒ stopped node's anchors removed, peer's kept, link + re-attached from the NIO (0.3 ms); stop ⇒ no stray anchors, bridges or + fabric sockets. + +The relay run exercises the same lifecycle with the IOL UDP NIOs (no tc +qdiscs, userspace `delay` at 100.7 ms, suspend via the synthetic +frequency_drop, `iol_bridge start_capture`), which is also the regression +net for the port-coordinate capture restore this work touched. + +The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the +fork), including the DOWN-anchor resilience and kernel-bridge interop this +design depends on. + +One pre-existing bug surfaced here and is fixed: the IOU NIO **update** route +never copied the body's `suspend` onto the NIO (Docker and QEMU routes do). +Harmless on the relay, where suspend is emulated by the synthetic +frequency_drop filter, but fatal for a kernel link, whose suspend *is* the +anchor's admin state — a "suspended" link kept forwarding. + +## Configuration + +```ini +[Server] +# Wire eligible links (same compute, both endpoints Ethernet and able to +# anchor) through kernel interfaces instead of the uBridge UDP relay. +enable_kernel_datapath = True +``` + +## Roadmap + +Dynamips next (hypervisor-created taps, enslavable as they are), then the +Ethernet switch (its anchors are ubridge-owned by design) and the IOL Docker +node (same `bridge add_nio_tap` shape as this one). Cross-compute kernel +links need VXLAN/GENEVE encapsulation — deferred until every node type is +kernelized. diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 1797ca4c6..90e336665 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -161,8 +161,15 @@ addresses involved, IPv4 has no per-device IGMP switch). ## Roadmap -IOU (its fabric would need a kernel-side port in `iol_bridge`), Dynamips +IOU (its fabric terminator in uBridge needs a TAP-terminated port — +`iol_bridge add_nio_tap`, frozen in +`docs/design/ubridge-iol-tap-anchor-spec.md`; unlike QEMU, one userspace hop +on the IOU leg is irreducible, the fabric is Unix sockets), Dynamips (hypervisor-created taps, enslavable as they are), and the Ethernet switch / cloud paths (their anchors are ubridge-owned by design) are still on the relay; cross-compute kernel links need VXLAN/GENEVE encapsulation, which is a separate project (it would serve Docker links the same way). + +Update: **IOU has landed** on the same mixin — see +`docs/features/iou-kernel-datapath.md` (Ethernet bays anchor on persistent +TAPs bound to the IOL fabric; serial links stay relay). From 4fa7f158fd569b5c869fc6ea1f7faf0478007b70 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 01:36:52 +0800 Subject: [PATCH 032/101] fix(schemas): one definition of the capabilities payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /computes/{id} answered with only part of the capabilities the compute reports: the response schema kept a second, private Capabilities class of its own (it knew ubridge_tc and nothing else), and FastAPI's response model silently drops fields it does not define — so ubridge_tap and ubridge_iol_tap never reached clients even though the compute had sent them and links were already being built on those datapaths. Import the one definition instead of duplicating it, so the next capability cannot drift the same way. --- gns3server/schemas/controller/computes.py | 23 +++---------- tests/api/routes/controller/test_computes.py | 34 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/gns3server/schemas/controller/computes.py b/gns3server/schemas/controller/computes.py index 836ac9607..4924827e5 100644 --- a/gns3server/schemas/controller/computes.py +++ b/gns3server/schemas/controller/computes.py @@ -21,8 +21,11 @@ from typing import Any, List, Optional, Union from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator from .base import DateTimeModelMixin -from .capabilities import UbridgeTcCapabilities -from .nodes import NodeType +# The capabilities payload has one definition (controller.capabilities): a +# second, private copy here silently stripped every field it did not know +# about from GET /computes/{id} — which is how ubridge_tap and +# ubridge_iol_tap went missing while the compute returned them. +from .capabilities import Capabilities class Protocol(str, Enum): @@ -93,22 +96,6 @@ class ComputeUpdate(ComputeBase): ) -class Capabilities(BaseModel): - """ - Capabilities supported by a compute. - """ - - version: str = Field(..., description="Compute version number") - node_types: List[NodeType] = Field(..., description="Node types supported by the compute") - platform: str = Field(..., description="Platform where the compute is running (Linux, Windows or macOS)") - cpus: int = Field(..., description="Number of CPUs on this compute") - memory: int = Field(..., description="Amount of memory on this compute") - disk_size: int = Field(..., description="Disk size on this compute") - ubridge_tc: Optional[UbridgeTcCapabilities] = Field( - None, description="uBridge tc-module capabilities; absent when the probe failed or uBridge has no tc module" - ) - - class Compute(DateTimeModelMixin, ComputeBase): """ Data returned for a compute. diff --git a/tests/api/routes/controller/test_computes.py b/tests/api/routes/controller/test_computes.py index 3c213c07c..83ced12b7 100644 --- a/tests/api/routes/controller/test_computes.py +++ b/tests/api/routes/controller/test_computes.py @@ -74,6 +74,40 @@ class TestComputeRoutes: assert response.status_code == status.HTTP_200_OK assert response.json()["compute_id"] == str(test_compute.compute_id) + async def test_compute_get_reports_every_capability(self, app: FastAPI, client: AsyncClient, controller) -> None: + """ + The capabilities payload the compute reports must survive the + response model intact. The local compute is the interesting one: it + is the controller object serialized straight through, and a second, + private Capabilities definition in the response schema silently + stripped the kernel-datapath fields (ubridge_tap, ubridge_iol_tap) + there — links were then built on a datapath clients could not see. + """ + + await controller.add_compute( + compute_id="local", name="local", host="127.0.0.1", port=3080, force=True, connect=False + ) + controller.get_compute("local")._capabilities.update( + { + "version": "3.1.0", + "node_types": ["docker"], + "platform": "linux", + "cpus": 4, + "memory": 1024, + "disk_size": 2048, + "ubridge_tc": {"netem": ["delay"], "ebpf": True, "ebpf_modes": ["nth"], "cbpf": True}, + "ubridge_tap": True, + "ubridge_iol_tap": False, + } + ) + + response = await client.get(app.url_path_for("get_compute", compute_id="local")) + assert response.status_code == status.HTTP_200_OK + capabilities = response.json()["capabilities"] + assert capabilities["ubridge_tap"] is True + assert capabilities["ubridge_iol_tap"] is False + assert capabilities["ubridge_tc"]["netem"] == ["delay"] + async def test_compute_get_local(self, app: FastAPI, client: AsyncClient, controller) -> None: await controller.add_compute( From 6633304e972af7356ae17ab53c406a16eb74aa23 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 01:36:57 +0800 Subject: [PATCH 033/101] fix(mcp): let link_list return kernel_datapath The tool descriptions have always advertised kernel_datapath as a requestable field, but the handler's whitelist did not contain it: asking for it was rejected as an invalid field, so the MCP clients could not see which links run on kernel interfaces at all. Also record what seeding a netem experiment on IOU costs: IOL emits its own CDP/keepalive frames and they draw from the netem RNG stream, so a seeded run only reproduces after `no cdp run` (verified bit-identical pcaps). --- docs/features/iou-kernel-datapath.md | 6 ++++++ .../gns3_copilot/gns3_client/api_handlers.py | 4 ++++ tests/agent/mcp/test_handlers.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index b90efc673..8396f3998 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -152,6 +152,12 @@ Harmless on the relay, where suspend is emulated by the synthetic frequency_drop filter, but fatal for a kernel link, whose suspend *is* the anchor's admin state — a "suspended" link kept forwarding. +**Measuring seeded impairments on IOU**: IOL emits CDP/keepalive frames of +its own, and they draw from the netem RNG stream — a `seed`-reproducible run +never matches while they are on (measured: three rounds of mismatches, then +bit-identical pcaps after `no cdp run`). Turn the platform chatter off before +comparing seeded runs; the extra frames are visible in the capture. + ## Configuration ```ini diff --git a/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py b/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py index a7eb0e04d..ba3fca6c7 100644 --- a/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py +++ b/gns3server/agent/gns3_copilot/gns3_client/api_handlers.py @@ -105,6 +105,10 @@ VALID_LINK_FIELDS = { "link_id", "project_id", "link_type", + # Runtime datapath fact (kernel bridge vs uBridge relay): the MCP tool + # descriptions have always advertised it, but it was missing here, so + # link_list could not return it at all. + "kernel_datapath", "nodes", "suspend", "link_style", diff --git a/tests/agent/mcp/test_handlers.py b/tests/agent/mcp/test_handlers.py index 9752c4422..cad7b89c0 100644 --- a/tests/agent/mcp/test_handlers.py +++ b/tests/agent/mcp/test_handlers.py @@ -425,6 +425,21 @@ class TestLink: result = get_links_handler({"project_id": "p1", "fields": ["link_id"]}, ctx) assert result["links"] == [{"link_id": "l1"}] + def test_list_can_ask_for_kernel_datapath(self, ctx): + """ + The datapath is a field the tool descriptions always advertised, so + it has to be requestable: a whitelist missing it rejected + fields=["kernel_datapath"] and made link_list unable to report which + links run on kernel interfaces. + """ + + from gns3server.agent.gns3_copilot.gns3_client.api_handlers import get_links_handler + + with patch(f"{AH}._get_connector") as m: + m.return_value = _mock_conn([{"link_id": "l1", "link_type": "ethernet", "kernel_datapath": True}]) + result = get_links_handler({"project_id": "p1", "fields": ["link_id", "kernel_datapath"]}, ctx) + assert result["links"] == [{"link_id": "l1", "kernel_datapath": True}] + def test_get(self, ctx): from gns3server.agent.gns3_copilot.gns3_client.api_handlers import get_link_handler From 7201a870dabf0b40bbb7e7177d2a1a0232e4ff54 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 11:55:04 +0800 Subject: [PATCH 034/101] feat(dynamips): anchor every Ethernet port on a persistent TAP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dynamips links can now ride the kernel datapath like Docker, QEMU and IOU: each Ethernet slot/port gets a persistent TAP anchor (gd{id}e{slot}p{port}), created through uBridge's tap module, and the Dynamips hypervisor opens it with nio create_tap — the hypervisor holds the fd, exactly QEMU's shape. Links attach the anchor to their per-link kernel bridge; the legacy UDP tunnel remains the relay datapath for every other port. Anchor lifecycle: born at node start (tap module probed with one create/delete cycle, sweep of stale devices, ownership handed to the server user, born down); a kernel link opens the anchor in the hypervisor, binds the slot/port and enslaves it; removing the link releases the fd again (nio delete) and detaches. A link bound while the router is stopped is wired by the next start (deferred wiring). Node close follows QEMU's stop order — the hypervisor's TAP NIOs go before tap delete, since it is Dynamips, not uBridge, that holds the fds. Stop is not close: vm stop only halts the emulated router, so the hypervisor, the TAP fds, the bindings and the per-link bridges all survive a stop/start untouched (attach skips ports that still hold their TAP NIO). Port selection follows the adapter registry: ETHERNET_ADAPTERS and ETHERNET_WICS name the models whose ports carry Ethernet frames (WIC-1ENET included, at its Dynamips port number); serial, ATM and POS ports never anchor, and a kernel link there errors with the relay-datapath message instead of half-wiring. Adapters hot-added to a running router (OIR) get their anchors at once. The compute NIO routes accept nio_bridge alongside nio_udp (the same widening QEMU needed), carry the endpoint's suspend flag on update (it is the anchor's admin state on a kernel link), skip the hypervisor delete for a kernel NIO (its TAP NIO was already released by the slot removal), and the close path leaves NIOBridge instances to _stop_ubridge. --- .../api/routes/compute/dynamips_nodes.py | 28 +- gns3server/compute/dynamips/__init__.py | 12 +- .../compute/dynamips/adapters/adapter.py | 31 + gns3server/compute/dynamips/nodes/router.py | 347 +++++++++- .../dynamips/test_dynamips_kernel_datapath.py | 614 ++++++++++++++++++ 5 files changed, 1019 insertions(+), 13 deletions(-) create mode 100644 tests/compute/dynamips/test_dynamips_kernel_datapath.py diff --git a/gns3server/api/routes/compute/dynamips_nodes.py b/gns3server/api/routes/compute/dynamips_nodes.py index 99c1b53cd..419d96b1a 100644 --- a/gns3server/api/routes/compute/dynamips_nodes.py +++ b/gns3server/api/routes/compute/dynamips_nodes.py @@ -29,6 +29,7 @@ from fastapi.responses import StreamingResponse from gns3server import schemas from gns3server.compute.dynamips import Dynamips from gns3server.compute.dynamips.nodes.router import Router +from gns3server.compute.nios.nio_bridge import NIOBridge from .dependencies.authentication import compute_authentication, ws_compute_authentication @@ -173,12 +174,15 @@ async def reload_router(node: Router = Depends(dep_node)) -> None: @router.post( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def create_nio( - adapter_number: int, port_number: int, nio_data: schemas.UDPNIO, node: Router = Depends(dep_node) -) -> schemas.UDPNIO: + adapter_number: int, + port_number: int, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], + node: Router = Depends(dep_node), +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Add a NIO (Network Input/Output) to the node. """ @@ -191,12 +195,15 @@ async def create_nio( @router.put( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def update_nio( - adapter_number: int, port_number: int, nio_data: schemas.UDPNIO, node: Router = Depends(dep_node) -) -> schemas.UDPNIO: + adapter_number: int, + port_number: int, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], + node: Router = Depends(dep_node), +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Update a NIO (Network Input/Output) on the node. """ @@ -206,6 +213,10 @@ async def update_nio( if nio_data.filters: nio.filters = nio_data.filters nio.markers = nio_data.markers or {} + # Suspend is what the compute turns into an admin-down anchor on a + # kernel link (native carrier), so it must reach the NIO like it does + # on the Docker/QEMU/IOU routes. + nio.suspend = getattr(nio_data, "suspend", None) or False await node.slot_update_nio_binding(adapter_number, port_number, nio) return nio.asdict() @@ -221,7 +232,10 @@ async def delete_nio(adapter_number: int, port_number: int, node: Router = Depen """ nio = await node.slot_remove_nio_binding(adapter_number, port_number) - await nio.delete() + if not isinstance(nio, NIOBridge): + # A kernel NIO has no hypervisor side left: its TAP NIO was deleted + # by the slot binding removal itself. + await nio.delete() @router.post( diff --git a/gns3server/compute/dynamips/__init__.py b/gns3server/compute/dynamips/__init__.py index 2013f0c35..b9674247f 100644 --- a/gns3server/compute/dynamips/__init__.py +++ b/gns3server/compute/dynamips/__init__.py @@ -39,6 +39,7 @@ from gns3server.utils.asyncio import subprocess_check_output, wait_run_in_execut from gns3server.utils.interfaces import is_interface_up from ..base_manager import BaseManager +from ..nios.nio_bridge import NIOBridge from ..port_manager import PortManager # Adapters @@ -364,7 +365,16 @@ class Dynamips(BaseManager): """ nio = None - if nio_settings["type"] == "nio_udp": + if nio_settings["type"] == "nio_bridge": + # Kernel-datapath NIO: the router binds the port's anchor TAP to + # a hypervisor nio_tap and enslaves it into this per-link bridge + # (Router._attach_kernel_nio). No hypervisor NIO is created here. + nio = NIOBridge(nio_settings["bridge"]) + nio.filters = nio_settings.get("filters", {}) + nio.markers = nio_settings.get("markers", {}) + nio.suspend = nio_settings.get("suspend", False) + return nio + elif nio_settings["type"] == "nio_udp": lport = nio_settings["lport"] rhost = nio_settings["rhost"] rport = nio_settings["rport"] diff --git a/gns3server/compute/dynamips/adapters/adapter.py b/gns3server/compute/dynamips/adapters/adapter.py index afa2a3e52..ced27bb82 100644 --- a/gns3server/compute/dynamips/adapters/adapter.py +++ b/gns3server/compute/dynamips/adapters/adapter.py @@ -15,6 +15,37 @@ # along with this program. If not, see . +# Adapter and WIC models whose ports carry Ethernet frames (by __str__ +# name, matching the controller's port matrix). Only these ports get a +# kernel-datapath anchor TAP: a TAP cannot carry the serial/ATM/POS +# encapsulations the other models speak. +ETHERNET_ADAPTERS = frozenset( + { + "C1700-MB-1FE", + "C2600-MB-1E", + "C2600-MB-1FE", + "C2600-MB-2E", + "C2600-MB-2FE", + "C7200-IO-2FE", + "C7200-IO-FE", + "C7200-IO-GE-E", + "GT96100-FE", + "Leopard-2FE", + "NM-16ESW", + "NM-1E", + "NM-1FE-TX", + "NM-4E", + "PA-2FE-TX", + "PA-4E", + "PA-8E", + "PA-FE-TX", + "PA-GE", + } +) + +ETHERNET_WICS = frozenset({"WIC-1ENET"}) + + class Adapter: """ Base class for adapters. diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index c82e39cbf..6c8afaf5f 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -22,6 +22,7 @@ http://github.com/GNS3/dynamips/blob/master/README.hypervisor#L77 import asyncio import base64 import binascii +import contextlib import glob import logging import os @@ -37,10 +38,16 @@ from gns3server.utils.hostname import is_ios_hostname_valid from gns3server.utils.images import md5sum from ...base_node import BaseNode +from ...error import NodeError +from ...kernel_datapath import KernelDatapathMixin +from ...nios.nio_bridge import NIOBridge +from ...nios.nio_tap import NIOTAP +from ...ubridge.ubridge_error import UbridgeError +from ..adapters.adapter import ETHERNET_ADAPTERS, ETHERNET_WICS from ..dynamips_error import DynamipsError -class Router(BaseNode): +class Router(KernelDatapathMixin, BaseNode): """ Dynamips router implementation. @@ -123,6 +130,13 @@ class Router(BaseNode): self._slots = [] self._ghost_flag = ghost_flag self._memory_watcher = None + # Kernel datapath: the persistent anchor TAP per Ethernet slot/port + # (in _kernel_taps) and the hypervisor TAP NIO holding each anchor + # open while a kernel link binds the port (in _tap_nios). + self._kernel_taps = {} + self._tap_nios = {} + self._tap_datapath = False + self._ubridge_tc_caps = None if not ghost_flag: if not dynamips_id: @@ -320,7 +334,16 @@ class Router(BaseNode): # an empty private-config can prevent a router to boot. private_config_path = "" - await self._hypervisor.send(f'vm set_config "{self._name}" "{startup_config_path}" "{private_config_path}"') + # uBridge owns the kernel datapath's anchor TAPs; start it and + # create them before the VM boots, so kernel links bound while + # the node was stopped wire themselves as IOS comes up. + await self._prepare_tap_datapath() + + await self._hypervisor.send( + 'vm set_config "{name}" "{startup}" "{private}"'.format( + name=self._name, startup=startup_config_path, private=private_config_path + ) + ) await self._hypervisor.send(f'vm start "{self._name}"') self.status = "started" log.debug(f'router "{self._name}" [{self._id}] has been started') @@ -421,7 +444,9 @@ class Router(BaseNode): for adapter in self._slots: if adapter is not None: for nio in adapter.ports.values(): - if nio: + # NIOBridge has no hypervisor side to close here — its + # release runs in _stop_ubridge below. + if nio and not isinstance(nio, NIOBridge): await nio.close() await self._stop_ubridge() @@ -1143,6 +1168,11 @@ class Router(BaseNode): log.debug(f'Router "{self._name}" [{self._id}]: OIR start event sent to slot {slot_number}') + if self._tap_datapath: + # The new adapter's Ethernet ports need their anchor TAPs now + # (not at next start) for kernel links to attach at once. + await self._create_taps() + async def slot_remove_binding(self, slot_number): """ Removes a slot binding (a module from a slot). @@ -1281,6 +1311,19 @@ class Router(BaseNode): if not adapter.port_exists(port_number): raise DynamipsError(f"Port {port_number} does not exist on adapter {adapter}") + if isinstance(nio, NIOBridge): + # Kernel datapath: the port's anchor TAP is opened in the + # hypervisor and enslaved into the NIO's per-link bridge (the + # wiring is deferred to the node's start when no anchors exist). + adapter.add_nio(port_number, nio) + await self._attach_kernel_nio(slot_number, port_number, nio) + log.debug( + 'Router "{name}" [{id}]: {nio} bound to port {slot_number}/{port_number}'.format( + name=self._name, id=self._id, nio=nio, slot_number=slot_number, port_number=port_number + ) + ) + return + try: await self._hypervisor.send(f'vm slot_add_nio_binding "{self._name}" {slot_number} {port_number} {nio}') except DynamipsError: @@ -1302,6 +1345,15 @@ class Router(BaseNode): :param nio: NIO instance to add to the slot/port """ + if isinstance(nio, NIOBridge): + # Filters, markers or suspend changed on a kernel link: re-apply + # on the anchor, no re-binding and no re-enslaving. A NIO bound + # while the node was stopped carries its state; start applies it. + anchor = self._kernel_host_ifc(slot_number, port_number) + if anchor is not None and self.ubridge: + await self._kernel_update(anchor, nio) + await self._set_adapter_carrier(slot_number, not nio.suspend, port_number) + return await nio.update() async def slot_remove_nio_binding(self, slot_number, port_number): @@ -1326,12 +1378,27 @@ class Router(BaseNode): raise DynamipsError(f"Port {port_number} does not exist on adapter {adapter}") await self.stop_capture(slot_number, port_number) - await self.slot_disable_nio(slot_number, port_number) - await self._hypervisor.send(f'vm slot_remove_nio_binding "{self._name}" {slot_number} {port_number}') nio = adapter.get_nio(port_number) if nio is None: return + + if isinstance(nio, NIOBridge): + await self._remove_kernel_nio_binding(slot_number, port_number, nio) + adapter.remove_nio(port_number) + log.debug( + 'Router "{name}" [{id}]: {nio} removed from port {slot_number}/{port_number}'.format( + name=self._name, id=self._id, nio=nio, slot_number=slot_number, port_number=port_number + ) + ) + return nio + + await self.slot_disable_nio(slot_number, port_number) + await self._hypervisor.send( + 'vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format( + name=self._name, slot_number=slot_number, port_number=port_number + ) + ) await nio.close() adapter.remove_nio(port_number) @@ -1421,6 +1488,21 @@ class Router(BaseNode): if not nio: raise DynamipsError(f"Port {slot_number}/{port_number} is not connected") + if isinstance(nio, NIOBridge): + # Kernel link: capture on the port's TAP anchor (AF_PACKET), + # not in the Dynamips NIO (this link has no Dynamips NIO). + nio.start_packet_capture(output_file, data_link_type) + if self.ubridge: + anchor = self._kernel_host_ifc(slot_number, port_number) + if anchor is not None: + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + log.debug( + 'Router "{name}" [{id}]: starting packet capture on port {slot_number}/{port_number}'.format( + name=self._name, id=self._id, slot_number=slot_number, port_number=port_number + ) + ) + return + if nio.input_filter[0] is not None and nio.output_filter[0] is not None: raise DynamipsError(f"Port {port_number} has already a filter applied on {adapter}") await nio.start_packet_capture(output_file, data_link_type) @@ -1448,10 +1530,265 @@ class Router(BaseNode): if not nio.capturing: return + + if isinstance(nio, NIOBridge): + nio.stop_packet_capture() + if self.ubridge and self._kernel_host_ifc(slot_number, port_number) is not None: + await self._ubridge_send("capture stop_kernel") + return await nio.stop_packet_capture() log.debug(f'Router "{self._name}" [{self._id}]: stopping packet capture on port {slot_number}/{port_number}') + # ------------------------------------------------------------------ + # Kernel datapath: the port anchor is a persistent TAP held by the + # Dynamips hypervisor (nio create_tap) — QEMU's shape, not IOU's + # ------------------------------------------------------------------ + + def _tap_name(self, slot_number, port_number): + """ + Deterministic anchor TAP name for a slot/port. The ``gd`` prefix + keeps it out of the ``gns3`` bridge/TAP name space (and apart from + Docker's ``gv``/``gc``, QEMU's ``gq`` and IOU's ``gi``); 8 hex chars + of the node id plus slot/port keep it unique and within IFNAMSIZ + (15), even for WIC port numbers (16, 32, 48). + """ + + return f"gd{self._id.replace('-', '')[:8]}e{slot_number}p{port_number}" + + def _kernel_host_ifc(self, slot_number, port_number=0): + """ + The persistent TAP this Ethernet slot/port owns, or None for serial, + ATM or POS ports (never anchored) and before the router started. + """ + + return self._kernel_taps.get((slot_number, port_number)) + + def _kernel_anchors(self): + """ + Every anchor TAP this router currently owns. + """ + + return set(self._kernel_taps.values()) + + def _kernel_error(self, message): + return DynamipsError(message) + + def _ethernet_slot_ports(self): + """ + Every (slot, port) that can carry Ethernet frames: slot adapters + whose model is Ethernet, plus WIC-1ENET ports (numbered from 16 per + WIC slot, the Dynamips convention) in any motherboard. + """ + + ports = [] + for slot_number, adapter in enumerate(self._slots): + if adapter is None: + continue + if str(adapter) in ETHERNET_ADAPTERS: + ports.extend((slot_number, port_number) for port_number in adapter.ports) + for wic_slot_number, wic in enumerate(adapter.wics or []): + if wic is not None and str(wic) in ETHERNET_WICS: + ports.append((slot_number, 16 * (wic_slot_number + 1))) + return ports + + @property + def _ethernet_adapters(self): + """ + Slot adapters owning at least one anchor — the mixin's per-link + bridge sweep walks them. Dynamips ports live in slot adapters (WIC + ports included), so this derives from the anchor map rather than + the slot list. + """ + + adapters = [] + for slot_number, _port_number in self._kernel_taps: + if slot_number < len(self._slots): + adapter = self._slots[slot_number] + if adapter is not None and adapter not in adapters: + adapters.append(adapter) + return adapters + + async def _prepare_tap_datapath(self): + """ + Start uBridge and create the persistent TAP every Ethernet slot/port + owns — the anchor role QEMU's TAPs and Docker's veth host ends play. + The Dynamips hypervisor opens the device (ownership handed to this + user by _create_taps), so it needs no privileges of its own to hold + the fd. A uBridge without the tap module keeps this router on the + relay datapath: every link rides the NIOUDP tunnel, exactly as + before. + + Nothing here is undone at node stop — the hypervisor, its NIO + bindings and the per-link kernel bridges all outlive ``vm stop``, + which is what makes a stopped router's links self-heal on the next + start. The anchors are only retired when the node closes. + """ + + if self._ghost_flag: + # Ghost IOS images share RAM with a real router and never link. + return + + try: + await self._start_ubridge() + except NodeError as e: + # A Dynamips router without links never needed uBridge; keep the + # node bootable and let the relay raise at link time (as before). + log.warning("Router '%s': %s", self._name, e) + return + + # uBridge (re)started: the tc capabilities must be probed again. + self._ubridge_tc_caps = None + if not self._tap_datapath: + probe = f"gd{self._id.replace('-', '')[:8]}prob" + try: + await self._ubridge_send(f'tap create "{probe}"') + except UbridgeError as e: + message = ( + f"Router '{self._name}': uBridge cannot create persistent TAPs ({e}); " + "this node runs on the relay datapath and cannot carry kernel links" + ) + log.warning(message) + self.project.emit("log.warning", {"message": message}) + return + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{probe}"') + self._tap_datapath = True + + await self._create_taps() + # Kernel links bound while the router was stopped could not wire (no + # anchors existed); wire them now, before IOS comes up. + for slot_number, adapter in enumerate(self._slots): + if adapter is None: + continue + for port_number, nio in list(adapter.ports.items()): + if isinstance(nio, NIOBridge): + await self._attach_kernel_nio(slot_number, port_number, nio) + + async def _create_taps(self): + """ + Create the persistent TAP every Ethernet slot/port owns. Each TAP + starts DOWN (carrier off until a kernel link attaches); uBridge + creates the device and hands ownership to this user, so the + unprivileged Dynamips hypervisor can open it. A port whose anchor + already exists (a restart, or an adapter hot-added to a running + router) keeps it — an anchor is never recreated under a live link. + A persistent TAP outlives its creator, so a leftover from a previous + run (crash, kill) is swept first. + """ + + for slot_number, port_number in self._ethernet_slot_ports(): + if (slot_number, port_number) in self._kernel_taps: + continue + tap = self._tap_name(slot_number, port_number) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._ubridge_send(f'tap create "{tap}"') + try: + await self._ubridge_send(f"tap set_owner {tap} {os.getuid()}") + except UbridgeError as e: + # Only root could open the TAP now: the hypervisor would fail + # to bind its NIO, which is worth surfacing at that point. + log.warning("Router '%s': could not hand TAP %s to uid %s: %s", self._name, tap, os.getuid(), e) + await self._ubridge_send(f'link set "{tap}" down') + self._kernel_taps[(slot_number, port_number)] = tap + + async def _attach_kernel_nio(self, slot_number, port_number, nio): + """ + Wire a kernel link's NIO (NIOBridge) on a port: open the port's + anchor TAP in the Dynamips hypervisor (``nio create_tap`` — the + hypervisor, not uBridge, holds the fd, the same shape as QEMU), bind + it to the slot/port, then enslave the anchor into the per-link + kernel bridge — the shared mixin flow. On a router that is not + running the wiring is deferred to its start (no anchors exist yet). + A port whose wiring survived a stop (nothing is torn down there) is + left alone: re-binding would collide on both hypervisor and bridge. + """ + + if (slot_number, port_number) in self._tap_nios: + return + anchor = self._kernel_host_ifc(slot_number, port_number) + if anchor is None: + if self.status != "started": + return + raise self._kernel_error( + "Port {slot}/{port} of router '{name}' runs on the relay datapath " + "(this uBridge cannot create persistent TAPs) and cannot carry a kernel link".format( + slot=slot_number, port=port_number, name=self._name + ) + ) + + tap_nio = NIOTAP(self._hypervisor, anchor) + await tap_nio.create() + try: + await self._hypervisor.send( + 'vm slot_add_nio_binding "{name}" {slot_number} {port_number} {nio}'.format( + name=self._name, slot_number=slot_number, port_number=port_number, nio=tap_nio.name + ) + ) + except DynamipsError: + with contextlib.suppress(DynamipsError): + await tap_nio.delete() + raise + await self.slot_enable_nio(slot_number, port_number) + self._tap_nios[(slot_number, port_number)] = tap_nio + await self._kernel_attach(anchor, nio) + await self._set_adapter_carrier(slot_number, not nio.suspend, port_number) + + async def _remove_kernel_nio_binding(self, slot_number, port_number, nio): + """ + Release a kernel link's NIO: tear the anchor out of the per-link + kernel bridge (markers, tc, delif — the shared mixin flow), then + unbind the port and delete the hypervisor's TAP NIO, releasing the + anchor fd. The anchor device itself survives for the next link. + """ + + if self.ubridge and self._kernel_host_ifc(slot_number, port_number) is not None: + await self._remove_kernel_nio(nio, slot_number, port_number) + tap_nio = self._tap_nios.pop((slot_number, port_number), None) + if tap_nio is not None: + with contextlib.suppress(DynamipsError, OSError): + await self.slot_disable_nio(slot_number, port_number) + with contextlib.suppress(DynamipsError, OSError): + await self._hypervisor.send( + 'vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format( + name=self._name, slot_number=slot_number, port_number=port_number + ) + ) + with contextlib.suppress(DynamipsError, OSError): + await tap_nio.delete() + + async def _stop_ubridge(self): + """ + Stops uBridge, retiring the kernel datapath's host state first — in + the QEMU order, since here it is the Dynamips hypervisor (not + uBridge) that holds the anchor fds: the per-link kernel bridges need + the control channel, and the anchor TAPs (persistent devices) must + be deleted through it after the hypervisor's TAP NIOs are gone, or a + device Dynamips still holds open would outlive its retirement. The + next start spawns a fresh uBridge, so the tc capabilities must be + probed again. + """ + + if self.ubridge: + for (slot_number, port_number), tap_nio in list(self._tap_nios.items()): + with contextlib.suppress(DynamipsError, OSError): + await self._hypervisor.send( + 'vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format( + name=self._name, slot_number=slot_number, port_number=port_number + ) + ) + with contextlib.suppress(DynamipsError, OSError): + await tap_nio.delete() + self._tap_nios.clear() + for tap in self._kernel_taps.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._remove_kernel_bridges() + self._kernel_taps.clear() + self._ubridge_tc_caps = None + await super()._stop_ubridge() + def _create_slots(self, numslots): """ Creates the appropriate number of slots for this router. diff --git a/tests/compute/dynamips/test_dynamips_kernel_datapath.py b/tests/compute/dynamips/test_dynamips_kernel_datapath.py new file mode 100644 index 000000000..b204f0dc5 --- /dev/null +++ b/tests/compute/dynamips/test_dynamips_kernel_datapath.py @@ -0,0 +1,614 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Kernel-datapath wiring for Dynamips routers: the persistent TAP every +Ethernet slot/port owns (Ethernet adapters and WIC-1ENET ports only — serial, +ATM and POS adapters never anchor), the hypervisor TAP NIO (nio create_tap) +that holds the anchor open, per-link enslavement, the deferred wiring of +kernel links bound while the node was stopped, and the stop order (the +hypervisor's fds released before the TAPs are unpersisted — QEMU's order, +since here it is Dynamips that holds them). +""" + +import os +import shutil +import contextlib + +import pytest +import pytest_asyncio + +from unittest.mock import MagicMock, patch + +from tests.utils import AsyncioMagicMock + +from gns3server.compute.dynamips import Dynamips +from gns3server.compute.dynamips.nodes.router import Router +from gns3server.compute.dynamips.dynamips_error import DynamipsError +from gns3server.compute.dynamips.adapters.gt96100_fe import GT96100_FE +from gns3server.compute.dynamips.adapters.pa_4t import PA_4T +from gns3server.compute.dynamips.adapters.pa_fe_tx import PA_FE_TX +from gns3server.compute.dynamips.adapters.wic_1enet import WIC_1ENET +from gns3server.compute.ubridge.ubridge_error import UbridgeError +from gns3server.compute.nios.nio_bridge import NIOBridge + + +NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" +BRIDGE = "gns3a1b2c3d4e5f" +TAP00 = "gd00010203e0p0" # the router fixture's node id, slot 0 port 0 +TAP01 = "gd00010203e0p1" +TAP016 = "gd00010203e0p16" # WIC-1ENET port in WIC slot 0 + + +class FakeDynamipsHypervisor: + """Records the hypervisor protocol; answers status queries.""" + + def __init__(self, status_code=2): + self.commands = [] + self._status_code = status_code + + def is_running(self): + return True + + async def send(self, command): + self.commands.append(command) + if command.startswith("vm get_status"): + return [str(self._status_code)] + return ["100-OK"] + + +@pytest_asyncio.fixture +async def manager(port_manager): + + m = Dynamips.instance() + m.port_manager = port_manager + return m + + +@pytest_asyncio.fixture +async def router(compute_project, manager): + + router = Router("R1", NODE_ID, compute_project, manager) + router._hypervisor = FakeDynamipsHypervisor() + router._ubridge_hypervisor = MagicMock() + router._ubridge_hypervisor.is_running.return_value = True + router._ubridge_send = AsyncioMagicMock() + # slot 0: a 2-port Ethernet motherboard with a WIC-1ENET in WIC slot 0 + # (port 16); slot 1: a serial PA that must never anchor. + router._create_slots(2) + router._slots[0] = GT96100_FE() + router._slots[0]._wics[0] = WIC_1ENET() + router._slots[1] = PA_4T() + yield router + # compute_project is the shared, id-fixed project every compute test + # file reuses (ProjectManager caches by id); leave its dynamips module + # directory as this fixture found it (absent) — a later manager test + # creates it with os.makedirs and no exist_ok. + shutil.rmtree(router._working_directory, ignore_errors=True) + with contextlib.suppress(OSError): + os.rmdir(compute_project.module_working_path(manager.module_name.lower())) + + +async def _nio(router, **kwargs): + settings = {"type": "nio_bridge", "bridge": BRIDGE} + settings.update(kwargs) + return await router.manager.create_nio(router, settings) + + +def _hypervisor_commands(router): + return list(router._hypervisor.commands) + + +def _ubridge_commands(router): + return [str(c[0][0]) if c[0] else "" for c in router._ubridge_send.call_args_list] + + +# --------------------------------------------------------------------------- +# Anchor topology: which ports anchor at all +# --------------------------------------------------------------------------- + + +def test_ethernet_slot_ports_skip_serial_and_include_wics(router): + """ + Only Ethernet-carrying ports anchor: the PA's two ports plus the + WIC-1ENET port (numbered from 16 per WIC slot), never the serial PA. + """ + + assert router._ethernet_slot_ports() == [(0, 0), (0, 1), (0, 16)] + + +def test_tap_name_fits_ifnamsiz(router): + """ + The anchor name has to fit IFNAMSIZ (15) even for WIC port numbers. + """ + + assert len(router._tap_name(6, 48)) <= 15 + + +def test_kernel_host_ifc_is_none_before_start(router): + assert router._kernel_host_ifc(0, 0) is None + + +# --------------------------------------------------------------------------- +# Anchor lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_creates_one_tap_per_ethernet_port(router): + """ + On a capable uBridge every Ethernet slot/port gets its persistent TAP — + swept of leftovers first, ownership handed to this user (so the + unprivileged hypervisor can open it), born down. + """ + + router._ubridge_send.reset_mock() + await router._prepare_tap_datapath() + + assert router._tap_datapath is True + assert router._kernel_taps == {(0, 0): TAP00, (0, 1): TAP01, (0, 16): TAP016} + router._ubridge_send.assert_any_call(f'tap delete "{TAP00}"') + router._ubridge_send.assert_any_call(f'tap create "{TAP00}"') + router._ubridge_send.assert_any_call(f"tap set_owner {TAP016} {os.getuid()}") + router._ubridge_send.assert_any_call(f'link set "{TAP016}" down') + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_falls_back_without_the_tap_module(router): + """ + A uBridge without the tap module keeps the router on the relay datapath — + it still starts, it just cannot carry kernel links. + """ + + async def refuse_probe(command): + if "prob" in command: + raise UbridgeError("Unknown command") + return ["100-OK"] + + router._ubridge_send.side_effect = refuse_probe + with patch.object(router.project, "emit") as emit: + await router._prepare_tap_datapath() + + assert router._tap_datapath is False + assert router._kernel_taps == {} + assert emit.called + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_skips_ghost_routers(compute_project, manager): + """ + Ghost IOS images share RAM with a real router and never link: no uBridge, + no anchors. + """ + + ghost = Router("Ghost", NODE_ID, compute_project, manager, ghost_flag=True) + ghost._start_ubridge = AsyncioMagicMock() + await ghost._prepare_tap_datapath() + + assert ghost._tap_datapath is False + assert not ghost._start_ubridge.called + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_keeps_existing_anchors_across_a_restart(router): + """ + Nothing is undone at node stop, so a restart must not recreate an anchor + a surviving kernel link is still enslaved to. + """ + + await router._prepare_tap_datapath() + router._ubridge_send.reset_mock() + + await router._prepare_tap_datapath() + + assert router._kernel_taps == {(0, 0): TAP00, (0, 1): TAP01, (0, 16): TAP016} + assert not any("tap create" in c or "tap delete" in c for c in _ubridge_commands(router)) + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_wires_nios_bound_while_stopped(router): + """ + A kernel link bound to a stopped router could only be stored (no anchors + existed); the start wires it — the hypervisor opens the anchor and binds + the port, the anchor joins the per-link bridge. + """ + + router.status = "stopped" + nio = await _nio(router) + router._slots[0].add_nio(0, nio) + + router._ubridge_send.reset_mock() + router._hypervisor.commands.clear() + await router._prepare_tap_datapath() + + hypervisor = _hypervisor_commands(router) + assert any(c.startswith("nio create_tap tap-") and c.endswith(f"{TAP00}") for c in hypervisor) + assert any(c.startswith(f"vm slot_add_nio_binding \"R1\" 0 0 tap-") for c in hypervisor) + router._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') + assert (0, 0) in router._tap_nios + + +# --------------------------------------------------------------------------- +# Link attach / detach +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_attach_kernel_nio_opens_the_anchor_in_the_hypervisor_and_enslaves_it(router): + """ + A kernel link opens the port's anchor TAP in the Dynamips hypervisor + (Dynamips holds the fd — QEMU's shape), binds it to the slot/port, then + enslaves the anchor into the per-link kernel bridge. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + + await router._attach_kernel_nio(0, 0, nio) + + hypervisor = _hypervisor_commands(router) + assert any(c.startswith("nio create_tap tap-") and c.endswith(f"{TAP00}") for c in hypervisor) + assert any(c.startswith('vm slot_add_nio_binding "R1" 0 0 tap-') for c in hypervisor) + router._ubridge_send.assert_any_call(f'brctl create "{BRIDGE}"') + router._ubridge_send.assert_any_call(f'link set "{BRIDGE}" up') + router._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') + router._ubridge_send.assert_any_call(f'link set "{TAP00}" up') + + +@pytest.mark.asyncio +async def test_attach_kernel_nio_without_anchor_on_a_running_node_raises(router): + """ + A kernel link on a serial port (or a router that never anchored) is an + error naming the relay datapath, not a silent fallback. + """ + + router.status = "started" + nio = await _nio(router) + + with pytest.raises(DynamipsError, match="relay datapath"): + await router._attach_kernel_nio(1, 0, nio) + + +@pytest.mark.asyncio +async def test_attach_kernel_nio_defers_on_a_stopped_node(router): + """ + On a stopped router there are no anchors yet: the NIO is stored and the + wiring happens at start (test above), so nothing is sent. + """ + + router.status = "stopped" + nio = await _nio(router) + + await router._attach_kernel_nio(1, 0, nio) + + assert _hypervisor_commands(router) == [] + assert not router._ubridge_send.called + + +@pytest.mark.asyncio +async def test_attach_kernel_nio_is_idempotent_across_a_restart(router): + """ + A stop tears nothing down, so a wired port must not re-bind on the next + start: a second nio create_tap would orphan the first NIO's fd and the + re-enslavement would collide on the bridge. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + await router._attach_kernel_nio(0, 0, nio) + hypervisor_first = list(_hypervisor_commands(router)) + router._ubridge_send.reset_mock() + + await router._attach_kernel_nio(0, 0, nio) + + assert _hypervisor_commands(router) == hypervisor_first + assert not router._ubridge_send.called + + +@pytest.mark.asyncio +async def test_add_nio_binding_relay_keeps_the_udp_tunnel(router): + """ + A relay link is wired exactly as before: the hypervisor NIO is a UDP NIO + tunneled through loopback into the node's uBridge relay. + """ + + router.status = "started" + nio = await router.manager.create_nio( + router, {"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001} + ) + + await router.slot_add_nio_binding(0, 0, nio) + + hypervisor = _hypervisor_commands(router) + assert any(c.startswith("nio create_udp udp-") for c in hypervisor) + assert any(c.startswith('vm slot_add_nio_binding "R1" 0 0 udp-') for c in hypervisor) + ubridge = _ubridge_commands(router) + assert any(c.startswith("bridge create DYNAMIPS-") for c in ubridge) + assert any(c.startswith("bridge add_nio_udp DYNAMIPS-") for c in ubridge) + assert any(c.startswith("bridge start DYNAMIPS-") for c in ubridge) + + +@pytest.mark.asyncio +async def test_add_nio_binding_kernel_binds_and_enslaves(router): + """ + The route-level add path dispatches a NIOBridge to the kernel wiring and + stores it in the slot adapter like any NIO. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + + await router.slot_add_nio_binding(0, 0, nio) + + assert router._slots[0].get_nio(0) is nio + router._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') + + +@pytest.mark.asyncio +async def test_update_nio_binding_kernel_reapplies_on_the_anchor(router): + """ + A filter update on an attached kernel link reconciles on the anchor + (tc) without re-binding the port or re-enslaving the TAP. + """ + + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router, filters={"delay": [10]}) + router._slots[0].add_nio(0, nio) + + await router.slot_update_nio_binding(0, 0, nio) + + router._ubridge_send.assert_any_call(f'tc reset "{TAP00}"') + router._ubridge_send.assert_any_call(f'tc netem set "{TAP00}" delay 10') + assert not any("slot_add_nio_binding" in c for c in _hypervisor_commands(router)) + assert not any("brctl addif" in c for c in _ubridge_commands(router)) + + +@pytest.mark.asyncio +async def test_update_nio_binding_kernel_suspend_flaps_the_anchor(router): + """ + Suspending a kernel link admin-downs the anchor — the native carrier, + which is why the route must copy suspend onto the NIO. + """ + + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + router._slots[0].add_nio(0, nio) + nio.suspend = True + + await router.slot_update_nio_binding(0, 0, nio) + + router._ubridge_send.assert_any_call(f'link set "{TAP00}" down') + + +@pytest.mark.asyncio +async def test_remove_nio_binding_kernel_releases_the_fd_and_detaches(router): + """ + Removing a kernel link detaches the anchor from the per-link bridge, + resets its qdisc, and unbinds the port plus deletes the hypervisor TAP + NIO — releasing Dynamips's fd while the control channels live. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + router._slots[0].add_nio(0, nio) + await router._attach_kernel_nio(0, 0, nio) + router._ubridge_send.reset_mock() + router._hypervisor.commands.clear() + + removed = await router.slot_remove_nio_binding(0, 0) + + assert removed is nio + assert router._slots[0].get_nio(0) is None + router._ubridge_send.assert_any_call(f'tc reset "{TAP00}"') + router._ubridge_send.assert_any_call(f'brctl delif "{BRIDGE}" "{TAP00}"') + router._ubridge_send.assert_any_call(f'brctl delete "{BRIDGE}"') + hypervisor = _hypervisor_commands(router) + assert any(c.startswith('vm slot_remove_nio_binding "R1" 0 0') for c in hypervisor) + assert any(c.startswith("nio delete tap-") for c in hypervisor) + assert (0, 0) not in router._tap_nios + + +# --------------------------------------------------------------------------- +# Stop order (the hypervisor holds the anchor fds — QEMU's order) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_stop_ubridge_releases_hypervisor_fds_before_deleting_taps(router): + """ + The hypervisor's TAP NIOs are unbound and deleted first (closing every + anchor fd), then the TAPs are unpersisted, then the per-link bridges go — + all before uBridge itself stops. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + router._slots[0].add_nio(0, nio) + await router._attach_kernel_nio(0, 0, nio) + router._ubridge_send.reset_mock() + router._hypervisor.commands.clear() + + with patch( + "gns3server.compute.base_node.BaseNode._stop_ubridge", new=AsyncioMagicMock() + ) as stop: + await router._stop_ubridge() + + hypervisor = _hypervisor_commands(router) + ubridge = _ubridge_commands(router) + unbind = next(i for i, c in enumerate(hypervisor) if c.startswith('vm slot_remove_nio_binding "R1" 0 0')) + nio_delete = next(i for i, c in enumerate(hypervisor) if c.startswith("nio delete tap-")) + tap_delete = next(i for i, c in enumerate(ubridge) if c == f'tap delete "{TAP00}"') + brctl_delete = next(i for i, c in enumerate(ubridge) if c == f'brctl delete "{BRIDGE}"') + assert unbind < nio_delete + assert tap_delete < brctl_delete + assert stop.called + assert router._kernel_taps == {} + assert router._tap_nios == {} + assert router._ubridge_tc_caps is None + + +@pytest.mark.asyncio +async def test_remove_kernel_bridges_covers_wic_ports(router): + """ + The bridge sweep walks adapters derived from the anchor map, so a kernel + link on a WIC-1ENET port (inside a non-Ethernet motherboard) is found. + """ + + router._kernel_taps[(0, 16)] = TAP016 + wic_nio = await _nio(router) + router._slots[0].add_nio(16, wic_nio) + + await router._remove_kernel_bridges() + + router._ubridge_send.assert_any_call(f'brctl delete "{BRIDGE}"') + + +@pytest.mark.asyncio +async def test_slot_add_binding_hot_adds_anchors(router): + """ + An adapter inserted while the router runs (OIR) gets its anchors created + at once, so a kernel link can attach without a restart. + """ + + router.status = "started" + router._tap_datapath = True + router._kernel_taps[(0, 0)] = TAP00 + router._slots[1] = None # free the serial slot for the hot-insert + + await router.slot_add_binding(1, PA_FE_TX()) + + assert router._kernel_taps[(1, 0)] == "gd00010203e1p0" + + +# --------------------------------------------------------------------------- +# Capture (kernel: the anchor; relay: the Dynamips NIO's own pcap) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_capture_on_a_kernel_link_uses_the_anchor(router): + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + router._slots[0].add_nio(0, nio) + + await router.start_capture(0, 0, "/tmp/test.pcap") + router._ubridge_send.assert_any_call(f'capture start_kernel {TAP00} "/tmp/test.pcap"') + assert not any("bind_filter" in c for c in _hypervisor_commands(router)) + + await router.stop_capture(0, 0) + router._ubridge_send.assert_any_call("capture stop_kernel") + + +@pytest.mark.asyncio +async def test_capture_on_a_relay_link_uses_the_dynamips_nio(router): + nio = await router.manager.create_nio( + router, {"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001} + ) + router._slots[0].add_nio(0, nio) + + await router.start_capture(0, 0, "/tmp/test.pcap") + assert not any("capture start_kernel" in c for c in _ubridge_commands(router)) + hypervisor = _hypervisor_commands(router) + assert any(c.startswith("nio bind_filter udp-") and c.endswith(" capture") for c in hypervisor) + assert any(c.startswith("nio setup_filter udp-") for c in hypervisor) + + +# --------------------------------------------------------------------------- +# Markers (the polymorphic anchor: relay bridge name on relay, TAP on kernel) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_markers_on_a_kernel_link_attach_to_the_anchor(router): + """ + The marker reconcile is datapath-agnostic: with the anchor as the + location, the attach/toggle/delete primitives translate to + marker *_kernel on the TAP. + """ + + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + nio.markers = { + "m1": {"bpf": "vlan 100", "link_id": "lk1", "enabled": True, "data_link_type": "DLT_EN10MB"}, + } + router._slots[0].add_nio(0, nio) + + await router._ubridge_apply_markers(TAP00, nio) + + router._ubridge_send.assert_any_call( + f'marker add_kernel m1 {TAP00} "vlan 100" link lk1 pcap "{router.project.markers_working_directory()}/' + f'{router.id}_lk1_m1.pcap"' + ) + assert router._marker_filter_bridges[("m1", "lk1")] == TAP00 + + await router._ubridge_set_marker_filter_state("m1", False) + router._ubridge_send.assert_any_call(f"marker enable_kernel {TAP00} m1 off") + + await router.delete_marker_capture("m1", "lk1", nio) + router._ubridge_send.assert_any_call(f"marker delete_kernel {TAP00} m1") + + +@pytest.mark.asyncio +async def test_markers_on_a_relay_link_keep_the_ubridge_shape(router): + """ + On the relay datapath the same reconcile talks to the uBridge relay + bridge behind the Dynamips UDP tunnel — unchanged behaviour. + """ + + nio = await router.manager.create_nio( + router, {"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001} + ) + nio.markers = { + "m1": {"bpf": "vlan 100", "link_id": "lk1", "enabled": True, "data_link_type": "DLT_EN10MB"}, + } + router._slots[0].add_nio(0, nio) + tunnel_bridge = "DYNAMIPS-20000-20001" + + await router._ubridge_apply_markers(tunnel_bridge, nio) + + router._ubridge_send.assert_any_call( + f'bridge add_packet_filter {tunnel_bridge} m1 mark "vlan 100" link lk1 pcap ' + f'"{router.project.markers_working_directory()}/{router.id}_lk1_m1.pcap"' + ) + + +# --------------------------------------------------------------------------- +# Manager NIO construction +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_manager_create_nio_builds_the_bridge_nio(router): + """ + The Dynamips manager accepts nio_bridge settings (the kernel-datapath + NIO the controller POSTs) — without it the route 422s before the router + ever sees the link. + """ + + nio = await router.manager.create_nio( + router, {"type": "nio_bridge", "bridge": BRIDGE, "filters": {"delay": [5]}, "suspend": True} + ) + assert isinstance(nio, NIOBridge) + assert nio.bridge == BRIDGE + assert nio.filters == {"delay": [5]} + assert nio.suspend is True + assert _hypervisor_commands(router) == [] From e5f65a510a983eac387a2899a65697b31fee9aa2 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 11:55:20 +0800 Subject: [PATCH 035/101] feat(controller): offer Dynamips the kernel datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Dynamips Ethernet port can anchor a kernel link whenever its compute's uBridge has the tap module — the anchors are uBridge-created TAPs the hypervisor merely opens, the same capability QEMU gates on, so no Dynamips-specific capability is needed. Serial, ATM and POS ports are excluded per port by the existing link-type check (the controller's port matrix decides), keeping them on the relay whatever the capabilities. Eligibility tests cover the mixed pairs (docker/qemu/iou endpoints on the same compute), the strict tap-capability requirement, and the serial exclusion. --- gns3server/controller/udp_link.py | 14 +++- tests/controller/test_kernel_datapath_link.py | 64 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 416bb8050..abe1a7f78 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -182,9 +182,12 @@ class UDPLink(Link): datapath and cannot carry a kernel link at all. IOU's Ethernet bays anchor on persistent TAPs too, bound to its IOL fabric (``iol_bridge add_nio_tap``) — a separate capability, since it can - be present or absent independently of the tap module. A kernel link - is an Ethernet segment, so non-Ethernet ports (IOU serial) stay on - the relay whatever the node's capabilities. + be present or absent independently of the tap module. Dynamips + Ethernet slot ports anchor on persistent TAPs the hypervisor opens + (the tap module again — the Dynamips binary itself creates nothing), + so it gates like QEMU. A kernel link is an Ethernet segment, so + non-Ethernet ports (IOU serial, Dynamips serial/ATM/POS adapters) + stay on the relay whatever the node's capabilities. """ if not Config.instance().settings.Server.enable_kernel_datapath: @@ -214,6 +217,11 @@ class UDPLink(Link): return capabilities.get("ubridge_tap") is True if node.node_type == "iou": return capabilities.get("ubridge_iol_tap") is True + if node.node_type == "dynamips": + # The anchors are uBridge-created TAPs the Dynamips hypervisor + # merely opens (nio create_tap) — the same tap-module capability + # QEMU needs, not a Dynamips-specific one. + return capabilities.get("ubridge_tap") is True return False async def _prepare(self): diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index b61d5a644..5542783e9 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -169,6 +169,70 @@ async def test_iou_eligibility_is_its_own_capability(project): assert link._kernel_datapath_eligible(node1, node2) is False +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_for_dynamips_with_tap_support(project): + """ + Dynamips Ethernet slot ports anchor on persistent TAPs the hypervisor + opens (nio create_tap on a uBridge-created device), so they gate on the + tap module like QEMU — no Dynamips-specific capability — and mix with + docker/qemu/iou endpoints on the same compute. + """ + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} + node1 = _node(project, compute, "r1", node_type="dynamips") + node2 = _node(project, compute, "r2", node_type="dynamips") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True + + for peer in ( + _node(project, compute, "docker1"), + _node(project, compute, "qemu1", node_type="qemu"), + _node(project, compute, "iou1", node_type="iou"), + ): + link = await _link(project, node1, peer) + assert link._kernel_datapath_eligible(node1, peer) is True + + +@pytest.mark.asyncio +async def test_dynamips_eligibility_follows_the_tap_capability(project): + """ + A compute whose uBridge cannot create persistent TAPs keeps Dynamips on + the relay (the hypervisor would have nothing to open), and so does an + unreported/failed probe. + """ + + node2 = _node(project, _tap_capable_compute(), "docker1") + for tap_support in (None, False): + compute = _tap_capable_compute(tap_support=tap_support) + node1 = _node(project, compute, "r1", node_type="dynamips") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_dynamips_serial_ports(project): + """ + A Dynamips serial port (NM-4T, WIC-2T, ...) stays on the relay whatever + the compute's capabilities — the same per-port exclusion as IOU serial, + decided by the controller's port matrix. + """ + + from gns3server.controller.ports.serial_port import SerialPort + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True} + + node1 = _node(project, compute, "r1", node_type="dynamips") + node1._ports = [SerialPort("Serial0/0", 0, 0, 0)] + node2 = _node(project, compute, "r2", node_type="dynamips") + node2._ports = [SerialPort("Serial0/0", 0, 0, 0)] + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + @pytest.mark.asyncio async def test_kernel_datapath_not_eligible_for_serial_ports(project): """ From 5c2c8cdb0ca9c8329a8433bf0a5253aa22515635 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 11:55:27 +0800 Subject: [PATCH 036/101] test(e2e): a live end-to-end suite, Dynamips as the first scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live e2e verification moves into the repository instead of being rewritten as a throwaway script every time: tests/e2e/ carries a shared harness (REST client, isolated-instance lifecycle, a deliberately dumb IOS console driver, host-side kernel inspection: bridge membership, tap admin state, tc qdiscs) and each node type adds only its scenario module. The e2e marker plus 'addopts = -m "not e2e"' keeps these privileged, image-dependent tests out of normal runs; they are opt-in: pytest -m e2e tests/e2e -s. The harness targets an existing server via GNS3_E2E_URL or starts an isolated instance (own config dir — DB and paths follow it — on a free port) and deletes its project afterwards (GNS3_E2E_KEEP=1 keeps it on failure). The first scenario drives two real c7200 routers on a real IOS image through the REST API and the consoles: the kernel link is created while the routers are stopped (deferred wiring), the per-link kernel bridge enslaves exactly the two anchors, real ICMP crosses it, delay 100 shows netem on both anchors and grows the RTT (~200 ms), suspend kills the traffic on an admin-down anchor, capture writes a pcap of ICMP records, a serial link between the same routers stays on the relay and still pings, delete/ re-create and stop/start leave the wiring consistent, and deleting the project removes every anchor and bridge. A negative control runs the same topology on an isolated relay-configured instance (no per-link bridge, nothing enslaved, still pings). --- tests/e2e/README.md | 46 ++ tests/e2e/__init__.py | 0 tests/e2e/harness.py | 526 +++++++++++++++++++++ tests/e2e/test_dynamips_kernel_datapath.py | 253 ++++++++++ tests/pytest.ini | 7 + 5 files changed, 832 insertions(+) create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/harness.py create mode 100644 tests/e2e/test_dynamips_kernel_datapath.py diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 000000000..1addb9d2f --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,46 @@ + + +# Live end-to-end tests + +Tests under this directory run against a **real** server: real uBridge, real +kernel interfaces (privileged), real emulator binaries and real images. They +verify what unit tests cannot — that a datapath actually forwards traffic, +that kernel bridges actually enslave anchors, that tc impairments actually +shape packets. They are marked `e2e` and **excluded from normal test runs** +(see `tests/pytest.ini`); run them explicitly: + +```bash +pytest -m e2e tests/e2e -s # -s shows the progress prints +pytest -m e2e tests/e2e -k dynamips # one scenario +``` + +Two run modes, chosen per test through `harness.live_server(kernel=...)`: + +* **Target an existing server** — set `GNS3_E2E_URL` (e.g. + `http://127.0.0.1:3080/v3`, plus `GNS3_E2E_USER`/`GNS3_E2E_PASSWORD`, + default admin/admin). The server must already be configured for the + datapath under test (kernel links need `enable_kernel_datapath = True`). +* **Isolated instance** (the default) — the harness starts its own server + process on a free port with its own config directory (the controller DB + and every path follow the config), so nothing on the machine is touched. + It needs `ubridge`, `dynamips` and an images directory + (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally. + +Either way each test creates its own project and deletes it again. Set +`GNS3_E2E_KEEP=1` to keep the project (and a started instance) on failure +and print where to look at it. + +`harness.py` holds the shared pieces — REST client, server lifecycle, the +IOS console driver, host-side kernel inspections (bridge membership, tap +admin state, tc qdiscs) — so a new node type adds only its own scenario +module. + +Current scenarios: + +* `test_dynamips_kernel_datapath.py` — two real c7200 routers: kernel link + (anchors, per-link bridge, netem delay, suspend, capture, delete/re-create, + stop/start) plus a serial link staying on the relay, and a negative + control on an isolated relay-configured instance. diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py new file mode 100644 index 000000000..d1690f2d0 --- /dev/null +++ b/tests/e2e/harness.py @@ -0,0 +1,526 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Shared harness for live end-to-end tests (pytest marker ``e2e``). + +These tests drive a **real** gns3-server process against real uBridge, real +kernel interfaces and real emulator binaries — things a unit-test run cannot +provide (privileges, images, a live kernel). They exist to verify what mocks +cannot: that kernel bridges actually enslave anchors, that tc impairments +actually shape traffic, that a datapath actually forwards. Run them with:: + + pytest -m e2e tests/e2e -s + +The harness targets an **already running server** when ``GNS3_E2E_URL`` is +set (``GNS3_E2E_USER`` / ``GNS3_E2E_PASSWORD`` default to admin/admin), and +otherwise starts an **isolated instance** — its own config dir, controller +DB, project paths and free port, all under a temporary directory that is +removed on exit. Either way every test creates its own project and deletes +it again (``GNS3_E2E_KEEP=1`` keeps the project for inspection). + +Environment knobs: + +* ``GNS3_E2E_URL`` — target server, e.g. ``http://127.0.0.1:3080/v3``; + when unset, an isolated instance is started for each test. +* ``GNS3_E2E_USER`` / ``GNS3_E2E_PASSWORD`` — credentials for the target. +* ``GNS3_E2E_IMAGES`` — images directory the isolated instance uses + (default ``~/GNS3/images``); also where the dynamips test finds its image. +* ``GNS3_E2E_KEEP=1`` — keep the project (and the isolated instance) when a + test fails, and print their ids, for inspection. +""" + +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +import pytest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# tc lives in /usr/sbin on most distributions; subprocesses do not always +# inherit a PATH that has it. +TC = shutil.which("tc", path=os.environ.get("PATH", "") + ":/usr/sbin:/sbin") + +# IOS prompt as the last thing on the stream: any hostname/mode shape. +PROMPT_RE = re.compile(r"[A-Za-z0-9\-_.()/]+[>#]\s*$") + +# Test-only default credentials (every real deployment overrides them via +# GNS3_E2E_USER / GNS3_E2E_PASSWORD); a freshly started isolated instance +# seeds exactly these. +DEFAULT_USER = "admin" +DEFAULT_PASSWORD = "admin" + + +def keep_on_failure(): + return os.environ.get("GNS3_E2E_KEEP") == "1" + + +# --------------------------------------------------------------------------- +# REST client +# --------------------------------------------------------------------------- + + +class Compute: + """REST client for one server (controller API), token-authenticated.""" + + def __init__(self, url, username=DEFAULT_USER, password=DEFAULT_PASSWORD): + self.url = url.rstrip("/") + self.username = username + self.password = password + self.token = None + + def call(self, method, path, body=None, timeout=60): + # S310: the base URL comes from the operator (GNS3_E2E_URL or the + # isolated instance this harness just started), not untrusted input + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(self.url + path, data=data, method=method.upper()) # noqa: S310 + req.add_header("Content-Type", "application/json") + if self.token: + req.add_header("Authorization", f"Bearer {self.token}") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 + payload = resp.read() + return json.loads(payload) if payload else None + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace") + raise AssertionError(f"{method} {path} -> {e.code}: {detail}") from None + + def login(self): + body = self.call("POST", "/access/users/authenticate", {"username": self.username, "password": self.password}) + self.token = body["access_token"] + return self + + # -- common controller operations ------------------------------------ + + def create_project(self, name="e2e"): + return self.call("POST", "/projects", {"name": f"{name}-{int(time.time())}"}) + + def create_dynamips_router(self, project_id, name, image, slots, platform="c7200", ram=512): + properties = {"platform": platform, "image": image, "ram": ram} + properties.update(slots) + return self.call( + "POST", + f"/projects/{project_id}/nodes", + {"compute_id": "local", "name": name, "node_type": "dynamips", "properties": properties}, + ) + + def create_link(self, project_id, a, b): + return self.call( + "POST", + f"/projects/{project_id}/links", + { + "nodes": [ + {"node_id": a[0], "adapter_number": a[1], "port_number": a[2]}, + {"node_id": b[0], "adapter_number": b[1], "port_number": b[2]}, + ] + }, + ) + + def delete_project(self, project_id): + self.call("DELETE", f"/projects/{project_id}") + + def node(self, project_id, node_id): + return self.call("GET", f"/projects/{project_id}/nodes/{node_id}") + + def capabilities(self): + return self.call("GET", "/computes/local").get("capabilities") or {} + + def dynamips_images(self): + return [image["filename"] for image in self.call("GET", "/computes/local/dynamips/images")] + + +# --------------------------------------------------------------------------- +# Server lifecycle +# --------------------------------------------------------------------------- + +ISOLATED_CONFIG = """\ +[Controller] +jwt_secret_key = 00000000000000000000000000000000000000000000000000000000000000e2 +default_admin_username = admin +default_admin_password = admin + +[Server] +host = 127.0.0.1 +port = {port} +ubridge_path = {ubridge} +enable_kernel_datapath = {kernel} +report_errors = False +projects_path = {tmp}/projects +configs_path = {tmp}/configs +images_path = {images} +appliances_path = {tmp}/appliances +symbols_path = {tmp}/symbols + +[Dynamips] +dynamips_path = {dynamips} +images_path = {images} +allocate_aux_console_ports = False +mmap_support = False +sparse_memory_support = False +ghost_ios_support = False +""" + + +def _free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class Server: + """A server under test: an external one, or a freshly started isolated + instance (own config dir — the controller DB and every path follow it — + under a temporary directory removed on stop).""" + + def __init__(self, compute, process=None, tmpdir=None): + self.compute = compute + self.process = process + self.tmpdir = tmpdir + + @property + def isolated(self): + return self.process is not None + + def log_tail(self, lines=40): + if not self.tmpdir: + return "" + path = os.path.join(self.tmpdir, "server.log") + if not os.path.exists(path): + return "" + with open(path) as f: + return "".join(f.readlines()[-lines:]) + + def stop(self): + if self.process and self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=15) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=10) + if self.tmpdir: + shutil.rmtree(self.tmpdir, ignore_errors=True) + + +def _start_isolated(kernel): + ubridge = shutil.which("ubridge") + dynamips = shutil.which("dynamips") + if not ubridge: + pytest.skip("ubridge is not in PATH (needed to start an isolated instance)") + if not dynamips: + pytest.skip("dynamips is not in PATH (needed to start an isolated instance)") + images = os.environ.get("GNS3_E2E_IMAGES", os.path.expanduser("~/GNS3/images")) + if not os.path.isdir(images): + pytest.skip(f"images directory not found: {images} (set GNS3_E2E_IMAGES)") + + tmpdir = tempfile.mkdtemp(prefix="gns3-e2e-") + os.makedirs(os.path.join(tmpdir, "projects"), exist_ok=True) + port = _free_port() + config = ISOLATED_CONFIG.format( + port=port, ubridge=ubridge, dynamips=dynamips, images=images, tmp=tmpdir, kernel=kernel + ) + config_path = os.path.join(tmpdir, "gns3_server.conf") + with open(config_path, "w") as f: + f.write(config) + + log = open(os.path.join(tmpdir, "server.log"), "w") + # S603: fixed argv, this interpreter, no shell involvement + process = subprocess.Popen( # noqa: S603 + [sys.executable, "-m", "gns3server", "--config", config_path], + cwd=REPO_ROOT, + stdout=log, + stderr=subprocess.STDOUT, + ) + process.log = log # keep the file object alive with the process handle + compute = Compute(f"http://127.0.0.1:{port}/v3") + + deadline = time.time() + 60 + while time.time() < deadline: + try: + # S310: the port is ours, the host loopback + with urllib.request.urlopen(f"http://127.0.0.1:{port}/v3/version", timeout=2): + break + except Exception: + if process.poll() is not None: + raise AssertionError( + f"isolated server exited early:\n{open(os.path.join(tmpdir, 'server.log')).read()[-2000:]}" + ) + time.sleep(0.3) + else: + raise AssertionError("isolated server did not come up within 60s") + + compute.login() + return Server(compute, process, tmpdir) + + +def live_server(kernel=True): + """The server under test. + + With ``GNS3_E2E_URL`` set, that server is used for kernel mode (it must + already be configured for it); everything else starts an isolated + instance. Relay-mode tests always need an isolated instance — the + datapath choice is a server configuration. + """ + url = os.environ.get("GNS3_E2E_URL") + if url and kernel: + compute = Compute( + url, + os.environ.get("GNS3_E2E_USER", "admin"), + os.environ.get("GNS3_E2E_PASSWORD", "admin"), + ).login() + return Server(compute) + return _start_isolated(kernel) + + +def release(server, project_id, failed=False): + """Delete the test project (unless it should be kept), then stop the + isolated instance if this harness started one.""" + if failed and keep_on_failure(): + print(f"\n!! kept project {project_id} on {server.compute.url} for inspection") + else: + try: + server.compute.delete_project(project_id) + except Exception as e: # keep the original failure visible + print(f"!! could not delete project {project_id}: {e}") + if server.isolated and not (failed and keep_on_failure()): + server.stop() + + +# --------------------------------------------------------------------------- +# IOS console (dynamips telnet console) +# --------------------------------------------------------------------------- + + +class IOSConsole: + """A deliberately dumb IOS console driver. + + ``run`` sends one line and returns *everything* received until the prompt + is back and the stream has been quiet — no split between "consumed" and + "remaining" buffers, so a caller can always parse the whole exchange. + Pacing matters: the IOS console drops characters typed while it is busy + (a long boot, the tail of a previous command), so every line goes out + after the previous exchange has settled. + """ + + def __init__(self, host, port): + self.sock = socket.create_connection((host, port), timeout=10) + self.sock.settimeout(0.3) + self.buf = "" + + def _read(self): + """Drain whatever arrived; True when at least one byte was read.""" + got = False + while True: + try: + data = self.sock.recv(65536) + except TimeoutError: + return got + if not data: + return got + self.buf += data.decode(errors="replace") + got = True + + def run(self, line, timeout=30, quiet=0.8): + """Send one line; return the full exchange once the prompt is back + and nothing new arrived for *quiet* seconds.""" + self.buf = "" + self.sock.sendall((line + "\r").encode()) + deadline = time.time() + timeout + last_growth = time.time() + while time.time() < deadline: + if self._read(): + last_growth = time.time() + if PROMPT_RE.search(self.buf) and time.time() - last_growth > quiet: + break + time.sleep(0.05) + out, self.buf = self.buf, "" + return out + + def boot_wait(self, timeout=360): + """Wait until the IOS prompt is ready, answering the initial config + dialog if it appears. The prompt only counts once the stream around + it goes quiet — a boot banner still printing is not a prompt.""" + answered = False + deadline = time.time() + timeout + self.buf = "" + last_growth = time.time() + while time.time() < deadline: + if self._read(): + last_growth = time.time() + if not answered and "initial configuration dialog" in self.buf: + self.sock.sendall(b"no\r") + answered = True + last_growth = time.time() + prompt_seen = ("Press RETURN" in self.buf or answered) and PROMPT_RE.search(self.buf) + if prompt_seen and time.time() - last_growth > 1.2: + self.buf = "" + return + if not prompt_seen and time.time() - last_growth > 1.5: + # nudge the "Press RETURN" prompt along; never nudge an + # already-visible prompt (the echo would restart the quiet + # timer forever) + self.sock.sendall(b"\r") + time.sleep(0.1) + raise AssertionError(f"IOS console did not reach a prompt; tail: {self.buf[-300:]!r}") + + def close(self): + self.sock.close() + + +def dynamips_console(server, project_id, node_id): + node = server.compute.node(project_id, node_id) + host = node["console_host"] if node["console_host"] not in ("0.0.0.0", "::") else "127.0.0.1" + return IOSConsole(host, node["console"]) + + +def configure_ios(console, hostname, eth_ip=None, serial_ip=None, clock=False, eth_if="f1/0", serial_if="s2/0"): + """Baseline config on a freshly booted router: hostname, no console spam, + optional interface addresses.""" + console.run("enable") + console.run("conf t") + console.run(f"hostname {hostname}") + console.run("no ip domain-lookup") + # global form: console logging off in one step, no line sub-mode to lose + # characters in + console.run("no logging console") + if eth_ip: + console.run(f"interface {eth_if}") + console.run(f"ip address {eth_ip} 255.255.255.0") + console.run("no shutdown") + console.run("exit") + if serial_ip: + console.run(f"interface {serial_if}") + console.run(f"ip address {serial_ip} 255.255.255.0") + if clock: + console.run("clock rate 64000") + console.run("no shutdown") + console.run("exit") + console.run("end") + + +def ping(console, target, repeat=3, timeout=1): + """Run an IOS ping; returns {success, avg, raw} (success in percent).""" + out = console.run(f"ping {target} repeat {repeat} timeout {timeout}", timeout=repeat * timeout + 20) + success = re.search(r"Success rate is (\d+) percent", out) + rtt = re.search(r"round-trip min/avg/max = (\d+)/(\d+)/(\d+)", out) + return {"success": int(success.group(1)) if success else 0, "avg": int(rtt.group(2)) if rtt else None, "raw": out} + + +def wait_ping(console, target, attempts=4, repeat=3, timeout=1): + """ping until it fully succeeds (the first packets of a fresh adjacency + are legitimately lost while the line protocol settles or ARP resolves); + returns the last result either way.""" + result = None + for _ in range(attempts): + result = ping(console, target, repeat=repeat, timeout=timeout) + if result["success"] == 100: + return result + time.sleep(1.5) + return result + + +# --------------------------------------------------------------------------- +# Host-side inspection (same machine as the compute) +# --------------------------------------------------------------------------- + + +def sysfs(path): + with open(path) as f: + return f.read().strip() + + +def tap_exists(name): + return os.path.exists(f"/sys/class/net/{name}") + + +def tap_up(name): + """The tap's administrative state (IFF_UP): what the kernel datapath + drives for carrier on attach/detach/suspend.""" + return bool(int(sysfs(f"/sys/class/net/{name}/flags"), 16) & 1) + + +def bridge_members(bridge): + """Interfaces enslaved to a kernel bridge, or None when the bridge does + not exist.""" + if not os.path.exists(f"/sys/class/net/{bridge}/brif"): + return None + return sorted(os.listdir(f"/sys/class/net/{bridge}/brif")) + + +def link_bridge_name(link_id): + """The per-link kernel bridge name the controller derives from a link + id (see UDPLink._prepare).""" + return "gns3" + link_id.replace("-", "")[:11] + + +def anchor_name(node_id, adapter, port): + """The dynamips anchor TAP name for a slot/port (see Router._tap_name).""" + return "gd" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" + + +def qdiscs(dev): + """The tc qdisc listing of a host interface ('' when tc is missing).""" + if not TC: + return "" + # S603: resolved binary, fixed argv, no shell + result = subprocess.run( # noqa: S603 + [TC, "qdisc", "show", "dev", dev], capture_output=True, text=True, check=False + ) + return result.stdout + + +def wait_until(predicate, timeout=10, interval=0.3): + deadline = time.time() + timeout + while time.time() < deadline: + if predicate(): + return True + time.sleep(interval) + return predicate() + + +def pcap_records(path): + """Count the records of a pcap file and collect the ethertype of the + first few (hex, e.g. '0800') — enough to prove a capture is real.""" + import struct + + ethertypes = [] + with open(path, "rb") as f: + header = f.read(24) + if len(header) < 24: + return 0, ethertypes + endian = "<" if header[:4] in (b"\xd4\xc3\xb2\xa1", b"\x4d\x3c\xb2\xa1") else ">" + count = 0 + while True: + rec = f.read(16) + if len(rec) < 16: + break + _ts, _tus, caplen, _orig = struct.unpack(endian + "IIII", rec) + data = f.read(caplen) + if len(data) < caplen: + break + count += 1 + if len(ethertypes) < 5 and caplen >= 14: + ethertypes.append(data[12:14].hex()) + return count, ethertypes diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py new file mode 100644 index 000000000..deb8305d4 --- /dev/null +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -0,0 +1,253 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Live end-to-end test for the Dynamips kernel datapath (pytest marker +``e2e``): two real c7200 routers on a real IOS image, one Ethernet link on +the kernel path and one serial link on the relay, driven through the +controller REST API and the IOS consoles. Real ICMP proves the datapath: + +* the link reports ``kernel_datapath`` and the per-link kernel bridge on + this host has exactly the two anchor TAPs enslaved (kernel objects, not + server bookkeeping); +* a ``delay 100`` filter shows up as netem on the anchors and the measured + RTT rises by ~2x100 ms; clearing it resets the qdiscs and the RTT; +* suspend drives the anchor administratively down and the pings stop; +* capture writes a pcap full of ICMP frames; +* a serial link between the same routers stays on the relay and still + pings (the relay regression net); +* deleting / re-creating the link, and stopping / starting a router, leave + the wiring consistent; +* deleting the project leaves no anchors or bridges behind. + +``test_dynamips_relay_control`` is the negative control: the same topology +on an isolated relay-configured instance wires nothing into the kernel — no +per-link bridge, nothing enslaved — and still pings, so the kernel objects +above can only come from the kernel datapath. (The anchors themselves exist +on both: they are the adapter's port interfaces, created by node start +whenever the tap module is available, exactly like QEMU's TAPs.) +""" + +import re +import time + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# c7200 slot 0 is the fixed IO slot; the Ethernet PA goes in slot 1 (two +# FE ports) and the serial PA in slot 2 (four ports). +SLOTS = {"slot1": "PA-2FE-TX", "slot2": "PA-4T+"} +ETH = (1, 0) +SERIAL = (2, 0) + + +def _pick_image(server): + images = server.compute.dynamips_images() + image = next((name for name in images if name.startswith("c7200")), None) + if not image: + pytest.skip(f"no c7200 image available on this server (images: {images[:5]})") + return image + + +def _boot(server, project_id, node_id, timeout=360): + console = harness.dynamips_console(server, project_id, node_id) + console.boot_wait(timeout=timeout) + return console + + +def test_dynamips_kernel_datapath(): + server = harness.live_server(kernel=True) + compute = server.compute + if compute.capabilities().get("ubridge_tap") is not True: + pytest.skip("this compute's uBridge cannot create persistent TAPs (ubridge_tap != True)") + image = _pick_image(server) + + project = compute.create_project("dyn-e2e") + pid = project["project_id"] + t1 = t2 = None + try: + r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS) + r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS) + r1_id, r2_id = r1["node_id"], r2["node_id"] + t1 = harness.anchor_name(r1_id, *ETH) + t2 = harness.anchor_name(r2_id, *ETH) + + # The link is created while both routers are stopped: the kernel NIO + # is bound with no anchors in existence yet and attached by node + # start (the deferred-wiring path). + link = compute.create_link(pid, (r1_id, *ETH), (r2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert not harness.tap_exists(t1), "anchors must not exist before start" + + print(".. starting routers and waiting for IOS boots") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{r2_id}/start") + c1 = _boot(server, pid, r1_id) + c2 = _boot(server, pid, r2_id) + + # Kernel objects: anchors exist and the per-link bridge has exactly + # the two of them enslaved. + assert harness.tap_exists(t1) and harness.tap_exists(t2), (t1, t2) + assert harness.bridge_members(bridge) == sorted([t1, t2]), harness.bridge_members(bridge) + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + out = c1.run("show ip int brief") + assert re.search(r"FastEthernet1/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + print(".. both routers configured, pinging over the kernel link") + baseline = harness.wait_ping(c1, "10.1.1.2") + assert baseline["success"] == 100, baseline["raw"] + + # delay 100: netem on both anchors (each impairs one direction), + # one-way ~100 ms => RTT grows by ~200 ms + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(t1) and "netem" in harness.qdiscs(t2), ( + harness.qdiscs(t1), + harness.qdiscs(t2), + ) + delayed = harness.wait_ping(c1, "10.1.1.2") + assert delayed["success"] == 100, delayed["raw"] + assert delayed["avg"] >= 150, (baseline, delayed) + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert "netem" not in harness.qdiscs(t1), harness.qdiscs(t1) + fast = harness.wait_ping(c1, "10.1.1.2") + assert fast["success"] == 100 and fast["avg"] < 50, fast + + # suspend: anchor admin-down, traffic dead; resume restores it + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) + assert not harness.tap_up(t1) + dead = harness.ping(c1, "10.1.1.2", repeat=3) + assert dead["success"] == 0, dead["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) + assert harness.tap_up(t1) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + + # capture: AF_PACKET on the anchor writes a real pcap + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.ping(c1, "10.1.1.2") + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # serial link between the same routers: per-port exclusion keeps it + # on the relay, where the real traffic still crosses + serial = compute.create_link(pid, (r1_id, *SERIAL), (r2_id, *SERIAL)) + assert serial["kernel_datapath"] is False, serial + harness.configure_ios(c1, "R1", serial_ip="10.2.2.1", clock=True) + harness.configure_ios(c2, "R2", serial_ip="10.2.2.2") + relay_ping = harness.wait_ping(c1, "10.2.2.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + compute.call("DELETE", f"/projects/{pid}/links/{serial['link_id']}") + + # delete / re-create the kernel link: anchors survive, the bridge + # goes and comes back + compute.call("DELETE", f"/projects/{pid}/links/{lid}") + assert harness.bridge_members(bridge) is None + assert harness.tap_exists(t1) and harness.tap_exists(t2) + link = compute.create_link(pid, (r1_id, *ETH), (r2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + # stop / start keeps the whole wiring (the Dynamips hypervisor + # outlives ``vm stop``); the restarted router just reboots and is + # reconfigured + print(".. stopping and restarting R1") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/stop") + assert harness.tap_exists(t1) + assert harness.bridge_members(bridge) == sorted([t1, t2]) + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + c1.close() + c1 = _boot(server, pid, r1_id) + assert harness.bridge_members(bridge) == sorted([t1, t2]) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closes the nodes: anchors and bridges go with them + assert harness.wait_until(lambda: not harness.tap_exists(t1) and not harness.tap_exists(t2), timeout=15), ( + t1, + t2, + ) + assert harness.wait_until(lambda: harness.bridge_members(bridge) is None, timeout=10), bridge + + +def test_dynamips_relay_control(): + """Negative control: the same topology with ``enable_kernel_datapath = + false`` (an isolated instance) wires nothing into the kernel — no + per-link bridge, no enslavement — and still pings over the relay. + + Anchors DO exist here (a Dynamips router creates its port TAPs at start + whenever the uBridge tap module is available — they are the adapter's + interfaces, exactly like QEMU's); the server configuration only decides + whether links are attached to them.""" + server = harness.live_server(kernel=False) + compute = server.compute + image = _pick_image(server) + + project = compute.create_project("dyn-e2e-relay") + pid = project["project_id"] + t1 = None + try: + r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS) + r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS) + r1_id, r2_id = r1["node_id"], r2["node_id"] + t1 = harness.anchor_name(r1_id, *ETH) + + link = compute.create_link(pid, (r1_id, *ETH), (r2_id, *ETH)) + lid = link["link_id"] + assert link["kernel_datapath"] is False, link + + print(".. starting routers and waiting for IOS boots (relay)") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{r2_id}/start") + c1 = _boot(server, pid, r1_id) + c2 = _boot(server, pid, r2_id) + + # no per-link kernel bridge on the relay: nothing is enslaved, the + # anchor (if present) is just an idle port interface + assert harness.bridge_members(harness.link_bridge_name(lid)) is None + if harness.tap_exists(t1): + assert not harness.tap_up(t1), "an unattached relay anchor should sit down" + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + relay_ping = harness.wait_ping(c1, "10.1.1.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + assert harness.wait_until(lambda: not harness.tap_exists(t1), timeout=15), t1 diff --git a/tests/pytest.ini b/tests/pytest.ini index 88c111a51..8914391ec 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,2 +1,9 @@ [pytest] asyncio_default_fixture_loop_scope=function + +; Live end-to-end tests (tests/e2e, see its README) need privileges, real +; images and emulator binaries — they never run in a normal test run. Run +; them explicitly with: pytest -m e2e tests/e2e +markers = + e2e: live end-to-end tests (real server, kernel interfaces, emulators) +addopts = -m "not e2e" From 3910a6b58597587df6459e0cabcb5d8f0d2cf99f Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 11:55:33 +0800 Subject: [PATCH 037/101] docs(dynamips): record the kernel datapath and its live verification New features/dynamips-kernel-datapath.md: the anchor model (persistent TAP per Ethernet port, the hypervisor opens it with nio create_tap), port selection via the adapter registry, the anchor lifecycle with QEMU's stop order and the stop-is-not-close semantics, link operations, capture/ markers/filters, the relay fallback, the live e2e results, configuration and the roadmap (Ethernet switch next). The IOU and QEMU roadmaps note that Dynamips has landed. --- docs/features/dynamips-kernel-datapath.md | 155 ++++++++++++++++++++++ docs/features/iou-kernel-datapath.md | 12 +- docs/features/qemu-kernel-datapath.md | 5 +- 3 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 docs/features/dynamips-kernel-datapath.md diff --git a/docs/features/dynamips-kernel-datapath.md b/docs/features/dynamips-kernel-datapath.md new file mode 100644 index 000000000..474af3936 --- /dev/null +++ b/docs/features/dynamips-kernel-datapath.md @@ -0,0 +1,155 @@ + + +# Dynamips kernel datapath + +Dynamips routers emulate every port inside the Dynamips process; today a link +reaches a router through a UDP tunnel (a Dynamips `nio create_udp`, a +loopback port pair, the node's uBridge relay, and the link's UDP NIOs). The +kernel datapath replaces that segment with the same anchor model Docker, +QEMU and IOU use: every **Ethernet slot/port** owns a persistent TAP +(`gd{node_id[:8]}e{slot}p{port}`), created by the server through uBridge's +tap module, and links attach to the anchor — a per-link Linux bridge +enslaves it, tc impairments run on it, capture and markers anchor on it. +The Dynamips **hypervisor** opens the TAP with `nio create_tap` (QEMU's +move: an external process holds the fd), unprivileged because the server +handed the device's ownership to its own user. + +``` + Dynamips hypervisor root namespace + ┌──────────────┐ nio create_tap ┌────────────────────────────────┐ + │ IOS router │◄────────────────►│ gd{id}e{slot}p{port} (TAP) │ + │ (emulated │ holds the fd │ │ per-link kernel bridge│ + │ ports) │ │ ▼ gns3{link_id} │ + └──────────────┘ └────────────────────────────────┘ +``` + +The legacy relay is untouched: a port without a kernel link keeps its +Dynamips UDP NIO and the node's uBridge relay tunnel exactly as before, and +a kernel link binds the same port to its anchor instead — per port, no +node-wide switch. + +## Datapath selection + +`UDPLink._kernel_datapath_eligible` extends the common rules with: + +* the compute must report **`ubridge_tap`** — the anchors are uBridge-created + TAPs the hypervisor merely opens, the same capability QEMU gates on; + there is no Dynamips-specific one; +* **both ports must be Ethernet**. The controller's port matrix decides + per port; serial, ATM and POS ports keep the relay whatever the + capabilities (the adapter registry in + `compute/dynamips/adapters/adapter.py` names the Ethernet models — + `ETHERNET_ADAPTERS` / `ETHERNET_WICS`, WIC-1ENET included, at its + Dynamips port number `16 * (wic_slot + 1)`). + +## Anchor lifecycle + +* **Birth (node start)** — `_prepare_tap_datapath` probes the tap module + (one create/delete cycle) and creates one persistent TAP per Ethernet + slot/port, ownership handed to the server user, born DOWN. A stale + leftover from a previous run is swept first; an anchor that already + exists (restart) is kept — an anchor is never recreated under a live + link. Serial/ATM/POS ports simply have no anchor. +* **Life** — a kernel link opens the anchor in the hypervisor + (`nio create_tap`), binds it to the slot/port and enslaves it into the + per-link bridge; removing the link unbinds the port, deletes the + hypervisor's TAP NIO (releasing the fd) and tears the anchor out of the + bridge. A link bound while the router is **stopped** is stored and wired + by the next start (deferred wiring — the NIO simply waits in the slot + adapter). +* **Death (node close)** — `_stop_ubridge` runs in **QEMU's order**, not + IOU's: here the Dynamips hypervisor holds the anchor fds, so its TAP + NIOs go first (`nio delete` closes the fd), then `tap delete` per anchor, + then the per-link kernel bridges, then uBridge itself. + +**A stop is not a close.** `stop` only halts the emulated router: the +hypervisor process — and with it the TAP fds, the port bindings and the +per-link bridges — survives `vm stop` (the same reason a relay link's +tunnel always survived a stop). Links therefore keep working through a +stop/start with no re-wiring at all; the restart's attach pass skips every +port that still has its TAP NIO. If the hypervisor died mid-flight (crash +or kill), the persistent TAPs remain and the next start re-attaches from +the stored NIOs. + +## Link operations + +| Operation | Kernel-datapath action | +|---|---| +| Create / attach | `nio create_tap` (hypervisor holds the fd) + `vm slot_add_nio_binding` + `brctl create` + `brctl addif` + carrier up + markers + `tc netem set` | +| Delete / detach | marker teardown → `tc reset` → carrier off → `brctl delif`/`brctl delete` → unbind + `nio delete` (release the fd) | +| Update (filters/markers/suspend changed) | reconcile on the anchor only — no re-binding, no re-enslaving | +| Suspend | anchor admin-down (traffic dead both ways) + resume restores | +| Adapter hot-add (OIR) | anchors for the new adapter's Ethernet ports created at once, so a kernel link can attach immediately | + +## Capture, markers, filters + +Capture on a kernel link is `capture start_kernel `; markers ride +the base reconcile with the anchor as the location (`marker add_kernel` — +the shared KernelDatapathMixin machinery, no Dynamips-specific code); +filters are one tc netem qdisc per anchor plus cls_bpf/eBPF drops. A relay +port keeps its Dynamips NIO and the node's uBridge relay bridge +(`DYNAMIPS-*`) with userspace filters, exactly as before. + +## Relay fallback + +A uBridge without the tap module keeps the node relay-only: `_prepare_tap_datapath` +warns, no anchors are created, the warning says the node cannot carry +kernel links, and every link keeps its `nio create_udp` tunnel. The +controller never offers such a node a kernel link (capability unreported). + +## Verified + +Unit level (`tests/compute/dynamips/test_dynamips_kernel_datapath.py`): +anchor topology (Ethernet models and WIC-1ENET port numbering only), the +lifecycle (create/sweep/skip-existing), deferred wiring of NIOs bound +while stopped, attach/update/remove command shapes on both datapaths, the +stop order (hypervisor NIO delete before tap delete before brctl delete), +capture and marker shapes, hot-added adapters, and the manager's +`nio_bridge` construction. Plus the controller-eligibility cases (mixable +with docker/qemu/iou, tap capability required, serial excluded) in +`tests/controller/test_kernel_datapath_link.py`. + +End-to-end (`tests/e2e/test_dynamips_kernel_datapath.py`, pytest marker +`e2e`) on a live server with **two real c7200 routers on a real IOS image**, +driven through the REST API and the IOS consoles with real ICMP: + +* a link created while both routers are stopped selects the kernel + datapath and is wired by node start (deferred wiring); +* the per-link kernel bridge on the host has exactly the two anchor TAPs + enslaved; `show ip int brief` is up/up and ping R1→R2 succeeds; +* `delay 100` ⇒ netem qdisc visible on both anchors and the ping RTT + grows by ~200 ms (one-way 100 per direction, each anchor impairing its + own ingress side); clearing resets the qdiscs and the RTT; +* suspend ⇒ anchor administratively down, ping 0%; resume restores both; +* capture writes a pcap full of ICMP-over-Ethernet records; +* a **serial** link between the same two routers stays on the relay + (per-port exclusion) and pings over it — the relay regression net; +* link delete/re-create: the bridge goes and comes back, the anchors + survive, traffic resumes; +* router stop/start keeps the wiring intact (bridge membership unchanged, + no duplicate NIOs) and traffic resumes after the reboot; +* deleting the project removes every anchor and bridge. + +`test_dynamips_relay_control` is the negative control: the same topology on +an isolated relay-configured instance (`enable_kernel_datapath = false`) +has no per-link bridge and nothing enslaved, and still pings — so the +kernel objects above can only come from the kernel datapath. + +## Configuration + +```ini +[Server] +# Wire eligible links (same compute, both endpoints Ethernet and able to +# anchor) through kernel interfaces instead of the uBridge UDP relay. +enable_kernel_datapath = True +``` + +## Roadmap + +The Ethernet switch is next (its ports are already uBridge-owned TAPs; the +kernel link should attach the peer's anchor straight into the switch's +per-node bridge), then the IOL Docker node. Cross-compute kernel links need +VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 8396f3998..4b9444433 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -169,8 +169,10 @@ enable_kernel_datapath = True ## Roadmap -Dynamips next (hypervisor-created taps, enslavable as they are), then the -Ethernet switch (its anchors are ubridge-owned by design) and the IOL Docker -node (same `bridge add_nio_tap` shape as this one). Cross-compute kernel -links need VXLAN/GENEVE encapsulation — deferred until every node type is -kernelized. +Dynamips has landed (see `dynamips-kernel-datapath.md` — the hypervisor +opens uBridge-created TAPs, the same shape QEMU uses). Next: the Ethernet +switch (its ports are already uBridge-owned TAPs; the kernel link should +attach the peer's anchor straight into the switch's per-node bridge), then +the IOL Docker node (same `bridge add_nio_tap` shape as this one). +Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred +until every node type is kernelized. diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 90e336665..9ee177e77 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -172,4 +172,7 @@ a separate project (it would serve Docker links the same way). Update: **IOU has landed** on the same mixin — see `docs/features/iou-kernel-datapath.md` (Ethernet bays anchor on persistent -TAPs bound to the IOL fabric; serial links stay relay). +TAPs bound to the IOL fabric; serial links stay relay). **Dynamips has +landed** too — see `docs/features/dynamips-kernel-datapath.md` (the +hypervisor opens uBridge-created TAPs with `nio create_tap`: the same +external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay). From c343ce0b8fbc7e009221a171526e1d174e00b320 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 22:41:37 +0800 Subject: [PATCH 038/101] feat(esw): wire peer anchors straight into the switch bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Ethernet switch's kernel fast path: instead of a link crossing the switch's per-port uBridge relay, the switch absorbs the peer's anchor into its own kernel bridge. The peer's interface (Docker veth host end, QEMU TAP, IOU port TAP, Dynamips port TAP) becomes the switch port, and frames cross anchor -> switch bridge -> switch ports entirely in the kernel. Wire format: the switch end receives nio_anchor ('join this interface to my own bridge' — the mirror of nio_bridge), the peer end nio_bridge with a null bridge ('your anchor is bridged elsewhere': carrier, markers and impairments only — never bridge membership, and never a bridge delete, on an interface this node does not own). Anchor names come from the shared naming contract (utils.kernel_anchor), a pure function of node type, id, adapter and port — the controller computes the peer's anchor while the node may be stopped, which is what makes links to stopped nodes wireable. Stopped peers defer the join; after any node of a link starts (UDPLink.node_started) the controller re-pushes the switch NIO, and the switch completes — or repairs, when a peer restart replaced its interface under the same name — the join against the kernel's actual membership instead of assuming it. VLAN changes now edit membership in place: only the ports whose mode/VLAN actually changed are touched, each losing exactly the previous mode's entries the new mode does not want (the dot1q admits-all entry is one range operation in both directions), and no port ever leaves the bridge to change its VLANs. This replaces the old 'reset by re-enslaving' shortcut (delif+addif on every wired port, added when the ports were the switch's own TAPs): on absorbed anchors, re-enslaving an interface owned by another node is where live testing watched links die — anchors occasionally lost their bridge membership right after a ports_mapping update (later updates then failed with delif EINVAL), reproducibly with qemu, IOU and Dynamips peers and never with Docker's veths. Eligibility: the switch side needs no capability of its own (its bridge exists by construction); the peer must be able to anchor. Switch-to-switch cascades stay on the relay — a kernel interface belongs to one bridge, and joining two bridges needs a veth pair, deferred. Verified live on a real server: qemu peers through four VLAN cycles with both anchors enslaved throughout; a mixed switch (2 IOU + 2 Dynamips, four kernel ports); Docker peers with real ICMP isolating and recovering (with both anchors still enslaved); VPCS relay ports likewise in place. --- docs/features/dynamips-kernel-datapath.md | 6 +- .../ethernet-switch-kernel-datapath.md | 158 ++++++ docs/features/iou-kernel-datapath.md | 13 +- .../routes/compute/ethernet_switch_nodes.py | 53 +- gns3server/compute/base_manager.py | 9 + .../compute/builtin/nodes/ethernet_switch.py | 291 ++++++++-- .../compute/docker/docker_kernel_datapath.py | 12 +- gns3server/compute/dynamips/nodes/router.py | 12 +- gns3server/compute/iou/iou_vm.py | 10 +- gns3server/compute/kernel_datapath.py | 39 +- gns3server/compute/nios/nio_anchor.py | 84 +++ gns3server/compute/nios/nio_bridge.py | 12 +- gns3server/controller/link.py | 25 +- gns3server/controller/node.py | 5 + gns3server/controller/udp_link.py | 110 +++- gns3server/schemas/__init__.py | 3 +- gns3server/schemas/compute/nios.py | 36 +- gns3server/utils/kernel_anchor.py | 68 +++ .../compute/test_ethernet_switch_nodes.py | 507 +++++++++++++++++- .../dynamips/test_dynamips_kernel_datapath.py | 62 +++ tests/controller/test_kernel_datapath_link.py | 128 +++++ tests/controller/test_node.py | 16 + tests/e2e/harness.py | 16 +- .../test_ethernet_switch_kernel_datapath.py | 290 ++++++++++ tests/utils/test_kernel_anchor.py | 47 ++ 25 files changed, 1902 insertions(+), 110 deletions(-) create mode 100644 docs/features/ethernet-switch-kernel-datapath.md create mode 100644 gns3server/compute/nios/nio_anchor.py create mode 100644 gns3server/utils/kernel_anchor.py create mode 100644 tests/e2e/test_ethernet_switch_kernel_datapath.py create mode 100644 tests/utils/test_kernel_anchor.py diff --git a/docs/features/dynamips-kernel-datapath.md b/docs/features/dynamips-kernel-datapath.md index 474af3936..64e18ad02 100644 --- a/docs/features/dynamips-kernel-datapath.md +++ b/docs/features/dynamips-kernel-datapath.md @@ -149,7 +149,7 @@ enable_kernel_datapath = True ## Roadmap -The Ethernet switch is next (its ports are already uBridge-owned TAPs; the -kernel link should attach the peer's anchor straight into the switch's -per-node bridge), then the IOL Docker node. Cross-compute kernel links need +The Ethernet switch has landed (see `ethernet-switch-kernel-datapath.md` — +it absorbs the peer's anchor into its own kernel bridge; switch-to-switch +cascades and the IOL Docker node are next). Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md new file mode 100644 index 000000000..2c309707a --- /dev/null +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -0,0 +1,158 @@ + + +# Ethernet switch kernel datapath + +The builtin Ethernet switch has been a real Linux kernel bridge since its +brctl migration: one bridge per switch node, every port a persistent TAP +enslaved to it (see `ethernet-switch-ubridge-brctl-migration` design notes +in `gns3server/compute/builtin/nodes/ethernet_switch.py`). What remained on +the userspace path was the *cable*: each port's TAP doubled as a uBridge +relay endpoint, and a link to another node crossed +`peer anchor/UDP tunnel → uBridge relay ↔ port TAP → bridge` — two extra +copies of every frame. + +The kernel datapath removes the cable's userspace leg: the switch absorbs +the **peer's anchor directly into its own bridge**. The peer's veth host +end / TAP becomes the switch port; frames cross +`peer anchor → switch bridge → other switch ports` entirely in the kernel. + +``` + relay (before): peer ──UDP── ubridge[nio_udp↔nio_tap] ──tap── kernel bridge + kernel (now): peer anchor ──────────────────────────► kernel bridge + (the peer's own interface IS the switch port) +``` + +## Datapath selection + +A link with an `ethernet_switch` endpoint rides the fast path when the +**peer** can anchor (same compute; Docker always, QEMU/IOU/Dynamips per +their capabilities) — the switch side needs no anchoring capability of its +own, its bridge and brctl exist by construction. Exclusions, all staying +on the relay: + +* **switch ↔ switch**: a kernel interface can belong to exactly one + bridge, so cascading two switch bridges needs a veth pair between them — + a separate mechanism, deferred; +* anchor-less peers (VPCS, cloud, the Dynamips-hosted hub); +* cross-compute links, non-Ethernet ports: the common rules. + +## The wire format: one NIO names the other end's interface + +``` + controller ──► switch compute: nio_anchor { anchor: "gv00010203e0p0", filters, markers } + controller ──► peer compute: nio_bridge { bridge: null, filters: {}, markers } +``` + +* `nio_anchor` (new, `NIOAnchor`) is the mirror of `nio_bridge`: "enslave + *this* interface into *my own* bridge". The switch joins the anchor and + applies the port's VLAN mode to it (`brctl addif` + the access/dot1q/ + qinq composition), exactly as it programs its own port TAPs. +* `nio_bridge` with `bridge: null` on the peer end means "externally + bridged": the peer keeps everything its node type does around the anchor + (Dynamips opens the TAP in the hypervisor, IOU binds the fabric to it, + QEMU's netdev, Docker's carrier) and applies markers/impairments, but + never touches bridge membership — including on detach, where deleting a + bridge it does not own would take the switch down. + +The anchor's **name** is the shared naming contract +(`gns3server/utils/kernel_anchor.py`): `g{node_id[:8]}e{adapter}p{port}`, +the same function the four node types create their anchors with. It is a +pure function of (node type, node id, adapter, port) — valid whether or +not the peer is running, which is what makes links to stopped nodes +wireable at all. + +**Ownership.** The switch owns everything about the anchor's bridge state +(membership, VLANs). The link's *link state* — capture, markers, tc +impairments, suspend — is attached to the same interface by exactly one +end: filters and capture go to the switch (the controller assigns them +single-sided; both ends would hit one interface twice), suspend rides both +(same admin state either way), markers follow the usual capture-node +routing and work from either end. + +## Stopped peers: deferred join and the re-push + +The peer's anchor exists only while the peer runs. A link created against +a stopped peer cannot be joined immediately, so the switch **defers**: it +remembers the anchor and touches nothing (the link must not fail — this is +the normal case when a project reopens while nodes are still starting). + +The controller completes the wiring: after any node of a link reaches the +started state (`Node.start` → `UDPLink.node_started`), a switch link's NIO +is re-pushed to the switch, which joins the anchor now. The same re-join +runs when a peer restart *replaced* its interface under the same name +(QEMU recreates its TAPs at every start): the membership is checked against +the kernel (`/sys/class/net//brif/`), not assumed. + +Port setting changes (`ports_mapping` update) edit VLAN membership **in +place**: only the ports whose mode/VLAN actually changed are touched, and +each change deletes the previous mode's entries the new mode does not want +and adds the new mode's (the dot1q admits-all entry is one range operation +in both directions). A port never leaves the bridge to change its VLANs — +no traffic gap, no FDB flush, and nothing that detaches an interface this +switch does not own. + +This replaced an earlier "reset by re-enslaving" shortcut (`delif` + +`addif`, so the kernel would hand the port a fresh default membership — +used because a mode change must drop stale entries, e.g. dot1q's all-VIDs +range). On the switch's own relay TAPs it merely cost a brief traffic gap; +on **absorbed anchors** re-enslaving an interface owned by another node +proved to be where links died: live testing showed an anchor occasionally +losing its bridge membership right after a `ports_mapping` update (later +updates then failed `delif` with EINVAL), reproducibly with qemu, IOU and +Dynamips peers, never with Docker's veths. Replaying the exact command +sequence by hand against uBridge (held TAPs, no server involved) did not +reproduce it, and the in-place reconcile removes the operation entirely — +the failure mode no longer exists by construction. + +## Verified + +Unit level (all under `tests/`): the switch's kernel-port lifecycle (join +with VLAN composition, deferred join, re-join, detach with marker/tc/ +carrier teardown, capture on the anchor, batch acceptance for project +open); the in-place VLAN reconcile (every mode transition, untouched ports +emit nothing, relay TAPs and absorbed anchors alike); the shared naming +contract; the mixin's externally-bridged branch (no bridge membership work +in attach/remove, the bridge sweep never deletes a bridge the node does not +own); the controller's eligibility table, wire format (anchor name, +single-sided filters, endpoint-order alignment), `kernel_datapath` +reporting and the node-start re-push. + +Live end-to-end on a real server (`tests/e2e/test_ethernet_switch_kernel_datapath.py` +plus scripted reproduction rounds over the REST API): + +* two qemu peers (the previously 2-out-of-2 failing scenario): deferred + links → start → both anchors joined; four `ports_mapping` cycles + (10/20 ↔ 10/10) with both anchors in the bridge throughout, no failed + update, kernel VLAN state correct at every step; +* a mixed switch (2 × IOU + 2 × Dynamips on one bridge, four kernel + ports): four anchors stable across updates that changed all ports and + across updates that changed only two; +* Docker peers with real ICMP: isolating VLANs cuts the ping with **both + anchors still enslaved** (the filter does the cutting, not a broken + link); restoring the VLAN restores the ping; +* VPCS peers (the relay path, same switch-side mechanics as cloud): the + relay TAPs likewise stay enslaved across VLAN changes, with a real ping + proving isolation and recovery. + +The host-side checks throughout are the kernel's own view: bridge +membership under `/sys/class/net//brif/`, the `master` link +attribute, `bridge vlan show`, and live pings where the peer allows it. + +## Configuration + +Same switch as every node type: + +```ini +[Server] +enable_kernel_datapath = True +``` + +## Roadmap + +Switch-to-switch links (a veth pair joining the two kernel bridges, with +each side's port mode applied at its own end) and the IOL Docker node +(`bridge add_nio_tap`); cross-compute kernel links need VXLAN/GENEVE +encapsulation, deferred until every node type is kernelized. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 4b9444433..17be06e3c 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -170,9 +170,10 @@ enable_kernel_datapath = True ## Roadmap Dynamips has landed (see `dynamips-kernel-datapath.md` — the hypervisor -opens uBridge-created TAPs, the same shape QEMU uses). Next: the Ethernet -switch (its ports are already uBridge-owned TAPs; the kernel link should -attach the peer's anchor straight into the switch's per-node bridge), then -the IOL Docker node (same `bridge add_nio_tap` shape as this one). -Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred -until every node type is kernelized. +opens uBridge-created TAPs, the same shape QEMU uses), and so has the +Ethernet switch (see `ethernet-switch-kernel-datapath.md` — it absorbs the +peer's anchor into its own kernel bridge). What remains: switch-to-switch +cascades (a veth pair between the two kernel bridges) and the IOL Docker +node (same `bridge add_nio_tap` shape as this one). Cross-compute kernel +links need VXLAN/GENEVE encapsulation — deferred until every node type is +kernelized. diff --git a/gns3server/api/routes/compute/ethernet_switch_nodes.py b/gns3server/api/routes/compute/ethernet_switch_nodes.py index e6e13e94d..47b8b8604 100644 --- a/gns3server/api/routes/compute/ethernet_switch_nodes.py +++ b/gns3server/api/routes/compute/ethernet_switch_nodes.py @@ -23,7 +23,7 @@ through uBridge's ``brctl`` module (see """ import os -from typing import Any +from typing import Any, Union from uuid import UUID from fastapi import APIRouter, Body, Depends, HTTPException, Path, status @@ -103,16 +103,20 @@ async def update_ethernet_switch( Update an Ethernet switch. """ - data = jsonable_encoder(node_data, exclude_unset=True) - if "name" in data and node.name != data["name"]: - node.name = data["name"] - if "usage" in data: - node.usage = data["usage"] - if "ports_mapping" in data: - node.ports_mapping = data["ports_mapping"] - await node.update_port_settings() - if "console_type" in data: - node.console_type = data["console_type"] + node_data = jsonable_encoder(node_data, exclude_unset=True) + if "name" in node_data and node.name != node_data["name"]: + node.name = node_data["name"] + if "usage" in node_data: + node.usage = node_data["usage"] + if "ports_mapping" in node_data: + # capture the mapping before the setter replaces it: the VLAN + # reconcile diffs old against new to touch only what changed, in + # place (the setter itself keeps the port-count guard) + previous_mapping = [dict(port) for port in node.ports_mapping] + node.ports_mapping = node_data["ports_mapping"] + await node.update_port_settings(previous_mapping) + if "console_type" in node_data: + node.console_type = node_data["console_type"] node.updated() return node.asdict() @@ -174,15 +178,20 @@ def reload_ethernet_switch(node: EthernetSwitch = Depends(dep_node)) -> None: @router.post( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.AnchorNIO], ) async def create_ethernet_switch_nio( *, adapter_number: int = Path(..., ge=0, le=0), port_number: int, - nio_data: schemas.UDPNIO, + nio_data: Union[schemas.UDPNIO, schemas.AnchorNIO], node: EthernetSwitch = Depends(dep_node), -) -> schemas.UDPNIO: +) -> Union[schemas.UDPNIO, schemas.AnchorNIO]: + """ + Add a NIO (Network Input/Output) to the node: a UDP NIO wires the relay + datapath, an anchor NIO absorbs the peer's kernel anchor into this + switch's kernel bridge. The adapter number on the switch is always 0. + """ nio = Builtin.instance().create_nio(jsonable_encoder(nio_data, exclude_unset=True)) await node.add_nio(nio, port_number) @@ -192,19 +201,21 @@ async def create_ethernet_switch_nio( @router.put( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.AnchorNIO], ) async def update_ethernet_switch_nio( *, adapter_number: int = Path(..., ge=0, le=0), port_number: int, - nio_data: schemas.UDPNIO, + nio_data: Union[schemas.UDPNIO, schemas.AnchorNIO], node: EthernetSwitch = Depends(dep_node), -) -> schemas.UDPNIO: +) -> Union[schemas.UDPNIO, schemas.AnchorNIO]: """ Update a NIO (Network Input/Output) on the node: re-apply the packet - filters and traffic-insight markers carried by the NIO onto the port's - uBridge relay. The adapter number on the switch is always 0. + filters, traffic-insight markers and suspend state carried by the NIO — + on the port's uBridge relay for a relay port, on the absorbed anchor + (tc + kernel markers + carrier) for a kernel port. The adapter number + on the switch is always 0. """ nio = node.get_nio(port_number) @@ -212,6 +223,10 @@ async def update_ethernet_switch_nio( if nio_data.filters: nio.filters = nio_data.filters nio.markers = nio_data.markers or {} + # Suspend is what the compute turns into an admin-down anchor on a + # kernel link (native carrier), so it must reach the NIO like it does + # on the other node routes. + nio.suspend = getattr(nio_data, "suspend", None) or False await node.update_nio(port_number, nio) return nio.asdict() diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index d63817152..35c8be784 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -41,6 +41,7 @@ from ..utils import force_unix_path from ..utils.images import default_images_directory, images_directories, list_images, md5sum, remove_checksum from .base_node import BaseNode from .error import ImageMissingError, NodeError +from .nios.nio_anchor import NIOAnchor from .nios.nio_bridge import NIOBridge from .nios.nio_ethernet import NIOEthernet from .nios.nio_tap import NIOTAP @@ -380,6 +381,14 @@ class BaseManager: # uBridge's AF_PACKET marker module (marker add_kernel). nio.markers = nio_settings.get("markers", {}) nio.suspend = nio_settings.get("suspend", False) + elif nio_settings["type"] == "nio_anchor": + # The node-owned-bridge mirror of nio_bridge: the Ethernet + # switch joins the named foreign anchor to its own kernel + # bridge (AnchorNIO/NIOAnchor). + nio = NIOAnchor(nio_settings["anchor"]) + nio.filters = nio_settings.get("filters", {}) + nio.markers = nio_settings.get("markers", {}) + nio.suspend = nio_settings.get("suspend", False) elif nio_settings["type"] == "nio_tap": tap_device = nio_settings["tap_device"] # if not is_interface_up(tap_device): diff --git a/gns3server/compute/builtin/nodes/ethernet_switch.py b/gns3server/compute/builtin/nodes/ethernet_switch.py index 929325976..63f26153f 100644 --- a/gns3server/compute/builtin/nodes/ethernet_switch.py +++ b/gns3server/compute/builtin/nodes/ethernet_switch.py @@ -37,12 +37,16 @@ be the switch). ESW ``access``/``dot1q``/``qinq`` port modes are composed from the ``brctl`` VLAN primitives here -- see ``_apply_port_vlan``. """ +import contextlib import logging +import os from gns3server.compute.ubridge.ubridge_error import UbridgeError from ...base_node import BaseNode from ...error import NodeError +from ...kernel_datapath import KernelDatapathMixin +from ...nios.nio_anchor import NIOAnchor from ...nios.nio_udp import NIOUDP log = logging.getLogger(__name__) @@ -55,7 +59,7 @@ _SUPPORTED_VLAN_ETHERTYPE = {"0x8100", "0x88a8"} _QINQ_ETHERTYPE = "0x88a8" -class EthernetSwitch(BaseNode): +class EthernetSwitch(KernelDatapathMixin, BaseNode): """ Ethernet switch. @@ -75,9 +79,11 @@ class EthernetSwitch(BaseNode): self._nios = {} self._tap_by_port = {} # port_number -> kernel TAP enslaved to the bridge + self._kernel_ports = {} # port_number -> foreign anchor absorbed into the bridge self._bridge_name = None # kernel bridge interface name (allocated on start) self._bridge_created = False self._bridge_proto_set = False # whether ``brctl setvlanproto`` has been applied + self._ubridge_tc_caps = None # probed once per uBridge process (mixin) # Idempotency flag for start(). Decoupled from ``status`` so the node can # report "started" (always-on, like the ESW) while ``duplicate_node`` still # sees status "stopped" and refuses only genuinely running stateful nodes. @@ -195,11 +201,16 @@ class EthernetSwitch(BaseNode): if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): await self._stop_ubridge() await self._start_ubridge(self._ubridge_require_privileged_access) + # A fresh uBridge process: the tc capabilities must be probed again. + self._ubridge_tc_caps = None await self._ensure_bridge() for port_number in self._nios: if self._nios[port_number]: try: - await self._add_ubridge_connection(self._nios[port_number], port_number) + if isinstance(self._nios[port_number], NIOAnchor): + await self._add_kernel_port(self._nios[port_number], port_number) + else: + await self._add_ubridge_connection(self._nios[port_number], port_number) except (UbridgeError, NodeError) as e: self._started = False raise e @@ -275,11 +286,24 @@ class EthernetSwitch(BaseNode): if nio and isinstance(nio, NIOUDP): self.manager.port_manager.release_udp_port(nio.lport, self._project) self._nios.clear() - self._tap_by_port.clear() if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running() and self._bridge_created: + # Project close runs every node's close concurrently: a peer's + # anchor may still be enslaved at this instant (its node's close + # is in flight), which would make the bridge delete fail EBUSY + # and leak an empty bridge. Detach every member we know first — + # a peer anchor that is already gone answers ENOENT (suppressed; + # the kernel removed it with the interface), and our own TAPs go + # with the relay teardown — so the delete always sees an empty + # bridge. The foreign anchors themselves are not ours to delete. + for anchor in self._kernel_ports.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{anchor}"') + for tap in self._tap_by_port.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{tap}"') try: - # Deleting the bridge releases its enslaved TAPs; uBridge destroys + # Deleting the bridge releases any enslaved TAPs; uBridge destroys # them when it stops below. await self._ubridge_send(f'brctl delete "{self._bridge_name}"') except UbridgeError as e: @@ -287,6 +311,8 @@ class EthernetSwitch(BaseNode): self._bridge_created = False self._bridge_proto_set = False self._bridge_name = None + self._tap_by_port.clear() + self._kernel_ports.clear() self._started = False await self._stop_ubridge() @@ -301,19 +327,27 @@ class EthernetSwitch(BaseNode): """ Adds a NIO as a new port on this switch. + Two kinds are accepted: a UDP NIO (relay datapath — the per-port + uBridge relay ends in a bridge TAP) and the kernel datapath's + NIOAnchor (the peer's anchor is absorbed straight into this switch's + kernel bridge). + :param nio: NIO instance to add :param port_number: port to allocate for the NIO """ if port_number in self._nios: raise NodeError(f"Port {port_number} isn't free") - if not isinstance(nio, NIOUDP): - raise NodeError("Ethernet switch ports only support UDP NIOs") + if not isinstance(nio, (NIOUDP, NIOAnchor)): + raise NodeError("Ethernet switch ports only support UDP NIOs and anchor NIOs") log.debug(f'Ethernet switch "{self._name}" [{self._id}]: NIO {nio} bound to port {port_number}') try: await self.start() - await self._add_ubridge_connection(nio, port_number) + if isinstance(nio, NIOAnchor): + await self._add_kernel_port(nio, port_number) + else: + await self._add_ubridge_connection(nio, port_number) self._nios[port_number] = nio except (NodeError, UbridgeError) as e: log.error(f'Cannot add NIO on Ethernet switch "{self._name}": {e}') @@ -351,6 +385,95 @@ class EthernetSwitch(BaseNode): await self._ubridge_send(f"bridge start {ubridge_bridge}") self._tap_by_port[port_number] = tap + # ------------------------------------------------------------------ # + # kernel datapath: absorb the peer's anchor (fast path) + # ------------------------------------------------------------------ # + + def _kernel_host_ifc(self, adapter_number, port_number=0): + """ + The foreign anchor absorbed on a port, or None for a relay port. The + switch has no adapters, so *adapter_number* is always 0. + """ + + return self._kernel_ports.get(port_number) + + def _kernel_anchors(self): + """ + Every absorbed anchor — the interfaces this switch bridges and on + which kernel markers may attach (the mixin's marker polymorphism). + """ + + return set(self._kernel_ports.values()) + + def _kernel_error(self, message): + return NodeError(message) + + async def _add_kernel_port(self, nio, port_number): + """ + Absorbs the peer's anchor for one port: joins the anchor straight to + this switch's kernel bridge and applies the port's VLAN mode — no + relay, no per-port TAP, the peer anchor *is* the port. + + The anchor only exists on the host while the peer is running (its + compute creates it at node start), so a link created against a + stopped peer defers the join: the port remembers the anchor and + ``update_nio`` completes the wiring when the controller re-pushes + the NIO after the peer starts (UdpLink.node_started). A peer + restart can also replace the interface under the same name, so the + membership is re-checked rather than assumed. + + The link state the anchored end carries (capture, markers, tc + impairments) is applied by the shared mixin flow: NIOAnchor reports + no per-link bridge, so only the link state is applied here, while + the bridge membership above is this switch's own. + """ + + port_settings = self._port_settings(port_number) + if port_settings is None: + raise NodeError(f"Port {port_number} doesn't exist on Ethernet switch '{self.name}'") + + anchor = nio.anchor + # Register before _kernel_attach: the marker reconcile recognises + # kernel anchors through _kernel_anchors(). + self._kernel_ports[port_number] = anchor + if not os.path.exists(f"/sys/class/net/{anchor}"): + log.info( + 'Ethernet switch "{name}" [{id}]: anchor {anchor} for port {port} does not exist yet ' + "(peer not started); join deferred".format( + name=self._name, id=self._id, anchor=anchor, port=port_number + ) + ) + return + try: + await self._ubridge_send(f'brctl addif "{self._bridge_name}" "{anchor}"') + await self._apply_port_vlan(port_settings, anchor) + await self._kernel_attach(anchor, nio) + await self._set_adapter_carrier(0, not nio.suspend, port_number) + except (NodeError, UbridgeError): + self._kernel_ports.pop(port_number, None) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{anchor}"') + raise + + async def _remove_kernel_port(self, port_number): + """ + Releases an absorbed anchor: detaches the link state (markers, + impairments, carrier) and takes the anchor out of the switch bridge. + The anchor itself belongs to the peer and survives. + """ + + anchor = self._kernel_ports.pop(port_number, None) + if anchor is None: + return + await self._remove_kernel_markers(anchor) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tc reset "{anchor}"') + if self._bridge_created: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{anchor}"') + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'link set "{anchor}" down') + async def _delete_ubridge_connection(self, port_number): """ Tears down one port's wiring: release the TAP from the bridge and delete @@ -388,7 +511,10 @@ class EthernetSwitch(BaseNode): log.debug(f'Ethernet switch "{self._name}" [{self._id}]: NIO {nio} removed from port {port_number}') del self._nios[port_number] if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - await self._delete_ubridge_connection(port_number) + if isinstance(nio, NIOAnchor): + await self._remove_kernel_port(port_number) + else: + await self._delete_ubridge_connection(port_number) return nio def get_nio(self, port_number): @@ -405,32 +531,85 @@ class EthernetSwitch(BaseNode): async def update_nio(self, port_number, nio): """ - Re-applies uBridge filters/markers for a port (called when a link is updated). + Re-applies filters/markers (and carrier) for a port when a link is + updated: on the per-port uBridge relay for a relay port, on the + absorbed anchor (tc + kernel markers) for a kernel port. """ + if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running()): + return + if port_number in self._kernel_ports: + anchor = self._kernel_ports[port_number] + if not os.path.exists(f"/sys/class/net/{self._bridge_name}/brif/{anchor}"): + # Not joined (a deferred link whose peer has just started, or + # a peer restart that replaced its anchor under the same + # name): complete the whole wiring now. + await self._add_kernel_port(nio, port_number) + return + # Kernel port: reconcile on the anchor — markers, tc impairments + # and the carrier the suspend flag drives. No re-joining, no + # VLAN re-programming (the port mode did not change). + await self._kernel_update(anchor, nio) + await self._set_adapter_carrier(0, not nio.suspend, port_number) + return ubridge_bridge = self._ubridge_bridge_name(port_number) - if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - await self._ubridge_apply_filters(ubridge_bridge, nio.filters) - await self._ubridge_apply_markers(ubridge_bridge, nio) + await self._ubridge_apply_filters(ubridge_bridge, nio.filters) + await self._ubridge_apply_markers(ubridge_bridge, nio) # ------------------------------------------------------------------ # # VLAN mode translation # ------------------------------------------------------------------ # - async def _reset_port_vlan(self, tap): + async def _reconfigure_port_vlan(self, iface, old_settings, new_settings): """ - Resets a port's VLAN membership to the kernel default (PVID 1, untagged) - by re-enslaving it. Used before re-applying a changed mode so stale VIDs - from the previous mode do not leak. + Moves one port's VLAN membership from its previous mode to the new one + **in place**: the port never leaves the bridge, so there is no traffic + gap, no FDB flush, and nothing that detaches an interface this switch + does not own (an absorbed kernel anchor belongs to the peer's node — + re-enslaving it from here is not this switch's business). + + Each mode's membership, as installed on a clean port by + ``_apply_port_vlan``: access(N) = {N, PVID untagged}; dot1q(V) = + {1..4094, tagged} plus V as PVID untagged (the all-VIDs entry is a + single range entry); qinq(O) = {O, PVID untagged}. The transition + deletes exactly the previous mode's entries the new mode does not + want, then adds the new mode's entries — the dot1q range is added and + deleted as one range operation. """ - await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{tap}"') - await self._ubridge_send(f'brctl addif "{self._bridge_name}" "{tap}"') + old_type, old_vlan = old_settings["type"], int(old_settings["vlan"]) + new_type, new_vlan = new_settings["type"], int(new_settings["vlan"]) + if old_type == new_type and old_vlan == new_vlan: + return + + br = self._bridge_name + commands = [] + + if old_type == "dot1q" and new_type != "dot1q": + # the admits-all-VIDs range, the old native VLAN included in it + commands.append(f'brctl vlan_del "{br}" "{iface}" 1 vid 4094') + elif old_type == "dot1q": + if old_vlan != new_vlan: + commands.append(f'brctl vlan_del "{br}" "{iface}" {old_vlan}') + elif new_type == "dot1q" or old_vlan != new_vlan: + # access/qinq single-VID entry, replaced by the new mode's + commands.append(f'brctl vlan_del "{br}" "{iface}" {old_vlan}') + + if new_type == "dot1q": + if old_type != "dot1q": + commands.append(f'brctl vlan_add "{br}" "{iface}" 1 vid 4094') + commands.append(f'brctl vlan_add "{br}" "{iface}" {new_vlan} pvid untagged') + else: + commands.append(f'brctl vlan_add "{br}" "{iface}" {new_vlan} pvid untagged') + + for command in commands: + await self._ubridge_send(command) async def _apply_port_vlan(self, port_settings, tap): """ - Translates an ESW port mode into ``brctl`` VLAN primitives. The port must - already be enslaved to the bridge and carry the default PVID 1. + Translates an ESW port mode into ``brctl`` VLAN primitives, for a + **newly wired** port. The port must already be enslaved to the bridge + and carry the default PVID 1. - access VLAN N: drop default 1, add N as PVID + egress untagged. - dot1q trunk (native N): drop default 1, admit all VIDs tagged, then mark @@ -438,6 +617,9 @@ class EthernetSwitch(BaseNode): VLAN per trunk port, so the trunk admits all VIDs, like the emulated ESW.) - qinq (outer N): the bridge-level protocol is set separately; the port gets the service VLAN as PVID + untagged so customer frames are S-tagged. + + Mode *changes* on an already-wired port go through + ``_reconfigure_port_vlan`` (in place, no re-enslaving) instead. """ br = self._bridge_name @@ -458,23 +640,41 @@ class EthernetSwitch(BaseNode): else: raise NodeError(f"Unknown port type '{port_type}' on Ethernet switch '{self.name}'") - async def update_port_settings(self): + async def update_port_settings(self, previous_mapping=None): """ - Re-applies port settings (called after ``ports_mapping`` is updated). For - ports already wired, reset then re-apply so a mode/VLAN change fully - replaces the previous VLAN membership. + Reconciles port settings after a ``ports_mapping`` update. Only the + wired ports whose mode/VLAN actually changed are touched, and each + change is applied in place (``_reconfigure_port_vlan``) — no port is + ever detached from the bridge just to change its VLANs, on the + switch's own relay TAPs and on absorbed kernel anchors alike. + + :param previous_mapping: the ports_mapping as it was before this + update (the caller's duty — the setter has already stored the new + one); without it nothing can be diffed and no port is touched. """ await self._apply_bridge_proto_if_needed() if not (self._ubridge_hypervisor and self._ubridge_hypervisor.is_running() and self._bridge_created): return - for port_settings in self._ports_mapping: - port_number = port_settings["port_number"] - tap = self._tap_by_port.get(port_number) - if tap is None: + + previous = {port["port_number"]: port for port in (previous_mapping or [])} + wired = dict(self._tap_by_port) + for port_number, anchor in self._kernel_ports.items(): + wired[port_number] = anchor + for port_number, iface in wired.items(): + new_settings = self._port_settings(port_number) + if new_settings is None: continue - await self._reset_port_vlan(tap) - await self._apply_port_vlan(port_settings, tap) + old_settings = previous.get(port_number) + if old_settings is None: + # a wired port always has previous settings; without them a + # diff is impossible and re-enslaving would be guessing + log.warning( + 'Ethernet switch "{name}" [{id}]: no previous settings for wired port {port}; ' + "VLAN reconfiguration skipped".format(name=self._name, id=self._id, port=port_number) + ) + continue + await self._reconfigure_port_vlan(iface, old_settings, new_settings) # ------------------------------------------------------------------ # # capture @@ -482,7 +682,9 @@ class EthernetSwitch(BaseNode): async def start_capture(self, port_number, output_file, data_link_type="DLT_EN10MB"): """ - Starts a packet capture on a port (uBridge captures on the per-port relay). + Starts a packet capture on a port — on the per-port relay bridge for + a relay port, on the absorbed anchor (AF_PACKET, capture start_kernel) + for a kernel port. :param port_number: allocated port number :param output_file: PCAP destination file for the capture @@ -494,9 +696,17 @@ class EthernetSwitch(BaseNode): raise NodeError(f"Packet capture is already activated on port {port_number}") nio.start_packet_capture(output_file) if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - ubridge_bridge = self._ubridge_bridge_name(port_number) - await self._ubridge_send(f'bridge start_capture {ubridge_bridge} "{output_file}"') - log.debug(f'Ethernet switch "{self.name}" [{self.id}]: starting packet capture on port {port_number}') + anchor = self._kernel_ports.get(port_number) + if anchor is not None: + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + else: + ubridge_bridge = self._ubridge_bridge_name(port_number) + await self._ubridge_send(f'bridge start_capture {ubridge_bridge} "{output_file}"') + log.debug( + 'Ethernet switch "{name}" [{id}]: starting packet capture on port {port}'.format( + name=self.name, id=self.id, port=port_number + ) + ) async def stop_capture(self, port_number): """ @@ -510,6 +720,13 @@ class EthernetSwitch(BaseNode): return nio.stop_packet_capture() if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - ubridge_bridge = self._ubridge_bridge_name(port_number) - await self._ubridge_send(f"bridge stop_capture {ubridge_bridge}") - log.debug(f'Ethernet switch "{self.name}" [{self.id}]: stopping packet capture on port {port_number}') + if port_number in self._kernel_ports: + await self._ubridge_send("capture stop_kernel") + else: + ubridge_bridge = self._ubridge_bridge_name(port_number) + await self._ubridge_send(f"bridge stop_capture {ubridge_bridge}") + log.debug( + 'Ethernet switch "{name}" [{id}]: stopping packet capture on port {port}'.format( + name=self.name, id=self.id, port=port_number + ) + ) diff --git a/gns3server/compute/docker/docker_kernel_datapath.py b/gns3server/compute/docker/docker_kernel_datapath.py index 4138bbce2..f0b215035 100644 --- a/gns3server/compute/docker/docker_kernel_datapath.py +++ b/gns3server/compute/docker/docker_kernel_datapath.py @@ -29,6 +29,7 @@ from gns3server.compute.kernel_datapath import KernelDatapathMixin from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError from .docker_error import DockerError +from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name import logging @@ -48,12 +49,15 @@ class DockerKernelDatapathMixin(KernelDatapathMixin): kernel bridge); the guest end is moved into the container namespace and renamed. The ``gv``/``gc`` prefixes keep these out of the ``gns3`` bridge/TAP name space and serve as search keys for stale-interface - cleanup. 8 hex chars of the node id + adapter/port keep the names - unique and within IFNAMSIZ (15). + cleanup; the host end's name is the shared anchor naming contract + (utils.kernel_anchor), which the controller also uses to name a + peer's anchor for an Ethernet-switch link. """ - suffix = f"{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" - return f"gv{suffix}", f"gc{suffix}" + return ( + kernel_anchor_name("docker", self._id, adapter_number, port_number), + f"gc{anchor_suffix(self._id, adapter_number, port_number)}", + ) # ------------------------------------------------------------------ # Anchor hooks (see KernelDatapathMixin) diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index 6c8afaf5f..4cb6cf964 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -36,6 +36,7 @@ from gns3server.utils.asyncio import monitor_process, wait_run_in_executor from gns3server.utils.file_watcher import FileWatcher from gns3server.utils.hostname import is_ios_hostname_valid from gns3server.utils.images import md5sum +from gns3server.utils.kernel_anchor import kernel_anchor_name from ...base_node import BaseNode from ...error import NodeError @@ -1547,14 +1548,13 @@ class Router(KernelDatapathMixin, BaseNode): def _tap_name(self, slot_number, port_number): """ - Deterministic anchor TAP name for a slot/port. The ``gd`` prefix - keeps it out of the ``gns3`` bridge/TAP name space (and apart from - Docker's ``gv``/``gc``, QEMU's ``gq`` and IOU's ``gi``); 8 hex chars - of the node id plus slot/port keep it unique and within IFNAMSIZ - (15), even for WIC port numbers (16, 32, 48). + Deterministic anchor TAP name for a slot/port (the shared + utils.kernel_anchor naming contract — the controller names a peer's + anchor with the same function when an Ethernet switch absorbs it; + fits IFNAMSIZ even for WIC port numbers 16/32/48). """ - return f"gd{self._id.replace('-', '')[:8]}e{slot_number}p{port_number}" + return kernel_anchor_name("dynamips", self._id, slot_number, port_number) def _kernel_host_ifc(self, slot_number, port_number=0): """ diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index 7c61c7d18..676d46e3a 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -43,6 +43,7 @@ from gns3server.utils.asyncio.ssh_server import AsyncioSSHServer from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer from gns3server.utils.file_watcher import FileWatcher from gns3server.utils.hostname import is_ios_hostname_valid +from gns3server.utils.kernel_anchor import kernel_anchor_name from ..adapters.ethernet_adapter import EthernetAdapter from ..adapters.serial_adapter import SerialAdapter @@ -862,13 +863,12 @@ class IOUVM(KernelDatapathMixin, BaseNode): def _tap_name(self, adapter_number, port_number): """ - Deterministic anchor TAP name for an Ethernet bay/unit. The ``gi`` - prefix keeps it out of the ``gns3`` bridge/TAP name space (and apart - from Docker's ``gv``/``gc`` and QEMU's ``gq``); 8 hex chars of the - node id plus bay/unit keep it unique and within IFNAMSIZ (15). + Deterministic anchor TAP name for an Ethernet bay/unit (the shared + utils.kernel_anchor naming contract — the controller names a peer's + anchor with the same function when an Ethernet switch absorbs it). """ - return f"gi{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" + return kernel_anchor_name("iou", self._id, adapter_number, port_number) def _kernel_host_ifc(self, adapter_number, port_number=0): """ diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index a993ac1fa..3e8665af2 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -146,15 +146,23 @@ class KernelDatapathMixin: Both link endpoints run this concurrently and either may create the bridge first: an existing bridge is success instead of a create race failure. + + A NIO whose ``bridge`` is None means the anchor is bridged + elsewhere — an Ethernet switch absorbed it into the switch's own + bridge — so the membership work is skipped here and only the link + state the anchor carries (capture, markers, impairment filters) is + applied. Exactly one end of such a link owns the anchor's bridge + state; the other end is a passive carrier. """ - try: - await self._ubridge_send(f'brctl create "{nio.bridge}"') - except UbridgeError: - # Raises again if the bridge genuinely does not exist. - await self._ubridge_send(f'brctl show "{nio.bridge}"') - await self._ubridge_send(f'link set "{nio.bridge}" up') - await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"') + if nio.bridge is not None: + try: + await self._ubridge_send(f'brctl create "{nio.bridge}"') + except UbridgeError: + # Raises again if the bridge genuinely does not exist. + await self._ubridge_send(f'brctl show "{nio.bridge}"') + await self._ubridge_send(f'link set "{nio.bridge}" up') + await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"') if nio.capturing: # Restore a capture that was active before a node restart # (mirrors the relay path's start_capture). @@ -235,6 +243,11 @@ class KernelDatapathMixin: this concurrently: deleting a bridge that still has the peer's port enslaved fails with EBUSY, so the last endpoint to remove its port wins the deletion and the loser's failure is expected and suppressed. + + A NIO whose ``bridge`` is None owns no bridge here: the anchor is + bridged by an Ethernet switch, which tears its own membership down + on its side; this end only releases the link state it applied + (markers, impairments, carrier). """ host_ifc = self._kernel_host_ifc(adapter_number, port_number) @@ -248,10 +261,12 @@ class KernelDatapathMixin: await self._ubridge_send(f'tc reset "{host_ifc}"') if self.status == "started": await self._set_adapter_carrier(adapter_number, False, port_number) + if nio.bridge is not None: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') + if nio.bridge is not None: with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"') - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'brctl delete "{nio.bridge}"') + await self._ubridge_send(f'brctl delete "{nio.bridge}"') async def _remove_kernel_bridges(self): """ @@ -269,7 +284,9 @@ class KernelDatapathMixin: return for adapter in self._ethernet_adapters: for nio in adapter.ports.values(): - if isinstance(nio, NIOBridge): + # bridge None = externally bridged (an Ethernet switch owns + # the bridge); never delete a bridge this node does not own. + if isinstance(nio, NIOBridge) and nio.bridge is not None: with contextlib.suppress(UbridgeError): await self._ubridge_send(f'brctl delete "{nio.bridge}"') diff --git a/gns3server/compute/nios/nio_anchor.py b/gns3server/compute/nios/nio_anchor.py new file mode 100644 index 000000000..cecfcc6f8 --- /dev/null +++ b/gns3server/compute/nios/nio_anchor.py @@ -0,0 +1,84 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Interface for anchor-absorbing NIOs (the Ethernet switch's kernel datapath). + +A NIO of this type instructs the node to join a *foreign* host interface — +another node's kernel-datapath anchor — into a bridge the node itself owns, +with the node's port settings applied to it. The Ethernet switch is the +consumer: its per-node kernel bridge absorbs the peer's anchor directly, so +frames cross peer anchor -> switch bridge -> switch ports entirely in the +kernel instead of riding the switch's per-port uBridge relay. + +This is the mirror of NIOBridge: NIOBridge says "enslave *my* anchor into +the bridge named here", NIOAnchor says "enslave *this* interface into my +own bridge". Impairment filters, markers and capture ride the NIO and +attach to the named anchor (tc on it, marker add_kernel / capture +start_kernel on it) — exactly one link end owns them. +""" + +from .nio import NIO + + +class NIOAnchor(NIO): + """ + Anchor-absorbing NIO. + + :param anchor: host interface to join to the node's own bridge + """ + + def __init__(self, anchor): + + super().__init__() + self._anchor = anchor + + @property + def anchor(self): + """ + Returns the interface name to absorb. + + :returns: interface name + """ + + return self._anchor + + @property + def bridge(self): + """ + Always None: this NIO owns no per-link bridge — the absorbed anchor + joins the node's *own* bridge. The None makes the shared + KernelDatapathMixin attach/remove flows skip the per-link bridge + membership work (their external-bridge case), which is exactly what + the switch needs on top of its own addif. + """ + + return None + + def __str__(self): + + return "NIO anchor" + + def asdict(self): + + return { + "type": "nio_anchor", + "anchor": self._anchor, + "suspend": self._suspended, + "filters": self._filters, + "markers": self._markers, + } diff --git a/gns3server/compute/nios/nio_bridge.py b/gns3server/compute/nios/nio_bridge.py index d65e38471..bdb00c9e5 100644 --- a/gns3server/compute/nios/nio_bridge.py +++ b/gns3server/compute/nios/nio_bridge.py @@ -25,17 +25,22 @@ filters ride the NIO and run as one tc netem qdisc per veth host end (the types without a netem equivalent — frequency_drop, bpf — stay relay-only); markers and packet capture are served by uBridge's AF_PACKET modules on the veth host end (marker add_kernel / capture start_kernel). + +``bridge`` may be ``None``: the anchor is then bridged elsewhere — an +Ethernet switch absorbed it into the switch's own kernel bridge — and this +node applies only the link state the anchor carries (carrier, markers, +impairments), never bridge membership. """ from .nio import NIO class NIOBridge(NIO): - """ Kernel-bridge NIO. - :param bridge: name of the per-link Linux kernel bridge + :param bridge: name of the per-link Linux kernel bridge, or None when + the anchor is bridged by an Ethernet switch (see module docstring) """ def __init__(self, bridge): @@ -46,7 +51,8 @@ class NIOBridge(NIO): @property def bridge(self): """ - Returns the kernel bridge name. + Returns the kernel bridge name, or None for an externally bridged + anchor. :returns: bridge name """ diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index b531d91ec..bcca97f89 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -202,13 +202,16 @@ class Link: @property def kernel_datapath(self): """ - Whether this link is wired on the kernel datapath (veth pairs - enslaved into a per-link Linux bridge — no uBridge relay in the - forwarding path). Impairment filters run as tc netem on the veth - host end; markers and capture are served by uBridge's AF_PACKET + Whether this link is wired on the kernel datapath (no uBridge relay + in the forwarding path): either both endpoints enslave their + anchors into a per-link Linux bridge (``nio_bridge``), or an + Ethernet switch absorbs the peer's anchor into its own kernel + bridge (``nio_anchor`` on the switch end, ``nio_bridge`` with no + bridge on the peer end). Impairment filters run as tc netem on the + anchors; markers and capture are served by uBridge's AF_PACKET module. """ - return any(d.get("type") == "nio_bridge" for d in (getattr(self, "_link_data", None) or [])) + return any(d.get("type") in ("nio_bridge", "nio_anchor") for d in (getattr(self, "_link_data", None) or [])) @property def markers(self): @@ -335,9 +338,7 @@ class Link: raise ControllerError( "Packet filter(s) {} only run on a kernel-datapath link (tc netem on the " "veth host end); this link is wired on the uBridge relay — delete and " - "recreate the link to switch it to the kernel datapath".format( - ", ".join(sorted(conflicts)) - ) + "recreate the link to switch it to the kernel datapath".format(", ".join(sorted(conflicts))) ) # An unchanged filters dict is normally a no-op skip, but a PUT that @@ -685,6 +686,14 @@ class Link: """ raise NotImplementedError + async def node_started(self, node): + """ + Called after a node member of the link reached the started state. + The base link has nothing to re-synchronise; UDPLink uses it for + the Ethernet-switch fast path (the switch must re-join a peer anchor + that only comes into existence when the peer starts). + """ + def default_capture_file_name(self): """ :returns: File name for a capture on this link diff --git a/gns3server/controller/node.py b/gns3server/controller/node.py index e96d4b704..3500cb2db 100644 --- a/gns3server/controller/node.py +++ b/gns3server/controller/node.py @@ -936,6 +936,11 @@ class Node: await self.post("/start", data=data, timeout=240) except asyncio.TimeoutError: raise ControllerTimeoutError(f"Timeout when starting {self._name}") + # A started node may have brought kernel anchors into existence that + # a link needs to finish wiring (the Ethernet-switch fast path joins + # the peer's anchor on the switch side — see UDPLink.node_started). + for link in self._links: + await link.node_started(self) async def stop(self): """ diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index abe1a7f78..5b37513d3 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -20,12 +20,14 @@ import asyncio import logging from gns3server.config import Config +from gns3server.utils.kernel_anchor import kernel_anchor_name from gns3server.utils.packet_filter_validation import ( KERNEL_UNSUPPORTED_FILTERS, split_kernel_only_features, validate_bpf_syntax, ) +from .compute import ComputeError from .controller_error import ControllerError, ControllerNotFoundError from .link import _UNSET, Link from .node_types import BUILTIN_NODE_TYPES @@ -111,6 +113,13 @@ class UDPLink(Link): if kernel is None: kernel = self.kernel_datapath if kernel: + # Ethernet switch fast path: the link's tc impairments apply on + # the absorbed anchor, which is a single interface — exactly one + # end may own them, and that end is the switch (it owns the + # anchor's bridge membership and port VLANs). + switch = self._switch_endpoint() + if switch is not None: + return (self._filters, {}) if switch[0] == 0 else ({}, self._filters) return self._filters, self._filters # Relay datapath: the netem-extension filters (rate, reorder, gemodel…) # have no relay equivalent. update_filters rejects them on created @@ -200,8 +209,32 @@ class UDPLink(Link): port = side.get("port") if port is not None and port.link_type != "ethernet": return False + # Ethernet switch fast path: the switch absorbs the peer's anchor + # into its own kernel bridge, so the switch side needs no anchoring + # capability of its own (its bridge and brctl already exist) — only + # the peer must be able to anchor. Two switches never qualify: a + # kernel interface belongs to exactly one bridge, so a cascaded + # link would need a veth pair between the two bridges and stays on + # the relay for now. + switch = self._switch_endpoint() + if switch is not None: + if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch": + return False + peer = node2 if switch[0] == 0 else node1 + return self._kernel_endpoint_ready(peer) return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2) + def _switch_endpoint(self): + """ + The Ethernet switch endpoint of this link as ``(index, node)``, or + None — the fast path where the switch owns the kernel bridge. + """ + + for index, side in enumerate(self._nodes): + if side["node"].node_type == "ethernet_switch": + return index, side["node"] + return None + @staticmethod def _kernel_endpoint_ready(node): """ @@ -250,6 +283,49 @@ class UDPLink(Link): port_number2 = self._nodes[1]["port_number"] if self._kernel_datapath_eligible(node1, node2): + node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True) + node1_markers, node2_markers = self._get_node_markers(node1, node2) + switch = self._switch_endpoint() + if switch is not None: + # Ethernet switch fast path: the switch absorbs the peer's + # anchor into its own kernel bridge — no per-link bridge at + # all, so the peer end carries no bridge name (its anchor is + # bridged by the switch; it only keeps its own interface, + # markers it hosts and its carrier). The switch end gets the + # anchor's name to join: the name is the shared naming + # contract (utils.kernel_anchor), a pure function of the + # peer's node type/id/adapter/port — valid whether or not + # the peer is currently running, which is what makes links + # to stopped nodes wireable (the switch joins once the + # anchor exists, re-pushed by node_started). + index = switch[0] + peer_index = 1 - index + peer = self._nodes[peer_index]["node"] + peer_anchor = kernel_anchor_name( + peer.node_type, + peer.id, + self._nodes[peer_index]["adapter_number"], + self._nodes[peer_index]["port_number"], + ) + peer_data = { + "type": "nio_bridge", + "bridge": None, + "filters": node1_filters if peer_index == 0 else node2_filters, + "markers": node1_markers if peer_index == 0 else node2_markers, + "suspend": self._suspended, + } + switch_data = { + "type": "nio_anchor", + "anchor": peer_anchor, + "filters": node1_filters if index == 0 else node2_filters, + "markers": node1_markers if index == 0 else node2_markers, + "suspend": self._suspended, + } + self._link_data = [switch_data, peer_data] if index == 0 else [peer_data, switch_data] + return [ + (node1, adapter_number1, port_number1, self._link_data[0]), + (node2, adapter_number2, port_number2, self._link_data[1]), + ] # Kernel datapath: each endpoint enslaves its veth host end into # a per-link Linux bridge. The name derives from the link id so # both ends compute it independently (no coordinator) and crash @@ -263,8 +339,6 @@ class UDPLink(Link): # the eBPF stateful classifier (frequency_drop, quota) — every # GNS3 filter type has a kernel equivalent now. bridge_name = "gns3" + self._id.replace("-", "")[:11] - node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True) - node1_markers, node2_markers = self._get_node_markers(node1, node2) self._link_data = [ { "type": "nio_bridge", @@ -580,6 +654,38 @@ class UDPLink(Link): # explicitly deletes a marker via the REST API, and a marker is torn # down automatically only when its link is deleted. + async def node_started(self, node): + """ + Called after a node of this link reached the started state. + + The Ethernet switch fast path wires itself on the switch side, but a + peer's anchor only comes into existence when the peer starts (and a + peer restart can replace the interface under the same name — e.g. + QEMU recreates its TAPs at every start). The switch cannot observe + that, so the controller re-pushes its NIO: the switch completes a + join it had to defer (or re-joins an anchor that disappeared), + then re-applies the link state it owns. + """ + + if not self._created: + return + switch = self._switch_endpoint() + if switch is None or not self.kernel_datapath: + return + index, switch_node = switch + adapter_number = self._nodes[index]["adapter_number"] + port_number = self._nodes[index]["port_number"] + try: + await switch_node.put( + f"/adapters/{adapter_number}/ports/{port_number}/nio", + data=self._link_data[index], + timeout=120, + ) + except (ComputeError, ControllerError): + # The link keeps its stored NIO; the next node start or link + # update retries. A failure here must not fail the node start. + log.warning("Could not re-push switch NIO for link %s after node start", self.id) + async def start_marker( self, name, diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index 196ada3ab..e66a39a4a 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -101,7 +101,7 @@ from .compute.iou_nodes import IOU, IOUCreate, IOUStart, IOUUpdate from .compute.nat_nodes import NAT, NATCreate, NATUpdate # Compute schemas -from .compute.nios import TAPNIO, UDPNIO, BridgeNIO, BatchNIOCreate, BatchNIOEntry, EthernetNIO, MarkerRebuild, MarkerToggle +from .compute.nios import AnchorNIO, TAPNIO, UDPNIO, BridgeNIO, BatchNIOCreate, BatchNIOEntry, EthernetNIO, MarkerRebuild, MarkerToggle from .compute.qemu_nodes import Qemu, QemuCreate, QemuUpdate from .compute.virtualbox_nodes import VirtualBox, VirtualBoxCreate, VirtualBoxUpdate from .compute.vmware_nodes import VMware, VMwareCreate, VMwareUpdate @@ -164,6 +164,7 @@ __all__ = [ "ApplianceVersion", "ApplianceVersionV8", "AutoIdlePC", + "AnchorNIO", "BatchNIOCreate", "BatchNIOEntry", "BridgeNIO", diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 9946cf755..842682e7c 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -84,10 +84,18 @@ class BridgeNIO(BaseModel): uBridge reporting cbpf), frequency_drop and quota as the eBPF stateful classifier (needs ebpf). Markers and packet capture are served by uBridge's AF_PACKET modules on the veth host end. + + ``bridge`` may be None: the anchor is then bridged by an Ethernet switch + (its own kernel bridge absorbed it) and the node applies only carrier, + markers and impairments — never bridge membership. """ type: BridgeNIOType - bridge: str = Field(..., description="Kernel bridge name e.g. gns3a1b2c3d4e5") + bridge: Optional[str] = Field( + None, + description="Kernel bridge name e.g. gns3a1b2c3d4e5; None when an Ethernet switch " + "owns the anchor's bridge membership (absorbed link)", + ) suspend: Optional[bool] = Field(None, description="Suspend the NIO") filters: Optional[dict] = Field( None, @@ -97,6 +105,30 @@ class BridgeNIO(BaseModel): markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the veth host end)") +class AnchorNIOType(str, Enum): + anchor = "nio_anchor" + + +class AnchorNIO(BaseModel): + """ + Anchor-absorbing Network Input/Output properties, for a node that owns a + kernel bridge (the Ethernet switch): the named foreign interface — the + peer's kernel-datapath anchor — is joined to the node's own bridge with + the node's port settings applied to it. The mirror of BridgeNIO; filters, + markers and capture ride the NIO and attach to the named anchor. + """ + + type: AnchorNIOType + anchor: str = Field(..., description="Host interface to absorb e.g. gv00010203e0p0") + suspend: Optional[bool] = Field(None, description="Suspend the NIO") + filters: Optional[dict] = Field( + None, + description="Packet filters served on the absorbed anchor (netem surface + extensions via one tc netem " + "qdisc, bpf via cls_bpf, frequency_drop/quota via the eBPF classifier)", + ) + markers: Optional[dict] = Field(None, description="Traffic-insight markers (attached to the absorbed anchor)") + + class MarkerToggle(BaseModel): """ Body for the per-marker enable/disable toggle endpoint: flips a running @@ -131,7 +163,7 @@ class BatchNIOEntry(BaseModel): node_id: str = Field(..., description="Node the NIO is attached to") adapter_number: int = Field(0, ge=0, description="Adapter number") port_number: int = Field(0, ge=0, description="Port number") - nio: Union[UDPNIO, BridgeNIO] = Field(..., description="NIO settings") + nio: Union[UDPNIO, BridgeNIO, AnchorNIO] = Field(..., description="NIO settings") class BatchNIOCreate(BaseModel): diff --git a/gns3server/utils/kernel_anchor.py b/gns3server/utils/kernel_anchor.py new file mode 100644 index 000000000..02e380c74 --- /dev/null +++ b/gns3server/utils/kernel_anchor.py @@ -0,0 +1,68 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +The kernel-datapath anchor naming contract, in one place. + +Every node type whose adapters own a host-side interface names that anchor +deterministically: ``g{node_id[:8]}e{adapter}p{port}`` — ``gv`` for a +Docker veth host end, ``gq`` for a QEMU TAP, ``gi`` for an IOU port TAP, +``gd`` for a Dynamips port TAP. The prefixes keep anchors out of the +``gns3`` bridge/TAP name space and name-collision-free on a host; the 8 +hex chars of the node id plus adapter/port keep every name unique and +within IFNAMSIZ (15). + +The compute classes create their anchors with these helpers (single source +of truth), and so does the controller when a switch link has to name the +peer's anchor — the name is a pure function of (node type, node id, +adapter, port), valid whether or not the node is currently running, which +is what makes links to stopped nodes wireable. +""" + +import logging + +log = logging.getLogger(__name__) + +# Node type -> the anchor prefix its interface names carry. ``gc`` (the +# Docker veth guest end) is container-local and deliberately absent. +ANCHOR_PREFIX = { + "docker": "gv", + "qemu": "gq", + "iou": "gi", + "dynamips": "gd", +} + + +def anchor_suffix(node_id, adapter_number, port_number=0): + """ + The shared ``{node_id[:8]}e{adapter}p{port}`` name stem (8 hex chars of + the node id, no dashes). + """ + + return f"{str(node_id).replace('-', '')[:8]}e{adapter_number}p{port_number}" + + +def kernel_anchor_name(node_type, node_id, adapter_number, port_number=0): + """ + The host-side kernel-datapath anchor of one adapter port, or None for a + node type without one (the anchor-less types never anchor a kernel + link). + """ + + prefix = ANCHOR_PREFIX.get(node_type) + if prefix is None: + return None + return prefix + anchor_suffix(node_id, adapter_number, port_number) diff --git a/tests/api/routes/compute/test_ethernet_switch_nodes.py b/tests/api/routes/compute/test_ethernet_switch_nodes.py index b942876c3..ad5521488 100644 --- a/tests/api/routes/compute/test_ethernet_switch_nodes.py +++ b/tests/api/routes/compute/test_ethernet_switch_nodes.py @@ -14,7 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from unittest.mock import MagicMock, call +import os +from unittest.mock import MagicMock, call, patch import pytest import pytest_asyncio @@ -535,3 +536,507 @@ class TestEthernetSwitchNodesRoutes: ) assert response.status_code == status.HTTP_204_NO_CONTENT node._ubridge_send.assert_any_call(f"bridge stop_capture {relay}") + + # ------------------------------------------------------------------ # + # kernel datapath: absorbed anchors (nio_anchor) + # ------------------------------------------------------------------ # + + @staticmethod + def _anchor_params(anchor: str = "gv00010203e0p0", **extra) -> dict: + params = {"type": "nio_anchor", "anchor": anchor} + params.update(extra) + return params + + @staticmethod + def _sysfs(anchor_exists: bool = True, bridge_member: bool = True): + """An ``os.path.exists`` that answers the /sys/class/net probes of the + kernel-port wiring, delegating everything else to the real one.""" + real = os.path.exists + + def fake(path): + value = str(path) + if "/brif/" in value: + return bridge_member + if value.startswith("/sys/class/net/"): + return anchor_exists + return real(path) + + return patch("os.path.exists", new=fake) + + async def test_ethernet_switch_create_anchor_nio( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + A kernel link joins the peer's anchor straight to the switch bridge + with the port's VLAN mode applied — no relay, no per-port TAP. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=True): + response = await compute_client.post(url, json=self._anchor_params(anchor)) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["type"] == "nio_anchor" + + node = compute_project.get_node(ethernet_switch["node_id"]) + br = node._bridge_name + # access VLAN 1 (default): the anchor replaces the port TAP as the + # bridge port, so it carries the port's membership + node._ubridge_send.assert_any_call(f'brctl addif "{br}" "{anchor}"') + node._ubridge_send.assert_any_call(f'brctl vlan_del "{br}" "{anchor}" 1') + node._ubridge_send.assert_any_call(f'brctl vlan_add "{br}" "{anchor}" 1 pvid untagged') + node._ubridge_send.assert_any_call(f'link set "{anchor}" up') + # no relay was created for this port + assert not any("bridge create" in str(c) for c in node._ubridge_send.call_args_list) + assert node._kernel_ports[0] == anchor + assert node.get_nio(0).anchor == anchor + + async def test_ethernet_switch_create_anchor_nio_defers_when_anchor_missing( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + The peer's anchor only exists while the peer runs: a link created + against a stopped peer must not fail (nor stop the switch's + uBridge) — the join is deferred to the re-push after the peer + starts. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=False): + response = await compute_client.post(url, json=self._anchor_params(anchor)) + assert response.status_code == status.HTTP_201_CREATED + + node = compute_project.get_node(ethernet_switch["node_id"]) + assert node._kernel_ports[0] == anchor + assert node.get_nio(0).anchor == anchor + assert not any("addif" in str(c) for c in node._ubridge_send.call_args_list) + assert not any("brctl" in str(c) for c in node._ubridge_send.call_args_list) + + async def test_ethernet_switch_update_anchor_nio_reconciles( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + Updating a joined kernel port reconciles on the anchor: tc + impairments, markers and the carrier the suspend flag drives. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=True): + await compute_client.post(url, json=self._anchor_params(anchor)) + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + update_url = app.url_path_for( + "compute:update_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=True, bridge_member=True): + response = await compute_client.put( + update_url, json=self._anchor_params(anchor, filters={"delay": [10]}, suspend=True) + ) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["suspend"] is True + + node._ubridge_send.assert_any_call(f'tc reset "{anchor}"') + node._ubridge_send.assert_any_call(f'tc netem set "{anchor}" delay 10') + node._ubridge_send.assert_any_call(f'link set "{anchor}" down') + # no re-join on a plain reconcile + assert not any("addif" in str(c) for c in node._ubridge_send.call_args_list) + + async def test_ethernet_switch_update_anchor_nio_completes_deferred_join( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + The controller re-pushes the switch NIO once the peer starts: a join + deferred at link creation completes (the anchor exists but is not a + bridge member), and a peer restart that replaced its anchor under + the same name re-joins. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=False): + await compute_client.post(url, json=self._anchor_params(anchor)) + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + update_url = app.url_path_for( + "compute:update_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + # the anchor exists now (peer started) but is not a bridge member + with self._sysfs(anchor_exists=True, bridge_member=False): + response = await compute_client.put(update_url, json=self._anchor_params(anchor)) + assert response.status_code == status.HTTP_201_CREATED + + br = node._bridge_name + node._ubridge_send.assert_any_call(f'brctl addif "{br}" "{anchor}"') + node._ubridge_send.assert_any_call(f'brctl vlan_add "{br}" "{anchor}" 1 pvid untagged') + node._ubridge_send.assert_any_call(f'link set "{anchor}" up') + + async def test_ethernet_switch_delete_anchor_nio( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + Removing a kernel link resets the anchor's qdisc, takes it out of + the switch bridge and drops its carrier — the anchor itself survives + (it belongs to the peer). + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=True): + await compute_client.post(url, json=self._anchor_params(anchor)) + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + with self._sysfs(anchor_exists=True): + response = await compute_client.delete(url) + assert response.status_code == status.HTTP_204_NO_CONTENT + + br = node._bridge_name + node._ubridge_send.assert_any_call(f'tc reset "{anchor}"') + node._ubridge_send.assert_any_call(f'brctl delif "{br}" "{anchor}"') + node._ubridge_send.assert_any_call(f'link set "{anchor}" down') + assert 0 not in node._kernel_ports + assert 0 not in node._nios + + async def test_ethernet_switch_capture_anchor_nio( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + Capture on a kernel port runs on the absorbed anchor (AF_PACKET), + not on a relay. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=True): + await compute_client.post(url, json=self._anchor_params(anchor)) + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + response = await compute_client.post( + app.url_path_for( + "compute:start_ethernet_switch_capture", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ), + json={"capture_file_name": "test.pcap", "data_link_type": "DLT_EN10MB"}, + ) + assert response.status_code == status.HTTP_200_OK + assert any( + c.args and str(c.args[0]).startswith(f'capture start_kernel {anchor} "') + for c in node._ubridge_send.call_args_list + ), node._ubridge_send.call_args_list + + response = await compute_client.post( + app.url_path_for( + "compute:stop_ethernet_switch_capture", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + ) + assert response.status_code == status.HTTP_204_NO_CONTENT + node._ubridge_send.assert_any_call("capture stop_kernel") + + async def test_ethernet_switch_batch_create_anchor_nio( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + The project-open batch endpoint accepts anchor NIOs (a reopen must + not 422 the switch's kernel links). + """ + + anchor = "gv00010203e0p3" + params = { + "nios": [ + { + "node_id": ethernet_switch["node_id"], + "adapter_number": 0, + "port_number": 3, + "nio": {"type": "nio_anchor", "anchor": anchor}, + } + ] + } + with self._sysfs(anchor_exists=False): + response = await compute_client.post( + app.url_path_for("compute:create_batch_nios", project_id=compute_project.id), json=params + ) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["added"] == 1 + + node = compute_project.get_node(ethernet_switch["node_id"]) + assert node._kernel_ports[3] == anchor + + # ------------------------------------------------------------------ # + # in-place VLAN reconfiguration (no port ever leaves the bridge) + # ------------------------------------------------------------------ # + + async def test_ethernet_switch_update_ports_reconfigures_in_place( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + Changing a wired port's VLAN edits its membership in place: delete the + old membership, add the new one — the port is never detached from the + bridge (a relay TAP here; the TAP is the cloud/relay-side port). + """ + + node = compute_project.get_node(ethernet_switch["node_id"]) + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + await compute_client.post(url, json=self._udp_params()) + node._ubridge_send.reset_mock() + + ports = [{"name": f"Ethernet{i}", "port_number": i, "type": "access", "vlan": 1} for i in range(8)] + ports[0]["vlan"] = 10 + response = await compute_client.put( + app.url_path_for( + "compute:update_ethernet_switch", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + ), + json={"ports_mapping": ports}, + ) + assert response.status_code == status.HTTP_200_OK + + br = node._bridge_name + tap = f"{br}-0" + commands = [c.args[0] for c in node._ubridge_send.call_args_list] + assert commands == [ + f'brctl vlan_del "{br}" "{tap}" 1', + f'brctl vlan_add "{br}" "{tap}" 10 pvid untagged', + ], commands + assert not any("delif" in c or "addif" in c for c in commands) + + async def test_ethernet_switch_update_ports_leaves_untouched_ports_alone( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + Only ports whose mode/VLAN changed are touched: re-saving the mapping + no longer re-programs every wired port. + """ + + node = compute_project.get_node(ethernet_switch["node_id"]) + for port_number in (0, 1): + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number=str(port_number), + ) + await compute_client.post(url, json=self._udp_params()) + node._ubridge_send.reset_mock() + + ports = [{"name": f"Ethernet{i}", "port_number": i, "type": "access", "vlan": 1} for i in range(8)] + ports[1]["vlan"] = 20 + response = await compute_client.put( + app.url_path_for( + "compute:update_ethernet_switch", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + ), + json={"ports_mapping": ports}, + ) + assert response.status_code == status.HTTP_200_OK + + br = node._bridge_name + commands = [c.args[0] for c in node._ubridge_send.call_args_list] + assert commands == [ + f'brctl vlan_del "{br}" "{br}-1" 1', + f'brctl vlan_add "{br}" "{br}-1" 20 pvid untagged', + ], commands + assert not any(f'"{br}-0"' in c for c in commands) + + async def test_ethernet_switch_vlan_transition_table(self, compute_project: Project, ethernet_switch: dict) -> None: + """ + Every mode transition is expressed as in-place VLAN edits: the old + mode's entries the new mode does not want, then the new mode's + entries. The dot1q admits-all entry is one range operation in both + directions, and same-to-same emits nothing. + """ + + node = compute_project.get_node(ethernet_switch["node_id"]) + br = node._bridge_name + iface = "gvdeadbeef0p0" + + cases = [ + ( + {"type": "access", "vlan": 10}, + {"type": "dot1q", "vlan": 20}, + [ + f'brctl vlan_del "{br}" "{iface}" 10', + f'brctl vlan_add "{br}" "{iface}" 1 vid 4094', + f'brctl vlan_add "{br}" "{iface}" 20 pvid untagged', + ], + ), + ( + {"type": "dot1q", "vlan": 20}, + {"type": "access", "vlan": 30}, + [ + f'brctl vlan_del "{br}" "{iface}" 1 vid 4094', + f'brctl vlan_add "{br}" "{iface}" 30 pvid untagged', + ], + ), + ( + {"type": "dot1q", "vlan": 20}, + {"type": "dot1q", "vlan": 30}, + [ + f'brctl vlan_del "{br}" "{iface}" 20', + f'brctl vlan_add "{br}" "{iface}" 30 pvid untagged', + ], + ), + ( + {"type": "qinq", "vlan": 2}, + {"type": "dot1q", "vlan": 5}, + [ + f'brctl vlan_del "{br}" "{iface}" 2', + f'brctl vlan_add "{br}" "{iface}" 1 vid 4094', + f'brctl vlan_add "{br}" "{iface}" 5 pvid untagged', + ], + ), + ( + {"type": "dot1q", "vlan": 5}, + {"type": "qinq", "vlan": 2}, + [ + f'brctl vlan_del "{br}" "{iface}" 1 vid 4094', + f'brctl vlan_add "{br}" "{iface}" 2 pvid untagged', + ], + ), + ( + {"type": "access", "vlan": 10}, + {"type": "qinq", "vlan": 2}, + [ + f'brctl vlan_del "{br}" "{iface}" 10', + f'brctl vlan_add "{br}" "{iface}" 2 pvid untagged', + ], + ), + ( + {"type": "qinq", "vlan": 2}, + {"type": "access", "vlan": 10}, + [ + f'brctl vlan_del "{br}" "{iface}" 2', + f'brctl vlan_add "{br}" "{iface}" 10 pvid untagged', + ], + ), + ( + {"type": "qinq", "vlan": 2}, + {"type": "qinq", "vlan": 9}, + [ + f'brctl vlan_del "{br}" "{iface}" 2', + f'brctl vlan_add "{br}" "{iface}" 9 pvid untagged', + ], + ), + ( + {"type": "access", "vlan": 7}, + {"type": "access", "vlan": 7}, + [], + ), + ] + for old_settings, new_settings, expected in cases: + node._ubridge_send.reset_mock() + await node._reconfigure_port_vlan(iface, old_settings, new_settings) + commands = [c.args[0] for c in node._ubridge_send.call_args_list] + assert commands == expected, (old_settings, new_settings, commands) + assert not any("delif" in c or "addif" in c for c in commands) + + async def test_ethernet_switch_update_ports_reconfigures_absorbed_anchor_in_place( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + The same in-place reconcile applies to an absorbed kernel anchor (a + peer node's interface): a ports_mapping update issues VLAN edits on + it and never detaches it from the bridge. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=True): + await compute_client.post(url, json=self._anchor_params(anchor)) + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + ports = [{"name": f"Ethernet{i}", "port_number": i, "type": "access", "vlan": 1} for i in range(8)] + ports[0]["vlan"] = 10 + response = await compute_client.put( + app.url_path_for( + "compute:update_ethernet_switch", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + ), + json={"ports_mapping": ports}, + ) + assert response.status_code == status.HTTP_200_OK + + br = node._bridge_name + commands = [c.args[0] for c in node._ubridge_send.call_args_list] + assert commands == [ + f'brctl vlan_del "{br}" "{anchor}" 1', + f'brctl vlan_add "{br}" "{anchor}" 10 pvid untagged', + ], commands + assert not any("delif" in c or "addif" in c for c in commands) diff --git a/tests/compute/dynamips/test_dynamips_kernel_datapath.py b/tests/compute/dynamips/test_dynamips_kernel_datapath.py index b204f0dc5..b241be526 100644 --- a/tests/compute/dynamips/test_dynamips_kernel_datapath.py +++ b/tests/compute/dynamips/test_dynamips_kernel_datapath.py @@ -612,3 +612,65 @@ async def test_manager_create_nio_builds_the_bridge_nio(router): assert nio.filters == {"delay": [5]} assert nio.suspend is True assert _hypervisor_commands(router) == [] + + +# --------------------------------------------------------------------------- +# Externally bridged anchors (an Ethernet switch absorbed them) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_external_bridge_nio_skips_bridge_membership(router): + """ + A NIOBridge with no bridge means an Ethernet switch owns the anchor's + bridge membership: the router still opens the anchor in the hypervisor + (the TAP needs its fd holder) but never touches any kernel bridge. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router, bridge=None) + + await router.slot_add_nio_binding(0, 0, nio) + + hypervisor = _hypervisor_commands(router) + assert any(c.startswith("nio create_tap tap-") for c in hypervisor) + assert any(c.startswith('vm slot_add_nio_binding "R1" 0 0 tap-') for c in hypervisor) + assert not any("brctl" in c for c in _ubridge_commands(router)) + + +@pytest.mark.asyncio +async def test_external_bridge_nio_remove_skips_bridge_ops(router): + """ + Detaching an externally bridged anchor releases the hypervisor's fd and + resets the link state, but must not delif or delete any bridge — the + switch owns them, and deleting the switch's bridge would take the whole + switch down. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router, bridge=None) + await router.slot_add_nio_binding(0, 0, nio) + router._ubridge_send.reset_mock() + + await router.slot_remove_nio_binding(0, 0) + + commands = _ubridge_commands(router) + assert not any("brctl" in c for c in commands), commands + assert any(c == f'tc reset "{TAP00}"' for c in commands) + + +@pytest.mark.asyncio +async def test_remove_kernel_bridges_skips_external_nios(router): + """ + The bridge sweep never deletes a bridge this node does not own. + """ + + router._kernel_taps[(0, 0)] = TAP00 + external = await _nio(router, bridge=None) + router._slots[0].add_nio(0, external) + + await router._remove_kernel_bridges() + + assert not any("brctl" in c for c in _ubridge_commands(router)) diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 5542783e9..1caff1f7e 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -802,3 +802,131 @@ async def test_available_filters_unconstrained_without_report(project): link, _node1, _node2 = await _kernel_link(project) link._link_data = [{"type": "nio_bridge"}] assert [f["type"] for f in link.available_filters()] == [f["type"] for f in FILTERS] + + +# --------------------------------------------------------------------------- +# Ethernet switch fast path: the switch absorbs the peer's anchor +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_switch_links_select_the_kernel_fast_path(project): + """ + A link with an Ethernet switch endpoint rides the kernel fast path when + the *peer* can anchor — the switch side needs no anchoring capability + (its bridge exists by construction). Switch-to-switch links never + qualify (a kernel interface belongs to exactly one bridge), nor do + anchor-less peers. + """ + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} + switch = _node(project, compute, "sw1", node_type="ethernet_switch") + + for peer in ( + _node(project, compute, "docker1"), + _node(project, compute, "qemu1", node_type="qemu"), + _node(project, compute, "iou1", node_type="iou"), + _node(project, compute, "r1", node_type="dynamips"), + ): + link = await _link(project, switch, peer) + assert link._kernel_datapath_eligible(switch, peer) is True, peer.node_type + + # a switch peer: cascades need a veth pair between two bridges — relay + switch2 = _node(project, compute, "sw2", node_type="ethernet_switch") + link = await _link(project, switch, switch2) + assert link._kernel_datapath_eligible(switch, switch2) is False + + # an anchor-less peer + vpcs = _node(project, compute, "pc1", node_type="vpcs") + link = await _link(project, switch, vpcs) + assert link._kernel_datapath_eligible(switch, vpcs) is False + + # a peer whose compute cannot anchor stays relay too + compute_no_tap = _tap_capable_compute(tap_support=None) + qemu = _node(project, compute_no_tap, "qemu1", node_type="qemu") + switch3 = _node(project, compute_no_tap, "sw3", node_type="ethernet_switch") + link = await _link(project, switch3, qemu) + assert link._kernel_datapath_eligible(switch3, qemu) is False + + +@pytest.mark.asyncio +async def test_switch_link_prepare_emits_anchor_and_external_bridge(project): + """ + The fast path's wire format: the switch receives ``nio_anchor`` naming + the peer's anchor (the shared naming contract, computable while the + node is stopped), the peer receives ``nio_bridge`` with no bridge — + its anchor is bridged by the switch. The link's tc impairments are + owned by exactly one end (the switch): both would hit the same + interface. + """ + + from gns3server.utils.kernel_anchor import kernel_anchor_name + + compute = MagicMock() + compute.id = "compute-1" + switch = _node(project, compute, "sw1", node_type="ethernet_switch") + peer = _node(project, compute, "docker1") + link = await _link(project, switch, peer) + link._filters = {"delay": [10]} + + entries = await link._prepare() + switch_nio = entries[0][3] + peer_nio = entries[1][3] + + assert switch_nio["type"] == "nio_anchor" + assert switch_nio["anchor"] == kernel_anchor_name("docker", peer.id, 0, 0) + assert switch_nio["filters"] == {"delay": [10]} + assert peer_nio["type"] == "nio_bridge" + assert peer_nio["bridge"] is None + assert peer_nio["filters"] == {} + assert link.kernel_datapath is True + + # the reverse endpoint order must keep index alignment + link = await _link(project, peer, switch) + link._filters = {"delay": [10]} + entries = await link._prepare() + assert entries[0][3]["type"] == "nio_bridge" + assert entries[0][3]["bridge"] is None + assert entries[0][3]["filters"] == {} + assert entries[1][3]["type"] == "nio_anchor" + assert entries[1][3]["filters"] == {"delay": [10]} + + +@pytest.mark.asyncio +async def test_switch_link_repushes_the_switch_nio_on_node_start(project): + """ + The switch cannot observe the peer's anchors coming into existence: the + controller re-pushes the switch's NIO after a node of the link starts, + which completes a deferred join (or re-joins an anchor a peer restart + replaced). Relay links do nothing. + """ + + from tests.utils import AsyncioMagicMock + + compute = MagicMock() + compute.id = "compute-1" + switch = _node(project, compute, "sw1", node_type="ethernet_switch") + peer = _node(project, compute, "docker1") + link = await _link(project, switch, peer) + entries = await link._prepare() + link._created = True + switch.put = AsyncioMagicMock() + + await link.node_started(peer) + switch.put.assert_called_once_with( + "/adapters/0/ports/0/nio", data=entries[0][3], timeout=120 + ) + + # a relay link never re-pushes anything + switch.put.reset_mock() + compute2 = MagicMock() + compute2.id = "compute-1" + relay_switch = _node(project, compute2, "sw2", node_type="ethernet_switch") + vpcs = _node(project, compute2, "pc1", node_type="vpcs") + relay_link = await _link(project, relay_switch, vpcs) + relay_link._created = True + relay_switch.put = AsyncioMagicMock() + await relay_link.node_started(vpcs) + assert not relay_switch.put.called diff --git a/tests/controller/test_node.py b/tests/controller/test_node.py index b80aa50c1..2c634c8f9 100644 --- a/tests/controller/test_node.py +++ b/tests/controller/test_node.py @@ -936,6 +936,22 @@ async def test_start(node, compute): compute.post.assert_called_with(f"/projects/{node.project.id}/vpcs/nodes/{node.id}/start", timeout=240) +@pytest.mark.asyncio +async def test_start_notifies_links(node, compute): + """ + A successful start tells the node's links: the Ethernet-switch fast + path completes a deferred anchor join only then (UDPLink.node_started). + """ + + compute.post = AsyncioMagicMock() + link = AsyncioMagicMock() + link.deferred = False + node.add_link(link) + + await node.start() + assert link.node_started.called + + @pytest.mark.asyncio async def test_start_iou(compute, project, controller): diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index d1690f2d0..239ce9bb8 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -282,6 +282,12 @@ def live_server(kernel=True): already be configured for it); everything else starts an isolated instance. Relay-mode tests always need an isolated instance — the datapath choice is a server configuration. + + Either way this waits for the controller's local compute to finish + connecting: the connection's ``GET /capabilities`` runs the uBridge + probes (several ubridge spawns, seconds), and until it completes the + controller caches a default dict with every capability None — a test + reading capabilities in that window would wrongly skip itself. """ url = os.environ.get("GNS3_E2E_URL") if url and kernel: @@ -290,8 +296,14 @@ def live_server(kernel=True): os.environ.get("GNS3_E2E_USER", "admin"), os.environ.get("GNS3_E2E_PASSWORD", "admin"), ).login() - return Server(compute) - return _start_isolated(kernel) + server = Server(compute) + else: + server = _start_isolated(kernel) + if not wait_until(lambda: (server.compute.capabilities() or {}).get("version"), timeout=45): + raise AssertionError( + f"{server.compute.url}: the controller's local compute did not report capabilities within 45s" + ) + return server def release(server, project_id, failed=False): diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py new file mode 100644 index 000000000..57a2b8f25 --- /dev/null +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -0,0 +1,290 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Live end-to-end test for the Ethernet switch kernel fast path (pytest +marker ``e2e``): a real switch (a Linux kernel bridge) between two real +c7200 routers, driven through the controller REST API and the IOS consoles. +Real ICMP crossing the switch proves the datapath: + +* links created while the routers are stopped report ``kernel_datapath`` + and defer the join; once the routers boot, the controller's re-push + makes the switch join their anchors — the switch bridge's members become + exactly the two router anchors (no relay TAPs anywhere); +* VLAN programming applies to absorbed anchors: different access VLANs + isolate the routers (ping dies), the same VLAN bridges them again; +* a ``delay 100`` filter lands as netem on the absorbed anchor (single + interface, switch-owned) and the measured RTT grows by ~100 ms; +* suspend admin-downs the anchor and kills the traffic; resume restores; +* capture (hosted by the switch side) writes a pcap of ICMP records; +* deleting a link detaches the anchor (which survives — the router owns + it) and re-creating re-joins it; +* a router stop/start keeps the wiring and the traffic; +* deleting the project removes the switch bridge and the anchors. + +``test_ethernet_switch_relay_control`` is the negative control on an +isolated relay-configured instance: the same topology rides the per-port +uBridge relays (the switch bridge's members are its own relay TAPs, never +the routers' anchors) and still pings. +""" + +import time + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# c7200 slot 0 is the fixed IO slot; the Ethernet PA goes in slot 1. +SLOTS = {"slot1": "PA-2FE-TX"} +ETH = (1, 0) + + +def _pick_image(server): + images = server.compute.dynamips_images() + image = next((name for name in images if name.startswith("c7200")), None) + if not image: + pytest.skip(f"no c7200 image available on this server (images: {images[:5]})") + return image + + +def _switch_bridge_name(switch_id): + # see EthernetSwitch._ensure_bridge + return "gns3" + switch_id.replace("-", "")[:6] + + +def _create_topology(compute, pid, image, name): + switch = compute.call( + "POST", + f"/projects/{pid}/nodes", + { + "compute_id": "local", + "name": f"{name}-SW", + "node_type": "ethernet_switch", + "properties": { + "ports_mapping": [ + {"name": "Ethernet0", "port_number": 0, "type": "access", "vlan": 1}, + {"name": "Ethernet1", "port_number": 1, "type": "access", "vlan": 1}, + ] + }, + }, + ) + r1 = compute.create_dynamips_router(pid, f"{name}-R1", image, SLOTS) + r2 = compute.create_dynamips_router(pid, f"{name}-R2", image, SLOTS) + return switch, r1, r2 + + +def _boot(server, project_id, node_id, timeout=360): + console = harness.dynamips_console(server, project_id, node_id) + console.boot_wait(timeout=timeout) + return console + + +def test_ethernet_switch_kernel_fast_path(): + server = harness.live_server(kernel=True) + compute = server.compute + if compute.capabilities().get("ubridge_tap") is not True: + pytest.skip("this compute's uBridge cannot create persistent TAPs (ubridge_tap != True)") + image = _pick_image(server) + + project = compute.create_project("esw-e2e") + pid = project["project_id"] + sw_bridge = None + try: + switch, r1, r2 = _create_topology(compute, pid, image, "E2E") + switch_id = switch["node_id"] + r1_id, r2_id = r1["node_id"], r2["node_id"] + sw_bridge = _switch_bridge_name(switch_id) + a1 = harness.anchor_name(r1_id, *ETH) + a2 = harness.anchor_name(r2_id, *ETH) + + # The switch bridge exists from the moment the switch node does; a + # link against a stopped router must not fail — the join defers. + assert harness.bridge_members(sw_bridge) == [] + link1 = compute.create_link(pid, (r1_id, *ETH), (switch_id, 0, 0)) + link2 = compute.create_link(pid, (r2_id, *ETH), (switch_id, 0, 1)) + assert link1["kernel_datapath"] is True, link1 + assert link2["kernel_datapath"] is True, link2 + assert harness.bridge_members(sw_bridge) == [] + + print(".. starting routers and waiting for IOS boots") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{r2_id}/start") + c1 = _boot(server, pid, r1_id) + c2 = _boot(server, pid, r2_id) + + # Deferred joins completed by the node-start re-push: the anchors + # ARE the switch ports now. + assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) == sorted([a1, a2]), timeout=15), ( + harness.bridge_members(sw_bridge) + ) + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + print(".. both routers configured, pinging through the switch") + baseline = harness.wait_ping(c1, "10.1.1.2") + assert baseline["success"] == 100, baseline["raw"] + + # VLAN programming on absorbed anchors: different access VLANs + # isolate, the same VLAN bridges again. + compute.call( + "PUT", + f"/projects/{pid}/nodes/{switch_id}", + { + "properties": { + "ports_mapping": [ + {"name": "Ethernet0", "port_number": 0, "type": "access", "vlan": 10}, + {"name": "Ethernet1", "port_number": 1, "type": "access", "vlan": 20}, + ] + } + }, + ) + assert harness.bridge_members(sw_bridge) == sorted([a1, a2]), harness.bridge_members(sw_bridge) + isolated = harness.ping(c1, "10.1.1.2", repeat=3) + assert isolated["success"] == 0, isolated["raw"] + assert harness.bridge_members(sw_bridge) == sorted([a1, a2]), harness.bridge_members(sw_bridge) + compute.call( + "PUT", + f"/projects/{pid}/nodes/{switch_id}", + { + "properties": { + "ports_mapping": [ + {"name": "Ethernet0", "port_number": 0, "type": "access", "vlan": 10}, + {"name": "Ethernet1", "port_number": 1, "type": "access", "vlan": 10}, + ] + } + }, + ) + rejoined = harness.wait_ping(c1, "10.1.1.2") + assert rejoined["success"] == 100, rejoined["raw"] + + # Single-owned impairment: the filter lands on the absorbed anchor + # (one interface — the switch's), RTT grows by ~100 ms one-way. + compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(a1), harness.qdiscs(a1) + assert "netem" not in harness.qdiscs(a2), harness.qdiscs(a2) + delayed = harness.wait_ping(c1, "10.1.1.2") + assert delayed["success"] == 100, delayed["raw"] + assert 60 <= delayed["avg"] <= 250, (baseline, delayed) + compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"filters": {}}) + assert "netem" not in harness.qdiscs(a1) + fast = harness.wait_ping(c1, "10.1.1.2") + assert fast["success"] == 100 and fast["avg"] < 50, fast + + # suspend: the anchor admin-downs, the link is dead; resume restores + compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"suspend": True}) + assert not harness.tap_up(a1) + dead = harness.ping(c1, "10.1.1.2", repeat=3) + assert dead["success"] == 0, dead["raw"] + compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"suspend": False}) + assert harness.tap_up(a1) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + + # capture (the controller hosts it on the switch — always-on local + # builtin): a pcap of ICMP records written off the absorbed anchor + capture = compute.call( + "POST", f"/projects/{pid}/links/{link1['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} + ) + harness.ping(c1, "10.1.1.2") + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{link1['link_id']}/capture/stop") + count, ethertypes = harness.pcap_records(capture["capture_file_path"]) + assert count >= 4, capture["capture_file_path"] + assert "0800" in ethertypes, ethertypes + + # link delete detaches the anchor (it survives — the router owns + # it); re-creating re-joins it + compute.call("DELETE", f"/projects/{pid}/links/{link2['link_id']}") + assert harness.bridge_members(sw_bridge) == [a1] + assert harness.tap_exists(a2) + link2 = compute.create_link(pid, (r2_id, *ETH), (switch_id, 0, 1)) + assert link2["kernel_datapath"] is True, link2 + assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) == sorted([a1, a2]), timeout=10) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + # router stop/start keeps the wiring: a Dynamips stop only halts the + # emulated router — the hypervisor, the anchor and its bridge + # membership all survive (the same stop-is-not-close semantics as + # the per-link bridge datapath) + print(".. stopping and restarting R1") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/stop") + assert harness.tap_exists(a1) + assert harness.bridge_members(sw_bridge) == sorted([a1, a2]) + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + c1.close() + c1 = _boot(server, pid, r1_id) + assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) == sorted([a1, a2]), timeout=15) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # project delete closed everything: switch bridge and anchors gone + assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) is None, timeout=15), sw_bridge + + +def test_ethernet_switch_relay_control(): + """Negative control: the same topology on an isolated relay-configured + instance rides the per-port uBridge relays — the switch bridge's + members are its own relay TAPs, never the routers' anchors — and still + pings.""" + server = harness.live_server(kernel=False) + compute = server.compute + image = _pick_image(server) + + project = compute.create_project("esw-e2e-relay") + pid = project["project_id"] + sw_bridge = None + try: + switch, r1, r2 = _create_topology(compute, pid, image, "E2E") + sw_bridge = _switch_bridge_name(switch["node_id"]) + r1_id, r2_id = r1["node_id"], r2["node_id"] + a1 = harness.anchor_name(r1_id, *ETH) + + link1 = compute.create_link(pid, (r1_id, *ETH), (switch["node_id"], 0, 0)) + link2 = compute.create_link(pid, (r2_id, *ETH), (switch["node_id"], 0, 1)) + assert link1["kernel_datapath"] is False, link1 + assert link2["kernel_datapath"] is False, link2 + + print(".. starting routers and waiting for IOS boots (relay)") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{r2_id}/start") + c1 = _boot(server, pid, r1_id) + c2 = _boot(server, pid, r2_id) + + members = harness.bridge_members(sw_bridge) + assert members == sorted([f"{sw_bridge}-0", f"{sw_bridge}-1"]), members + assert a1 not in members + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + relay_ping = harness.wait_ping(c1, "10.1.1.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) is None, timeout=15), sw_bridge diff --git a/tests/utils/test_kernel_anchor.py b/tests/utils/test_kernel_anchor.py new file mode 100644 index 000000000..e5d091ec2 --- /dev/null +++ b/tests/utils/test_kernel_anchor.py @@ -0,0 +1,47 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +The kernel-datapath anchor naming contract (shared by the compute classes +that create anchors and the controller that names a peer's anchor for an +Ethernet-switch link). +""" + +from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name + +NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" + + +def test_every_node_type_has_its_own_prefix(): + assert kernel_anchor_name("docker", NODE_ID, 0, 0) == "gv00010203e0p0" + assert kernel_anchor_name("qemu", NODE_ID, 1, 2) == "gq00010203e1p2" + assert kernel_anchor_name("iou", NODE_ID, 0, 3) == "gi00010203e0p3" + assert kernel_anchor_name("dynamips", NODE_ID, 6, 48) == "gd00010203e6p48" + + +def test_unknown_node_types_have_no_anchor(): + assert kernel_anchor_name("ethernet_switch", NODE_ID, 0, 0) is None + assert kernel_anchor_name("vpcs", NODE_ID, 0, 0) is None + + +def test_names_fit_ifnamsiz(): + # worst case: WIC port numbers reach 48 with the longest prefix + assert len(kernel_anchor_name("dynamips", NODE_ID, 6, 48)) <= 15 + + +def test_suffix_is_stable_across_the_node_type(): + assert anchor_suffix(NODE_ID, 1, 2) == "00010203e1p2" + assert kernel_anchor_name("docker", NODE_ID, 1, 2) == "gv" + anchor_suffix(NODE_ID, 1, 2) From 0b78d313eb40ae7e37e41946693544a1d0ea53cf Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 22:41:50 +0800 Subject: [PATCH 039/101] fix(qemu): release the datapath after the process is gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QEMU is the guest side of every adapter TAP, and uBridge's tap delete refuses a device another process holds open ('Device or resource busy') — a best-effort delete whose failure is suppressed. QEMU's stop did the delete first and stopped the process after, so every node stop or project delete leaked the persistent TAPs: found by inspecting the host after a project delete (16 taps left behind, carrier down, no holder), and the held-tap behaviour reproduced against uBridge directly. Stop the QEMU process first, then remove the TAPs and the per-link kernel bridges — process side first, then the devices, the same lesson as IOU's reverse stop order. Verified live: node stop and project delete now leave no taps and no processes behind, while a restart recreates the TAPs. The adapter anchor naming also goes through the shared utils.kernel_anchor helper, like the other node types. --- docs/features/qemu-kernel-datapath.md | 18 +++++++--- gns3server/compute/qemu/qemu_vm.py | 19 +++++++---- .../compute/qemu/test_qemu_kernel_datapath.py | 33 ++++++++++++++++++- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 9ee177e77..91ee4fc8d 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -83,11 +83,15 @@ datapaths (delete + re-create, or a project reopen). * **Life (running)** — the TAP is never created or destroyed again. Link create/delete/switch, suspend, capture and markers only change *what is attached to it*. -* **Death (node stop)** — `_remove_taps` un-persists the TAPs (`tap delete`; - QEMU's open fd keeps each device alive until it exits) and deletes the - per-link kernel bridges the node still holds. Deleting a node never runs the - link-teardown path, so those bridges would otherwise stay behind as empty - orphans; a restart rebuilds them from the NIO. +* **Death (node stop)** — the QEMU process is stopped **first**, then + `_remove_taps` un-persists the TAPs (`tap delete`) and deletes the + per-link kernel bridges the node still holds. The order matters: uBridge's + `tap delete` refuses a device another process holds open ("Device or + resource busy") and that best-effort delete is suppressed, so deleting the + TAPs while QEMU still ran leaked every persistent TAP (process side first, + then the devices — the same lesson as IOU's reverse stop order). Deleting + a node never runs the link-teardown path, so those bridges would otherwise + stay behind as empty orphans; a restart rebuilds them from the NIO. ## Link operations @@ -176,3 +180,7 @@ TAPs bound to the IOL fabric; serial links stay relay). **Dynamips has landed** too — see `docs/features/dynamips-kernel-datapath.md` (the hypervisor opens uBridge-created TAPs with `nio create_tap`: the same external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay). +**The Ethernet switch has landed** — see +`docs/features/ethernet-switch-kernel-datapath.md` (the switch absorbs a +peer's anchor into its own kernel bridge; switch-to-switch cascades stay +relay for now). diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 9309e25b5..7467acdcf 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -40,6 +40,7 @@ import gns3server from gns3server.schemas.compute.qemu_nodes import Qemu, QemuPlatform from gns3server.utils import parse_version from gns3server.utils.asyncio import cancellable_wait_run_in_executor, subprocess_check_output +from gns3server.utils.kernel_anchor import kernel_anchor_name from ...utils import int_to_macaddress, is_ipv6_enabled, macaddress_to_int from ...utils.asyncio import monitor_process @@ -1232,7 +1233,6 @@ class QemuVM(KernelDatapathMixin, BaseNode): Stops this QEMU VM. """ - await self._stop_ubridge() async with self._execute_lock: # stop the QEMU process self._hw_virtualization = False @@ -1274,6 +1274,14 @@ class QemuVM(KernelDatapathMixin, BaseNode): await self._export_config() await super().stop() + # Release the datapath only once QEMU is gone: the process holds the + # adapter TAPs open (it is the guest side), and uBridge's tap delete + # refuses a held device ("Device or resource busy") — deleting them + # while QEMU still ran leaked the persistent TAPs, because that + # best-effort delete is suppressed. Order: process side first, then + # the devices (the same lesson as IOU's reverse stop order). + await self._stop_ubridge() + async def _open_qemu_monitor_connection_vm(self, timeout=10): """ Opens a connection to the QEMU monitor. @@ -1494,13 +1502,12 @@ class QemuVM(KernelDatapathMixin, BaseNode): def _tap_name(self, adapter_number, port_number=0): """ - Deterministic anchor TAP name for an adapter port. The ``gq`` prefix - keeps it out of the ``gns3`` bridge/TAP name space (and apart from - Docker's ``gv``/``gc`` veth names); 8 hex chars of the node id plus - adapter/port keep it unique and within IFNAMSIZ (15). + Deterministic anchor TAP name for an adapter port (the shared + utils.kernel_anchor naming contract — the controller names a peer's + anchor with the same function when an Ethernet switch absorbs it). """ - return f"gq{self._id.replace('-', '')[:8]}e{adapter_number}p{port_number}" + return kernel_anchor_name("qemu", self._id, adapter_number, port_number) def _kernel_host_ifc(self, adapter_number, port_number=0): """ diff --git a/tests/compute/qemu/test_qemu_kernel_datapath.py b/tests/compute/qemu/test_qemu_kernel_datapath.py index a568e4327..707f582c1 100644 --- a/tests/compute/qemu/test_qemu_kernel_datapath.py +++ b/tests/compute/qemu/test_qemu_kernel_datapath.py @@ -270,7 +270,7 @@ async def test_update_nio_binding_reapplies_filters_on_the_anchor(vm): vm._ubridge_send.assert_any_call(f'tc reset "{TAP0}"') vm._ubridge_send.assert_any_call(f'tc netem set "{TAP0}" delay 10') - assert not any(f'brctl addif' in str(c) for c in vm._ubridge_send.call_args_list) + assert not any(f"brctl addif" in str(c) for c in vm._ubridge_send.call_args_list) @pytest.mark.asyncio @@ -346,3 +346,34 @@ async def test_capture_on_a_relay_link_uses_the_relay_bridge(vm): await vm.stop_capture(0) vm._ubridge_send.assert_any_call(f"bridge stop_capture QEMU-{vm.id}-0") + + +@pytest.mark.asyncio +async def test_stop_stops_the_process_before_deleting_the_taps(vm): + """ + The QEMU process is the guest side of every adapter TAP, and uBridge's + tap delete refuses a held device ("Device or resource busy") — that + best-effort delete is suppressed, so removing the taps while QEMU still + ran leaked the persistent TAPs. The process must be gone first (the same + lesson as IOU's reverse stop order: process side, then the devices). + """ + + _running(vm) + order = [] + + async def fake_termination(process, timeout=None): + order.append("process") + + async def fake_stop_ubridge(): + order.append("taps") + + with ( + patch("gns3server.utils.asyncio.wait_for_process_termination", new=fake_termination), + patch.object(vm, "_stop_ubridge", new=fake_stop_ubridge), + asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._export_config"), + asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._clear_save_vm_stated"), + asyncio_patch("gns3server.compute.base_node.BaseNode.stop"), + ): + await vm.stop() + + assert order == ["process", "taps"] From a40503e44babe988fdb7bf63745e18d083efd0d6 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 23:10:55 +0800 Subject: [PATCH 040/101] fix(qemu): release the datapath under the execution lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QEMU's process monitor calls stop() by itself when the process dies, so an API stop and a process-death stop run concurrently in the normal case — and the datapath release sat outside _execute_lock, letting both stops sweep the adapter TAP map at the same time. Whichever finished first cleared _kernel_taps under the other's iterator, which died with "dictionary changed size during iteration" (the monitor's task, so it surfaced as "Task exception was never retrieved") with the rest of its sweep, the orphan-bridge removal and uBridge's shutdown skipped. Keep the previous fix's ordering (process side first, then the devices) and hold the lock across the release: serialized, the second stop finds uBridge gone and sweeps nothing. Reproduced deterministically with two concurrent stop() calls, the first parked mid-sweep — it raised at qemu_vm.py:1659 before the change, and every TAP is swept exactly once after it. --- gns3server/compute/qemu/qemu_vm.py | 23 ++++--- .../compute/qemu/test_qemu_kernel_datapath.py | 63 +++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 7467acdcf..873986ace 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1274,13 +1274,22 @@ class QemuVM(KernelDatapathMixin, BaseNode): await self._export_config() await super().stop() - # Release the datapath only once QEMU is gone: the process holds the - # adapter TAPs open (it is the guest side), and uBridge's tap delete - # refuses a held device ("Device or resource busy") — deleting them - # while QEMU still ran leaked the persistent TAPs, because that - # best-effort delete is suppressed. Order: process side first, then - # the devices (the same lesson as IOU's reverse stop order). - await self._stop_ubridge() + # Release the datapath only once QEMU is gone: the process holds + # the adapter TAPs open (it is the guest side), and uBridge's tap + # delete refuses a held device ("Device or resource busy") — + # deleting them while QEMU still ran leaked the persistent TAPs, + # because that best-effort delete is suppressed. Order: process + # side first, then the devices (the same lesson as IOU's reverse + # stop order). + # + # It stays under the lock: the process monitor calls stop() on its + # own when QEMU dies, so an API stop and a process-death stop run + # concurrently in the normal case — and the TAP sweep walks the + # adapter map across awaits, which the second stop was free to + # clear underneath it ("dictionary changed size during + # iteration"). Serialized, it finds uBridge already gone and + # sweeps nothing. + await self._stop_ubridge() async def _open_qemu_monitor_connection_vm(self, timeout=10): """ diff --git a/tests/compute/qemu/test_qemu_kernel_datapath.py b/tests/compute/qemu/test_qemu_kernel_datapath.py index 707f582c1..ccee4796c 100644 --- a/tests/compute/qemu/test_qemu_kernel_datapath.py +++ b/tests/compute/qemu/test_qemu_kernel_datapath.py @@ -21,6 +21,7 @@ netdev, kernel-link enslavement, relay attach on the anchor, capture, and the legacy socket-netdev fallback for a uBridge without the tap module. """ +import asyncio import os import stat import sys @@ -41,6 +42,7 @@ from gns3server.compute.ubridge.ubridge_error import UbridgeError BRIDGE = "gns3a1b2c3d4e5f" TAP0 = "gq00010203e0p0" # the vm fixture's node id, adapter 0 +TAP1 = "gq00010203e1p0" # ... adapter 1 @pytest_asyncio.fixture @@ -377,3 +379,64 @@ async def test_stop_stops_the_process_before_deleting_the_taps(vm): await vm.stop() assert order == ["process", "taps"] + + +@pytest.mark.asyncio +async def test_concurrent_stops_sweep_every_tap_exactly_once(vm): + """ + The process monitor calls stop() by itself when QEMU dies, so an API stop + and a process-death stop run concurrently in the normal case. Both used to + reach the TAP sweep, which walks the adapter map across awaits: whichever + swept first cleared the map under the other's iterator, and the survivor + died with "dictionary changed size during iteration" — logged against the + monitor's task as "Task exception was never retrieved", with the rest of + its sweep skipped. + + The first stop is parked inside the sweep while the second one runs to + completion (in the fixed code it waits on the execution lock instead), then + released: every TAP must be swept exactly once and the sweep must survive. + """ + + _running(vm) + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + vm._kernel_taps[(1, 0)] = TAP1 + + release = asyncio.Event() + deletes = [] + + async def send(command): + if command.startswith("tap delete"): + deletes.append(command) + if len(deletes) == 1: + # park the first stop in the middle of its sweep + await release.wait() + + vm._ubridge_send = AsyncioMagicMock(side_effect=send) + + async def fake_termination(process, timeout=None): + pass + + with ( + patch("gns3server.utils.asyncio.wait_for_process_termination", new=fake_termination), + asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._export_config"), + asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._clear_save_vm_stated"), + asyncio_patch("gns3server.compute.base_node.BaseNode.stop"), + ): + api_stop = asyncio.ensure_future(vm.stop()) + for _ in range(50): + if deletes: + break + await asyncio.sleep(0) + assert deletes, "the first stop never reached the TAP sweep" + + monitor_stop = asyncio.ensure_future(vm.stop()) + for _ in range(50): + if monitor_stop.done(): + break + await asyncio.sleep(0) + + release.set() + await asyncio.gather(api_stop, monitor_stop) + + assert sorted(deletes) == [f'tap delete "{TAP0}"', f'tap delete "{TAP1}"'] From caa054986f5ba02b050b50f3b1a1859f6f7a153e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 1 Oct 2026 23:34:17 +0800 Subject: [PATCH 041/101] fix(controller): keep IOL runner containers off the kernel datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unix-socket container gate matched GNS3_UNIX_SOCKET_NIO in the node environment only, but the IOL runner reaches the same wiring through the GNS3_IOL_RUNNER marker: the compute selects IOLDockerVM on it, and that class forces the AF_UNIX socket pairs whatever the generic knob says — and the documented environment for those nodes carries only the marker. Such a container was therefore handed a kernel link its compute cannot serve. The plain case failed at link creation (no veth end to enslave); on the switch fast path it was worse — the switch defers a join against a missing anchor by design (a link to a stopped node must not fail), so the cable was silently dead instead of merely slow. Match both markers, through the shared is_iol_runner_environment helper so the controller and the compute select the same way. Verified live on a real server: docker↔IOL-XE and switch↔IOL-XE links stay relay while the docker↔switch control link still takes the kernel datapath. --- gns3server/controller/udp_link.py | 13 ++++-- tests/controller/test_kernel_datapath_link.py | 45 +++++++++++++++++-- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 5b37513d3..4ed74b3e8 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -20,6 +20,7 @@ import asyncio import logging from gns3server.config import Config +from gns3server.utils.application_id import is_iol_runner_environment from gns3server.utils.kernel_anchor import kernel_anchor_name from gns3server.utils.packet_filter_validation import ( KERNEL_UNSUPPORTED_FILTERS, @@ -56,14 +57,18 @@ log = logging.getLogger(__name__) def _is_unix_socket_docker(node): """ Best-effort detection of vendor containers bridged through AF_UNIX - socket pairs (GNS3_UNIX_SOCKET_NIO in their environment): they have no - host-side interface to enslave into a kernel bridge. The compute side + socket pairs: they have no host-side interface to enslave into a kernel + bridge. Two ways in — the generic GNS3_UNIX_SOCKET_NIO knob, or the IOL + runner marker, whose class (IOLDockerVM, selected on GNS3_IOL_RUNNER) + forces the unix-socket wiring whatever the knob says. The compute side independently rejects kernel-datapath NIOs on these containers, so a - missed detection here only surfaces as a clearer-late error. + missed detection surfaces as a clearer-late error — except on the switch + fast path, where an absent anchor is deferred forever by design (a link + to a stopped node must not fail), making the miss a silently dead cable. """ environment = (node.properties or {}).get("environment") or "" - return "GNS3_UNIX_SOCKET_NIO" in environment + return "GNS3_UNIX_SOCKET_NIO" in environment or is_iol_runner_environment(environment) class UDPLink(Link): diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 1caff1f7e..9087fe720 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -419,6 +419,47 @@ async def test_kernel_datapath_not_eligible_for_unix_socket_containers(project): assert link._kernel_datapath_eligible(node1, node2) is False +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_iol_runner_containers(project): + """ + The IOL runner marker is the other way into the unix-socket datapath: the + compute selects IOLDockerVM on GNS3_IOL_RUNNER and that class wires the + adapters through AF_UNIX socket pairs whatever the generic + GNS3_UNIX_SOCKET_NIO knob says — and the documented environment for + those nodes carries only the marker. Matching the knob alone let them + through to a kernel link their compute cannot serve. + """ + + link, node1, node2 = await _kernel_link(project) + node2._properties = {"environment": "GNS3_IOL_RUNNER=1"} + assert link._kernel_datapath_eligible(node1, node2) is False + + # the marker travels in a multi-line environment like any other knob + node2._properties = {"environment": "GNS3_IOL_RUNNER=1\nGNS3_IOL_STARTUP_CONFIG=cfg.txt"} + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_an_iol_runner_container_on_a_switch(project): + """ + The switch fast path absorbs the peer's anchor, and a container bridged + through unix sockets has none — nor will it ever. The switch defers a + join against a stopped peer by design (it must not fail the link), so + such a cable would be silently dead instead of merely slow: the gate has + to run before the switch branch, keeping the link on the relay the + switch's own port TAP serves. + """ + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} + switch = _node(project, compute, "sw1", node_type="ethernet_switch") + peer = _node(project, compute, "iol1", environment="GNS3_IOL_RUNNER=1") + link = await _link(project, switch, peer) + + assert link._kernel_datapath_eligible(switch, peer) is False + + @pytest.mark.asyncio async def test_kernel_datapath_disabled_by_config(project): @@ -915,9 +956,7 @@ async def test_switch_link_repushes_the_switch_nio_on_node_start(project): switch.put = AsyncioMagicMock() await link.node_started(peer) - switch.put.assert_called_once_with( - "/adapters/0/ports/0/nio", data=entries[0][3], timeout=120 - ) + switch.put.assert_called_once_with("/adapters/0/ports/0/nio", data=entries[0][3], timeout=120) # a relay link never re-pushes anything switch.put.reset_mock() From 90bcb324730620ed7047526f2782fed48ada0f12 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 00:31:47 +0800 Subject: [PATCH 042/101] feat(esw): cascade two switches through a veth pair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch-to-switch links were the last shape left on the relay: the switch fast path works by absorbing the *peer's* anchor, and a switch has none — its ports are its own bridge, and a kernel interface belongs to exactly one bridge, so two bridges cannot be joined by enslaving anything. Join them with a veth pair instead: one end per bridge, each side enslaving its own end and applying its own port mode to it — an access port on one side and a trunk on the other behaves like the real cable it models. uBridge's docker create_veth/delete_veth (the primitive every Docker adapter is already born from) creates and destroys the pair; the names derive from the link id (utils.kernel_anchor.kernel_cascade_names), so each side computes its own end and the names survive a restart. The pair belongs to the link, and the two ends are wired by two independent uBridge processes — the project-open batch drives different nodes concurrently — so the protocol is order-free: both sides create the pair when their own end is missing (one create wins; the loser's failure is re-checked against the kernel and "the pair is there" is not an error), and either side's teardown destroys it (deleting one veth end takes the other with it, so both converge on the same gone pair and the second delif answers ENOENT). A pre-existing end — this switch restarted, or a leftover from a previous life of the link id — is reused rather than recreated, because the other end may still be a live port of the peer switch; a stale tc qdisc is all a reused end can carry. No new wire format was needed: a cascade is an anchor NIO on both ends, the absorbed-anchor case plus the peer end's name. That also corrects the filter routing — the single-sided rule exists because an absorbed anchor is one interface, while a cascade has one per switch, so impairments go to both ends like any other kernel link. Tests: the wire format, the eligibility flip and the both-ends re-push on node start (controller); the naming contract (utils); the switch-side lifecycle — create, reuse, lost create race, delete destroys the pair (compute routes). Drive-by: the relay filter test's compute mock carried no awaitable `put`, so tests/controller/test_kernel_datapath_link.py failed when run alone while an argument-less full run passed; callers now get one. --- gns3server/compute/base_manager.py | 5 +- .../compute/builtin/nodes/ethernet_switch.py | 50 +++++- gns3server/compute/nios/nio_anchor.py | 28 ++- gns3server/controller/link.py | 7 +- gns3server/controller/udp_link.py | 106 ++++++++--- gns3server/schemas/compute/nios.py | 5 + gns3server/utils/kernel_anchor.py | 16 ++ .../compute/test_ethernet_switch_nodes.py | 168 ++++++++++++++++++ tests/controller/test_kernel_datapath_link.py | 103 ++++++++++- tests/utils/test_kernel_anchor.py | 25 ++- 10 files changed, 472 insertions(+), 41 deletions(-) diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index 35c8be784..8df8703b7 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -384,8 +384,9 @@ class BaseManager: elif nio_settings["type"] == "nio_anchor": # The node-owned-bridge mirror of nio_bridge: the Ethernet # switch joins the named foreign anchor to its own kernel - # bridge (AnchorNIO/NIOAnchor). - nio = NIOAnchor(nio_settings["anchor"]) + # bridge (AnchorNIO/NIOAnchor). A cascade link carries the + # other veth end's name in "peer" on the owning side. + nio = NIOAnchor(nio_settings["anchor"], nio_settings.get("peer")) nio.filters = nio_settings.get("filters", {}) nio.markers = nio_settings.get("markers", {}) nio.suspend = nio_settings.get("suspend", False) diff --git a/gns3server/compute/builtin/nodes/ethernet_switch.py b/gns3server/compute/builtin/nodes/ethernet_switch.py index 63f26153f..48188bde1 100644 --- a/gns3server/compute/builtin/nodes/ethernet_switch.py +++ b/gns3server/compute/builtin/nodes/ethernet_switch.py @@ -80,6 +80,11 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): self._nios = {} self._tap_by_port = {} # port_number -> kernel TAP enslaved to the bridge self._kernel_ports = {} # port_number -> foreign anchor absorbed into the bridge + # port_number -> the other end's name, for the ports that are one end + # of a switch-to-switch cascade veth pair (created when missing, + # destroyed on teardown — symmetrically, from either side). Absent + # for absorbed anchors and relay ports. + self._cascade_peers = {} self._bridge_name = None # kernel bridge interface name (allocated on start) self._bridge_created = False self._bridge_proto_set = False # whether ``brctl setvlanproto`` has been applied @@ -299,6 +304,17 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): for anchor in self._kernel_ports.values(): with contextlib.suppress(UbridgeError): await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{anchor}"') + # Cascade pairs die with the switch: the delif above released + # this end, and deleting it takes the peer end (still enslaved + # on the other side, if that close is still in flight) out of + # its bridge as well — which is where the pair is headed anyway, + # the link is going away with the project. The other switch's + # own delete answers ENOENT, suppressed. + for port_number in self._cascade_peers: + anchor = self._kernel_ports.get(port_number) + if anchor is not None: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{anchor}"') for tap in self._tap_by_port.values(): with contextlib.suppress(UbridgeError): await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{tap}"') @@ -313,6 +329,7 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): self._bridge_name = None self._tap_by_port.clear() self._kernel_ports.clear() + self._cascade_peers.clear() self._started = False await self._stop_ubridge() @@ -433,6 +450,29 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): raise NodeError(f"Port {port_number} doesn't exist on Ethernet switch '{self.name}'") anchor = nio.anchor + if nio.peer is not None: + # Cascade: this port is one end of a link-owned veth pair. Both + # sides create the pair when their end is missing — the two + # uBridge processes may race (the project-open batch even wires + # different nodes concurrently), one create wins and the other's + # failure is re-checked against the kernel: an existing pair + # means the race was lost, anything else is a real failure the + # link must not paper over. A pre-existing end (this switch + # restarted, or a leftover from a previous life of this link id + # — the names are a function of it) is reused as-is: the other + # end may still be a live port of the peer switch, and a stale + # tc qdisc is all a reused end can carry (VLAN entries die with + # the bridge that held them). + self._cascade_peers[port_number] = nio.peer + if not os.path.exists(f"/sys/class/net/{anchor}"): + try: + await self._ubridge_send(f'docker create_veth "{anchor}" "{nio.peer}"') + except UbridgeError: + if not os.path.exists(f"/sys/class/net/{anchor}"): + raise + else: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tc reset "{anchor}"') # Register before _kernel_attach: the marker reconcile recognises # kernel anchors through _kernel_anchors(). self._kernel_ports[port_number] = anchor @@ -459,10 +499,15 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): """ Releases an absorbed anchor: detaches the link state (markers, impairments, carrier) and takes the anchor out of the switch bridge. - The anchor itself belongs to the peer and survives. + The anchor itself belongs to the peer and survives. A cascade end + dies with the link instead: deleting one veth end destroys the pair + (and the peer end with it), so either side's teardown converges on + the same gone pair — the other side's delif then answers ENOENT, + suppressed. """ anchor = self._kernel_ports.pop(port_number, None) + peer = self._cascade_peers.pop(port_number, None) if anchor is None: return await self._remove_kernel_markers(anchor) @@ -473,6 +518,9 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): await self._ubridge_send(f'brctl delif "{self._bridge_name}" "{anchor}"') with contextlib.suppress(UbridgeError): await self._ubridge_send(f'link set "{anchor}" down') + if peer is not None: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'docker delete_veth "{anchor}"') async def _delete_ubridge_connection(self, port_number): """ diff --git a/gns3server/compute/nios/nio_anchor.py b/gns3server/compute/nios/nio_anchor.py index cecfcc6f8..35174dfe9 100644 --- a/gns3server/compute/nios/nio_anchor.py +++ b/gns3server/compute/nios/nio_anchor.py @@ -40,12 +40,22 @@ class NIOAnchor(NIO): Anchor-absorbing NIO. :param anchor: host interface to join to the node's own bridge + :param peer: for a switch-to-switch cascade link, the *other* end of the + veth pair this port is one end of. Both sides carry it: each creates + the pair when its end is missing (``docker create_veth + ``, the race between the two uBridge processes lost benignly + by whoever's create answers "exists"), and either side's teardown + may destroy it — deleting one veth end takes the whole pair, so the + deletions converge. Absent on absorbed-anchor links, where the + interface belongs to the peer's node and is never this switch's to + create or destroy. """ - def __init__(self, anchor): + def __init__(self, anchor, peer=None): super().__init__() self._anchor = anchor + self._peer = peer @property def anchor(self): @@ -57,6 +67,17 @@ class NIOAnchor(NIO): return self._anchor + @property + def peer(self): + """ + Returns the other end of the cascade veth pair, or None when the + anchor is a foreign interface the switch merely absorbs. + + :returns: interface name or None + """ + + return self._peer + @property def bridge(self): """ @@ -75,10 +96,13 @@ class NIOAnchor(NIO): def asdict(self): - return { + data = { "type": "nio_anchor", "anchor": self._anchor, "suspend": self._suspended, "filters": self._filters, "markers": self._markers, } + if self._peer is not None: + data["peer"] = self._peer + return data diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index bcca97f89..013b7e509 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -207,9 +207,10 @@ class Link: anchors into a per-link Linux bridge (``nio_bridge``), or an Ethernet switch absorbs the peer's anchor into its own kernel bridge (``nio_anchor`` on the switch end, ``nio_bridge`` with no - bridge on the peer end). Impairment filters run as tc netem on the - anchors; markers and capture are served by uBridge's AF_PACKET - module. + bridge on the peer end), or two switches are cascaded through a + veth pair (``nio_anchor`` on both ends). Impairment filters run as + tc netem on the anchors; markers and capture are served by + uBridge's AF_PACKET module. """ return any(d.get("type") in ("nio_bridge", "nio_anchor") for d in (getattr(self, "_link_data", None) or [])) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 4ed74b3e8..cc8e3b356 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -21,7 +21,7 @@ import logging from gns3server.config import Config from gns3server.utils.application_id import is_iol_runner_environment -from gns3server.utils.kernel_anchor import kernel_anchor_name +from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_cascade_names from gns3server.utils.packet_filter_validation import ( KERNEL_UNSUPPORTED_FILTERS, split_kernel_only_features, @@ -121,9 +121,15 @@ class UDPLink(Link): # Ethernet switch fast path: the link's tc impairments apply on # the absorbed anchor, which is a single interface — exactly one # end may own them, and that end is the switch (it owns the - # anchor's bridge membership and port VLANs). + # anchor's bridge membership and port VLANs). A cascade link is + # two interfaces (one veth end per switch), so it takes the + # normal two-sided treatment like any other kernel link. switch = self._switch_endpoint() - if switch is not None: + both_switches = ( + self._nodes[0]["node"].node_type == "ethernet_switch" + and self._nodes[1]["node"].node_type == "ethernet_switch" + ) + if switch is not None and not both_switches: return (self._filters, {}) if switch[0] == 0 else ({}, self._filters) return self._filters, self._filters # Relay datapath: the netem-extension filters (rate, reorder, gemodel…) @@ -217,14 +223,14 @@ class UDPLink(Link): # Ethernet switch fast path: the switch absorbs the peer's anchor # into its own kernel bridge, so the switch side needs no anchoring # capability of its own (its bridge and brctl already exist) — only - # the peer must be able to anchor. Two switches never qualify: a - # kernel interface belongs to exactly one bridge, so a cascaded - # link would need a veth pair between the two bridges and stays on - # the relay for now. + # the peer must be able to anchor. Two switches cascade instead: a + # veth pair joins the two kernel bridges, each side enslaving its + # own end (a kernel interface belongs to exactly one bridge, so the + # pair is the only shape a bridge-to-bridge cable can take). switch = self._switch_endpoint() if switch is not None: if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch": - return False + return True peer = node2 if switch[0] == 0 else node1 return self._kernel_endpoint_ready(peer) return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2) @@ -292,6 +298,47 @@ class UDPLink(Link): node1_markers, node2_markers = self._get_node_markers(node1, node2) switch = self._switch_endpoint() if switch is not None: + if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch": + # Switch-to-switch cascade: one veth pair joins the two + # kernel bridges, each side enslaving its own end into + # its own bridge and applying its own port mode to it — + # the kernel-native shape of a cable between two + # bridges (an interface belongs to exactly one bridge, + # so the pair is the only way to express it). The names + # derive from the link id (utils.kernel_anchor) and both + # sides carry the other end's name: each creates the + # pair when its end is missing (the two uBridge + # processes may race — one create wins, the other's + # failure is re-checked against the kernel), and either + # side's teardown may destroy it (deleting one veth end + # takes the whole pair), so no ordering between the two + # NIO posts is needed. Filters are two-sided (each end + # is its own interface, so each direction is impaired + # exactly once, like any other kernel link — not the + # single-sided absorbed-anchor case). + end0, end1 = kernel_cascade_names(self._id) + self._link_data = [ + { + "type": "nio_anchor", + "anchor": end0, + "peer": end1, + "filters": node1_filters, + "markers": node1_markers, + "suspend": self._suspended, + }, + { + "type": "nio_anchor", + "anchor": end1, + "peer": end0, + "filters": node2_filters, + "markers": node2_markers, + "suspend": self._suspended, + }, + ] + return [ + (node1, adapter_number1, port_number1, self._link_data[0]), + (node2, adapter_number2, port_number2, self._link_data[1]), + ] # Ethernet switch fast path: the switch absorbs the peer's # anchor into its own kernel bridge — no per-link bridge at # all, so the peer end carries no bridge name (its anchor is @@ -430,8 +477,12 @@ class UDPLink(Link): # The two ends are independent once the ports and peer addresses are # known — each node talks to its own compute/uBridge with no shared - # lock between them — so the two POSTs overlap. If either fails, roll - # back whichever side succeeded before re-raising the first error. + # lock between them — so the two POSTs overlap. That includes a + # cascade link: both sides create the veth pair when their end is + # missing (a race one side is designated to lose benignly), so + # neither post depends on the other's completion. If either fails, + # roll back whichever side succeeded before re-raising the first + # error. results = await asyncio.gather( node1.post(f"/adapters/{adapter_number1}/ports/{port_number1}/nio", data=nio_data1, timeout=120), node2.post(f"/adapters/{adapter_number2}/ports/{port_number2}/nio", data=nio_data2, timeout=120), @@ -677,19 +728,28 @@ class UDPLink(Link): switch = self._switch_endpoint() if switch is None or not self.kernel_datapath: return - index, switch_node = switch - adapter_number = self._nodes[index]["adapter_number"] - port_number = self._nodes[index]["port_number"] - try: - await switch_node.put( - f"/adapters/{adapter_number}/ports/{port_number}/nio", - data=self._link_data[index], - timeout=120, - ) - except (ComputeError, ControllerError): - # The link keeps its stored NIO; the next node start or link - # update retries. A failure here must not fail the node start. - log.warning("Could not re-push switch NIO for link %s after node start", self.id) + both_switches = ( + self._nodes[0]["node"].node_type == "ethernet_switch" + and self._nodes[1]["node"].node_type == "ethernet_switch" + ) + # A cascade link has a switch on both ends and each end is its own + # veth half: either switch starting (recreating its bridge) can + # leave its own half unjoined, so both ends are re-pushed. + indices = [0, 1] if both_switches else [switch[0]] + for index in indices: + switch_node = self._nodes[index]["node"] + adapter_number = self._nodes[index]["adapter_number"] + port_number = self._nodes[index]["port_number"] + try: + await switch_node.put( + f"/adapters/{adapter_number}/ports/{port_number}/nio", + data=self._link_data[index], + timeout=120, + ) + except (ComputeError, ControllerError): + # The link keeps its stored NIO; the next node start or link + # update retries. A failure here must not fail the node start. + log.warning("Could not re-push switch NIO for link %s after node start", self.id) async def start_marker( self, diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index 842682e7c..d3be033e1 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -120,6 +120,11 @@ class AnchorNIO(BaseModel): type: AnchorNIOType anchor: str = Field(..., description="Host interface to absorb e.g. gv00010203e0p0") + peer: Optional[str] = Field( + None, + description="Switch-to-switch cascade: the other end of the veth pair this port is one end of. Present on " + "exactly one side (the owner, which creates and destroys the pair); absent for an absorbed anchor", + ) suspend: Optional[bool] = Field(None, description="Suspend the NIO") filters: Optional[dict] = Field( None, diff --git a/gns3server/utils/kernel_anchor.py b/gns3server/utils/kernel_anchor.py index 02e380c74..556694097 100644 --- a/gns3server/utils/kernel_anchor.py +++ b/gns3server/utils/kernel_anchor.py @@ -66,3 +66,19 @@ def kernel_anchor_name(node_type, node_id, adapter_number, port_number=0): if prefix is None: return None return prefix + anchor_suffix(node_id, adapter_number, port_number) + + +def kernel_cascade_names(link_id): + """ + The two veth ends of a switch-to-switch cascade link, as + ``(owner end, peer end)``. The pair belongs to the *link* (not to either + switch): both names are a pure function of the link id, so each side + computes its own end and the controller can hand each switch exactly + the name to enslave — the same deterministic-name trick per-link + bridges use. ``gs`` keeps the cascade ends out of the node-anchor and + ``gns3`` name spaces; 10 hex chars of the link id plus the side index + stay within IFNAMSIZ (15). + """ + + stem = str(link_id).replace("-", "")[:10] + return f"gs{stem}0", f"gs{stem}1" diff --git a/tests/api/routes/compute/test_ethernet_switch_nodes.py b/tests/api/routes/compute/test_ethernet_switch_nodes.py index ad5521488..215c1d9ad 100644 --- a/tests/api/routes/compute/test_ethernet_switch_nodes.py +++ b/tests/api/routes/compute/test_ethernet_switch_nodes.py @@ -821,6 +821,174 @@ class TestEthernetSwitchNodesRoutes: node = compute_project.get_node(ethernet_switch["node_id"]) assert node._kernel_ports[3] == anchor + # ------------------------------------------------------------------ # + # switch-to-switch cascade: one veth pair joins the two bridges + # ------------------------------------------------------------------ # + + @staticmethod + def _cascade_params(anchor: str, peer: str) -> dict: + return {"type": "nio_anchor", "anchor": anchor, "peer": peer} + + def _cascade_sysfs(self, anchor: str, exists: dict): + """An ``os.path.exists`` whose answer for *one* interface flips with + the ``exists`` flag — for the create-race test, where the pair comes + into being between the pre-check and the failed create.""" + real = os.path.exists + + def fake(path): + value = str(path) + if value == f"/sys/class/net/{anchor}": + return exists["value"] + if "/brif/" in value: + return True + if value.startswith("/sys/class/net/"): + return True + return real(value) + + return patch("os.path.exists", new=fake) + + async def test_ethernet_switch_create_cascade_nio_creates_the_pair( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + A cascade port is one end of a link-owned veth pair: the switch + creates the pair (uBridge's docker create_veth — the same primitive + every Docker adapter is born from) and enslaves its own end into its + bridge with the port's VLAN mode, exactly like an absorbed anchor. + """ + + anchor, peer = "gs1a2b3c4d5e0", "gs1a2b3c4d5e1" + node = compute_project.get_node(ethernet_switch["node_id"]) + exists = {"value": False} + + async def create_the_pair(command): + if "create_veth" in str(command): + exists["value"] = True + + node._ubridge_send.side_effect = create_the_pair + + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._cascade_sysfs(anchor, exists): + response = await compute_client.post(url, json=self._cascade_params(anchor, peer)) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["peer"] == peer + + node = compute_project.get_node(ethernet_switch["node_id"]) + br = node._bridge_name + node._ubridge_send.assert_any_call(f'docker create_veth "{anchor}" "{peer}"') + node._ubridge_send.assert_any_call(f'brctl addif "{br}" "{anchor}"') + node._ubridge_send.assert_any_call(f'brctl vlan_add "{br}" "{anchor}" 1 pvid untagged') + node._ubridge_send.assert_any_call(f'link set "{anchor}" up') + assert node._kernel_ports[0] == anchor + assert node._cascade_peers[0] == peer + + async def test_ethernet_switch_create_cascade_nio_reuses_an_existing_pair( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + An existing end (this switch restarted, or a leftover from a previous + life of this link id) is reused as-is: the other end may still be a + live port of the peer switch, so the pair is never destroyed here — + a stale tc qdisc is all a reused end can carry, and the reset clears + it before the join. + """ + + anchor, peer = "gs1a2b3c4d5e0", "gs1a2b3c4d5e1" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._cascade_sysfs(anchor, {"value": True}): + response = await compute_client.post(url, json=self._cascade_params(anchor, peer)) + assert response.status_code == status.HTTP_201_CREATED + + node = compute_project.get_node(ethernet_switch["node_id"]) + br = node._bridge_name + node._ubridge_send.assert_any_call(f'tc reset "{anchor}"') + node._ubridge_send.assert_any_call(f'brctl addif "{br}" "{anchor}"') + assert not any("create_veth" in str(c) for c in node._ubridge_send.call_args_list) + + async def test_ethernet_switch_create_cascade_nio_lost_create_race( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + Both cascade sides create the pair when their end is missing, so the + two uBridge processes can race: the loser's create fails with + "exists", the kernel re-check shows the pair is there, and the join + proceeds — a lost race is not an error. + """ + + from gns3server.compute.ubridge.ubridge_error import UbridgeError + + anchor, peer = "gs1a2b3c4d5e0", "gs1a2b3c4d5e1" + node = compute_project.get_node(ethernet_switch["node_id"]) + exists = {"value": False} + + async def lose_the_race(command): + if "create_veth" in str(command): + # the peer switch's create won between our pre-check and now + exists["value"] = True + raise UbridgeError("202-interface already exists") + + node._ubridge_send.side_effect = lose_the_race + + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._cascade_sysfs(anchor, exists): + response = await compute_client.post(url, json=self._cascade_params(anchor, peer)) + assert response.status_code == status.HTTP_201_CREATED + + br = node._bridge_name + node._ubridge_send.assert_any_call(f'brctl addif "{br}" "{anchor}"') + assert node._kernel_ports[0] == anchor + + async def test_ethernet_switch_delete_cascade_nio_destroys_the_pair( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + The cascade pair dies with the link: the teardown detaches the link + state, delifs the end, and destroys the pair (deleting one veth end + takes the peer end with it — either side's teardown converges on the + same gone pair). + """ + + anchor, peer = "gs1a2b3c4d5e0", "gs1a2b3c4d5e1" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._cascade_sysfs(anchor, {"value": True}): + await compute_client.post(url, json=self._cascade_params(anchor, peer)) + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + response = await compute_client.delete(url) + assert response.status_code == status.HTTP_204_NO_CONTENT + + br = node._bridge_name + node._ubridge_send.assert_any_call(f'brctl delif "{br}" "{anchor}"') + node._ubridge_send.assert_any_call(f'docker delete_veth "{anchor}"') + assert node._kernel_ports == {} + assert node._cascade_peers == {} + # ------------------------------------------------------------------ # # in-place VLAN reconfiguration (no port ever leaves the bridge) # ------------------------------------------------------------------ # diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 9087fe720..834172dde 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -53,19 +53,30 @@ async def _kernel_link(project, **kwargs): def _mock_compute_http(compute): - """Wire the async compute HTTP callbacks the relay _prepare path needs.""" + """Wire the async compute HTTP callbacks the relay paths need: POST + (port reservation, NIO creation) and PUT (filter updates — without an + awaitable put, a relay filter update blows up on the sync MagicMock, + which is what made this file fail when run alone: some other module's + fixtures happened to mask it in a full run).""" async def subnet_callback(other): return ("192.168.1.1", "192.168.1.2") compute.get_ip_on_same_subnet.side_effect = subnet_callback - async def port_callback(path, data=None, **kwargs): + async def post_callback(path, data=None, **kwargs): response = MagicMock() response.json = {"udp_port": 1024} return response - compute.post.side_effect = port_callback + compute.post.side_effect = post_callback + + async def put_callback(path, data=None, **kwargs): + response = MagicMock() + response.json = {} + return response + + compute.put.side_effect = put_callback async def _relay_link(project): @@ -855,9 +866,10 @@ async def test_switch_links_select_the_kernel_fast_path(project): """ A link with an Ethernet switch endpoint rides the kernel fast path when the *peer* can anchor — the switch side needs no anchoring capability - (its bridge exists by construction). Switch-to-switch links never - qualify (a kernel interface belongs to exactly one bridge), nor do - anchor-less peers. + (its bridge exists by construction). A switch peer cascades through a + veth pair (one end per bridge — a kernel interface belongs to exactly + one bridge, so the pair is the only cable shape); anchor-less peers + stay on the relay. """ compute = MagicMock() @@ -874,10 +886,10 @@ async def test_switch_links_select_the_kernel_fast_path(project): link = await _link(project, switch, peer) assert link._kernel_datapath_eligible(switch, peer) is True, peer.node_type - # a switch peer: cascades need a veth pair between two bridges — relay + # a switch peer: cascade — the veth pair joins the two kernel bridges switch2 = _node(project, compute, "sw2", node_type="ethernet_switch") link = await _link(project, switch, switch2) - assert link._kernel_datapath_eligible(switch, switch2) is False + assert link._kernel_datapath_eligible(switch, switch2) is True # an anchor-less peer vpcs = _node(project, compute, "pc1", node_type="vpcs") @@ -969,3 +981,78 @@ async def test_switch_link_repushes_the_switch_nio_on_node_start(project): relay_switch.put = AsyncioMagicMock() await relay_link.node_started(vpcs) assert not relay_switch.put.called + + +# --------------------------------------------------------------------------- +# Switch-to-switch cascade: one veth pair joins the two kernel bridges +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_prepare_cascade_link_emits_paired_anchor_nios(project): + """ + A link between two Ethernet switches rides the kernel datapath through + one veth pair: each side gets an anchor NIO naming its own end and + carrying the other end's name (both sides create the pair when their + end is missing, so no ordering between the two NIO posts is needed). + Unlike the absorbed-anchor fast path, filters are two-sided — each end + is its own interface, so each direction of the link is impaired exactly + once like any other kernel link. + """ + + from gns3server.utils.kernel_anchor import kernel_cascade_names + + compute = MagicMock() + compute.id = "compute-1" + sw1 = _node(project, compute, "sw1", node_type="ethernet_switch") + sw2 = _node(project, compute, "sw2", node_type="ethernet_switch") + link = await _link(project, sw1, sw2) + link._filters = {"delay": [10]} + + entries = await link._prepare() + end0, end1 = kernel_cascade_names(link.id) + + assert entries[0][3] == { + "type": "nio_anchor", + "anchor": end0, + "peer": end1, + "filters": {"delay": [10]}, + "markers": {}, + "suspend": False, + } + assert entries[1][3] == { + "type": "nio_anchor", + "anchor": end1, + "peer": end0, + "filters": {"delay": [10]}, + "markers": {}, + "suspend": False, + } + assert link.kernel_datapath is True + + +@pytest.mark.asyncio +async def test_node_started_repushes_both_cascade_ends(project): + """ + A cascade link has a switch on both ends and each end is its own veth + half: whichever switch starts (recreating its bridge and losing its + half's membership), both ends are re-pushed — each switch's update + re-checks and re-joins its own half. + """ + + from tests.utils import AsyncioMagicMock + + compute = MagicMock() + compute.id = "compute-1" + sw1 = _node(project, compute, "sw1", node_type="ethernet_switch") + sw2 = _node(project, compute, "sw2", node_type="ethernet_switch") + link = await _link(project, sw1, sw2) + entries = await link._prepare() + link._created = True + sw1.put = AsyncioMagicMock() + sw2.put = AsyncioMagicMock() + + await link.node_started(sw1) + + sw1.put.assert_called_once_with("/adapters/0/ports/0/nio", data=entries[0][3], timeout=120) + sw2.put.assert_called_once_with("/adapters/0/ports/0/nio", data=entries[1][3], timeout=120) diff --git a/tests/utils/test_kernel_anchor.py b/tests/utils/test_kernel_anchor.py index e5d091ec2..7b88ffd9c 100644 --- a/tests/utils/test_kernel_anchor.py +++ b/tests/utils/test_kernel_anchor.py @@ -17,12 +17,14 @@ """ The kernel-datapath anchor naming contract (shared by the compute classes that create anchors and the controller that names a peer's anchor for an -Ethernet-switch link). +Ethernet-switch link), plus the cascade veth naming for switch-to-switch +links. """ -from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name +from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name, kernel_cascade_names NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" +LINK_ID = "1a2b3c4d-5e6f-4a5b-8c9d-0e1f2a3b4c5d" def test_every_node_type_has_its_own_prefix(): @@ -45,3 +47,22 @@ def test_names_fit_ifnamsiz(): def test_suffix_is_stable_across_the_node_type(): assert anchor_suffix(NODE_ID, 1, 2) == "00010203e1p2" assert kernel_anchor_name("docker", NODE_ID, 1, 2) == "gv" + anchor_suffix(NODE_ID, 1, 2) + + +def test_cascade_names_are_a_pair_of_distinct_sides(): + end0, end1 = kernel_cascade_names(LINK_ID) + assert end0 == "gs1a2b3c4d5e0" + assert end1 == "gs1a2b3c4d5e1" + assert end0 != end1 + + +def test_cascade_names_fit_ifnamsiz(): + for end in kernel_cascade_names(LINK_ID): + assert len(end) <= 15 + + +def test_cascade_names_stay_out_of_the_node_anchor_namespace(): + # "gs" cannot collide with any node anchor prefix (gv/gq/gi/gd), so a + # cascade end is never mistaken for another node's anchor + for end in kernel_cascade_names(LINK_ID): + assert not any(end.startswith(prefix) for prefix in ("gv", "gq", "gi", "gd")) From d6851695cc23779bc3008ab962946becfa5fcdb1 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 00:31:57 +0800 Subject: [PATCH 043/101] test: give the argument-less pytest run the e2e-excluding configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs the suite through `just test` -> `just unit`, i.e. `pytest` with no path argument and the repository root as rootdir — which does not load tests/pytest.ini, so its `addopts = -m "not e2e"` never applied there. The e2e tests were collected and executed on every CI run and only stayed green because a runner has no ~/GNS3/images directory: the harness skips itself. A local run on a machine that does have images went ahead and booted real servers for twelve minutes instead. Declare the same policy in pyproject.toml, the file that argument-less run does resolve, and add testpaths so the package-internal tests (gns3server/agent/.../tests/) stay collected — `pytest tests` never reached them either. That shape keeps resolving tests/pytest.ini and is unchanged. --- pyproject.toml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 696d785a8..65c67d1bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,23 @@ gns3server-uninstall-ai-features = "gns3server.utils.uninstall_ai_features:main" packages = ["gns3server"] +[tool.pytest.ini_options] +# The configuration a root-level, argument-less `pytest` (what CI runs via +# `just test` -> `just unit`) resolves to. It must match tests/pytest.ini's +# e2e policy: live end-to-end tests are excluded by marker, not left to the +# harness's images-directory skip. Without this, the root run collected the +# e2e tests and only stayed green because CI machines happen to have no +# images directory. testpaths also keeps the package-internal tests under +# gns3server/ collected (an argument-less run picks up everything; a +# `pytest tests` run never did). +testpaths = ["tests", "gns3server"] +addopts = ["-m", "not e2e"] +markers = [ + "e2e: live end-to-end tests (real server, kernel interfaces, emulators)", +] +asyncio_default_fixture_loop_scope = "function" + + [tool.ruff] target-version = "py310" line-length = 120 From c20125c1801ac714fa6c258cf1815c244790c0f0 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 13:30:24 +0800 Subject: [PATCH 044/101] feat(compute): probe uBridge's swappable bridge TAP leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bridge module gained delete_nio_tap (uBridge's gns3server-integration contract, section 'bridge — generic NIO relay with a swappable TAP leg'): a per-port bridge can swap its topology leg between UDP and a persistent TAP through stop -> delete -> add -> start, which is what lets an IOL runner container's port bridge carry a kernel-link anchor. The probe needs no scratch objects and no capabilities: one command against a bridge that cannot exist — 202 (unknown command) is an old build, 214 (bridge doesn't exist) is a new one. Cached by binary identity like every other probe; any failure reports unknown. GET /v3/compute/capabilities gains ubridge_bridge_tap; the controller's default capability dict carries it too (the single schema definition, per the 1e67fbf12 lesson). --- gns3server/api/routes/compute/capabilities.py | 13 ++- gns3server/compute/ubridge/tc_probe.py | 79 +++++++++++++++++++ gns3server/controller/compute.py | 1 + gns3server/schemas/controller/capabilities.py | 6 ++ tests/api/routes/compute/test_capabilities.py | 23 ++++++ tests/compute/ubridge/test_tc_probe.py | 58 +++++++++++++- tests/controller/test_compute.py | 2 +- 7 files changed, 179 insertions(+), 3 deletions(-) diff --git a/gns3server/api/routes/compute/capabilities.py b/gns3server/api/routes/compute/capabilities.py index df5e9d0e6..9e6a2d2bd 100644 --- a/gns3server/api/routes/compute/capabilities.py +++ b/gns3server/api/routes/compute/capabilities.py @@ -25,7 +25,12 @@ from fastapi import APIRouter, Request from gns3server import schemas from gns3server.compute import MODULES -from gns3server.compute.ubridge.tc_probe import probe_iol_tap_support, probe_tc_capabilities, probe_tap_support +from gns3server.compute.ubridge.tc_probe import ( + probe_bridge_tap_support, + probe_iol_tap_support, + probe_tc_capabilities, + probe_tap_support, +) from gns3server.utils.path import get_default_project_directory from gns3server.utils.tc_capabilities import usable_ebpf_modes from gns3server.version import __version__ @@ -60,6 +65,11 @@ async def get_capabilities(request: Request) -> dict: # Ethernet ports need for the kernel datapath. None = unknown. ubridge_iol_tap = await probe_iol_tap_support() + # Swappable TAP leg on the generic bridge module (bridge delete_nio_tap): + # what lets an IOL runner container's per-port bridge carry a kernel-link + # anchor. None = unknown (probe failed or uBridge too old). + ubridge_bridge_tap = await probe_bridge_tap_support() + # record the controller hostname or IP address if request.client: request.app.state.controller_host = request.client.host @@ -74,4 +84,5 @@ async def get_capabilities(request: Request) -> dict: "ubridge_tc": ubridge_tc, "ubridge_tap": ubridge_tap, "ubridge_iol_tap": ubridge_iol_tap, + "ubridge_bridge_tap": ubridge_bridge_tap, } diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py index 23ed916fb..7e5e19d1c 100644 --- a/gns3server/compute/ubridge/tc_probe.py +++ b/gns3server/compute/ubridge/tc_probe.py @@ -53,6 +53,8 @@ _cache = {} _tap_cache = {} # binary identity -> whether iol_bridge can bind a port to a TAP (None = unknown) _iol_tap_cache = {} +# binary identity -> whether the bridge module can release a named TAP NIO (None = unknown) +_bridge_tap_cache = {} _lock = asyncio.Lock() @@ -274,3 +276,80 @@ async def _probe_iol_tap(path, config, timeout): with contextlib.suppress(Exception): await hypervisor.stop() shutil.rmtree(working_dir, ignore_errors=True) + + +async def probe_bridge_tap_support(timeout: float = 15.0): + """ + Whether this host's uBridge has ``bridge delete_nio_tap`` — the command + that makes the bridge module's TAP leg swappable (stop -> delete -> add -> + start), which is what lets an IOL runner container's per-port bridge swap + its topology leg between UDP (relay) and a persistent TAP anchor (kernel + datapath). Probed with no scratch objects and no capabilities needed: + deleting from a bridge that doesn't exist answers 214 on a build with the + command and 202 (unknown command) on one without. Cached by binary + identity like the other probes; any failure means unknown, reported as + None and cached as such. + """ + + config = Config.instance() + path = shutil.which(config.settings.Server.ubridge_path) + if not path: + return None + try: + stat = os.stat(path) + key = (path, stat.st_mtime_ns, stat.st_size) + except OSError: + return None + + async with _lock: + if key in _bridge_tap_cache: + return _bridge_tap_cache[key] + supported = await _probe_bridge_tap(path, config, timeout) + _bridge_tap_cache[key] = supported + return supported + + +async def _probe_bridge_tap(path, config, timeout): + """ + Spawn a throwaway uBridge and issue the probe command against a bridge + name that cannot exist (uuid). The reply splits the builds: "bridge ... + doesn't exist" (214) means the command ran — this is the new build; + "Unknown command" (202) is the old one. Both arrive as UbridgeError (the + wrapper raises on any non-100 reply), so the message is the discriminator. + """ + + working_dir = tempfile.mkdtemp(prefix="gns3-brtap-probe-") + hypervisor = Hypervisor( + None, + path, + working_dir, + config.settings.Server.ubridge_control_transport, + config.settings.Server.host, + str(uuid.uuid4()), + ) + try: + + async def ask(): + await hypervisor.start() + await hypervisor.connect() + try: + await hypervisor.send(f"bridge delete_nio_tap gns3brtapprobe{uuid.uuid4().hex[:4]} any") + except UbridgeError as e: + if "doesn't exist" in str(e): + return True + # "Unknown command" (and anything else): old or broken build. + log.debug("uBridge bridge-tap probe answered: %s", e) + return None + # A 100 here would mean the bridge existed — impossible on this + # uuid name; treat it as unknown rather than supported. + return None + + try: + return await asyncio.wait_for(ask(), timeout=timeout) + except (OSError, asyncio.TimeoutError, ValueError) as e: + log.debug("uBridge bridge-tap probe failed: %s", e) + return None + finally: + with contextlib.suppress(Exception): + await hypervisor.stop() + shutil.rmtree(working_dir, ignore_errors=True) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 8162a5e9c..863e7a8c0 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -102,6 +102,7 @@ class Compute: "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None, + "ubridge_bridge_tap": None, } self.name = name # Cache of interfaces on remote host diff --git a/gns3server/schemas/controller/capabilities.py b/gns3server/schemas/controller/capabilities.py index 106df4b0f..d8aaf591b 100644 --- a/gns3server/schemas/controller/capabilities.py +++ b/gns3server/schemas/controller/capabilities.py @@ -61,3 +61,9 @@ class Capabilities(BaseModel): "the anchor IOU Ethernet ports need to wire IOU links on the kernel datapath); absent when the " "probe failed", ) + ubridge_bridge_tap: Optional[bool] = Field( + None, + description="Whether uBridge's generic bridge module can release a named TAP NIO " + "(bridge delete_nio_tap, the swappable TAP leg an IOL runner container's per-port bridge " + "needs to wire its links on the kernel datapath); absent when the probe failed", + ) diff --git a/tests/api/routes/compute/test_capabilities.py b/tests/api/routes/compute/test_capabilities.py index b196de34c..97d7948c5 100644 --- a/tests/api/routes/compute/test_capabilities.py +++ b/tests/api/routes/compute/test_capabilities.py @@ -43,6 +43,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -69,6 +70,7 @@ class TestCapabilitiesRoutes: "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None, + "ubridge_bridge_tap": None, } async def test_get_on_gns3vm(self, app: FastAPI, compute_client: AsyncClient, on_gns3vm, monkeypatch) -> None: @@ -79,6 +81,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -105,6 +108,7 @@ class TestCapabilitiesRoutes: "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None, + "ubridge_bridge_tap": None, } async def test_get_reports_ubridge_tc(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: @@ -115,6 +119,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", fake_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_tc"] == { @@ -135,6 +140,7 @@ class TestCapabilitiesRoutes: monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", fake_tap_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_tap"] is True @@ -153,3 +159,20 @@ class TestCapabilitiesRoutes: response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_iol_tap"] is True + + async def test_get_reports_ubridge_bridge_tap(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: + """ + The swappable-bridge-TAP-leg capability is what makes an IOL runner + container kernel-eligible, so it travels with the other compute + capabilities. + """ + + async def fake_bridge_tap_probe(): + return True + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", fake_bridge_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) + response = await compute_client.get(app.url_path_for("compute:get_capabilities")) + assert response.status_code == status.HTTP_200_OK + assert response.json()["ubridge_bridge_tap"] is True diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py index 96735ab8e..9408c70c0 100644 --- a/tests/compute/ubridge/test_tc_probe.py +++ b/tests/compute/ubridge/test_tc_probe.py @@ -22,7 +22,12 @@ payload: spawn a throwaway uBridge, ask once, cache by binary identity. import pytest from gns3server.compute.ubridge import tc_probe -from gns3server.compute.ubridge.tc_probe import probe_iol_tap_support, probe_tc_capabilities, probe_tap_support +from gns3server.compute.ubridge.tc_probe import ( + probe_bridge_tap_support, + probe_iol_tap_support, + probe_tc_capabilities, + probe_tap_support, +) from gns3server.compute.ubridge.ubridge_error import UbridgeError pytestmark = pytest.mark.asyncio @@ -36,6 +41,10 @@ class FakeHypervisor: reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] error = None error_prefix = None # fail only commands starting with this (a build missing one command) + # what `bridge delete_nio_tap` answers: a new build's 214 (the bridge + # doesn't exist), None = answer 100 (an impossible reply on the probe's + # uuid name — the probe must treat it as unknown) + bridge_tap_error = "214-bridge 'nosuch' doesn't exist" spawned = 0 def __init__(self, project, path, working_dir, transport, host, node_id): @@ -57,6 +66,10 @@ class FakeHypervisor: raise UbridgeError(self.error) if self.error_prefix and command.startswith(self.error_prefix): raise UbridgeError(f"202-Unknown command") + if command.startswith("bridge delete_nio_tap "): + if FakeHypervisor.bridge_tap_error: + raise UbridgeError(FakeHypervisor.bridge_tap_error) + return ["OK"] if command == "tc capabilities": return FakeHypervisor.reply return ["OK"] @@ -71,10 +84,12 @@ def probe_env(monkeypatch, tmp_path): FakeHypervisor.spawned = 0 FakeHypervisor.error = None FakeHypervisor.error_prefix = None + FakeHypervisor.bridge_tap_error = "214-bridge 'nosuch' doesn't exist" FakeHypervisor.reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] tc_probe._cache.clear() tc_probe._tap_cache.clear() tc_probe._iol_tap_cache.clear() + tc_probe._bridge_tap_cache.clear() FakeHypervisor.last = None # a real file on disk: the cache is keyed on its (path, mtime, size) binary = tmp_path / "ubridge" @@ -193,3 +208,44 @@ async def test_probe_iol_tap_support_missing_binary(monkeypatch): monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) assert await probe_iol_tap_support() is None assert FakeHypervisor.spawned == 0 + + +async def test_probe_bridge_tap_support_asks_one_unused_bridge(): + """ + The probe needs no scratch objects and no capabilities: one command + against a bridge that cannot exist, whose 214 answer means the command + ran (new build). Nothing is created, nothing needs cleaning up. + """ + + assert await probe_bridge_tap_support() is True + assert FakeHypervisor.spawned == 1 + commands = FakeHypervisor.last.commands + assert len(commands) == 1 + assert commands[0].startswith("bridge delete_nio_tap gns3brtapprobe") + # cached like the other probes: no respawn on every /capabilities hit + assert await probe_bridge_tap_support() is True + assert FakeHypervisor.spawned == 1 + + +async def test_probe_bridge_tap_support_unknown_on_an_old_build(): + """ + A uBridge without the command answers 202 "Unknown command": unknown, so + IOL runner containers stay on the relay datapath. A 100 answer is + unknown too — the bridge "existed" on the probe's uuid name, which + cannot happen on a sane build. + """ + + FakeHypervisor.error_prefix = "bridge delete_nio_tap" + assert await probe_bridge_tap_support() is None + + FakeHypervisor.error_prefix = None + FakeHypervisor.bridge_tap_error = None + tc_probe._bridge_tap_cache.clear() + assert await probe_bridge_tap_support() is None + + +async def test_probe_bridge_tap_support_missing_binary(monkeypatch): + + monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) + assert await probe_bridge_tap_support() is None + assert FakeHypervisor.spawned == 0 diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index ee48a143e..17b96b4ad 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -381,7 +381,7 @@ async def test_json(compute): "disk_usage_percent": 0, "connected": True, "last_error": None, - "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None}, + "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None, "ubridge_bridge_tap": None}, } assert compute.asdict(topology_dump=True) == { "compute_id": "my_compute_id", From 4b4a0fc35fcde909a3d573702301d425662ebe0f Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 13:30:35 +0800 Subject: [PATCH 045/101] feat(iol): anchor IOL runner containers on persistent TAPs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IOL runner containers (IOLDockerVM, GNS3_IOL_RUNNER) had no kernel datapath: their guest leg is the runner's AF_UNIX socket pairs, so every link rode the uBridge relay. Their Ethernet bays now own persistent TAP anchors (gx names, the iol_docker key of the shared naming contract — born at node start like IOU's, before any link, so the switch fast path's deferred join can never miss them), and the per-port bridge's topology leg swaps between UDP (relay) and the anchor (kernel link) through the frozen stop -> delete -> add -> start contract — the unix binding never re-binds across a swap. - iol_docker_vm: KernelDatapathMixin hooks, the anchor lifecycle (_prepare_tap_datapath at _start_ubridge: probe ubridge_bridge_tap + ubridge_tap, sweep leftover, tap create born down), the kernel attach (ensure-then-add — tap create is strictly create-only, IFF_TUN_EXCL, so the ensure checks /sys/class/net first: an unconditional create answers EBUSY on the device the start loop just made), the release (port bridge stop + delete_nio_tap before the shared teardown) and the stop order the fd holders dictate (port bridges first, then tap delete, then the per-link bridges — all before the hypervisor stops). - vendor_docker_vm: the port-bridge creation factored into _ensure_unix_port_bridge (idempotent on the registered name), keeping the NIOBridge rejection for the generic unix-socket containers. - docker_vm: the three _kernel_veths lookups go through the _kernel_host_ifc hook — behavior-identical for DockerVM, and it is what lets the subclass route without overriding the binding methods. - kernel_anchor: the iol_docker key (gx) and kernel_anchor_type, the controller-side mapping for naming an IOL runner's anchor on the switch fast path (the marker check stays inline: this module sits below everything and must not import the controller package). --- gns3server/compute/docker/docker_vm.py | 19 +- gns3server/compute/docker/iol_docker_vm.py | 290 +++++++++++++ gns3server/compute/docker/vendor_docker_vm.py | 37 +- gns3server/utils/kernel_anchor.py | 33 +- .../docker/test_iol_docker_kernel_datapath.py | 395 ++++++++++++++++++ tests/compute/docker/test_iol_docker_vm.py | 8 +- tests/utils/test_kernel_anchor.py | 22 +- 7 files changed, 773 insertions(+), 31 deletions(-) create mode 100644 tests/compute/docker/test_iol_docker_kernel_datapath.py diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 25948a344..9d123478c 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1807,16 +1807,17 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): async def _connect_nio(self, adapter_number, nio, port_number=0): - host_ifc = self._kernel_veths.get((adapter_number, port_number)) + host_ifc = self._kernel_host_ifc(adapter_number, port_number) if isinstance(nio, NIOBridge): if host_ifc is None: # Only reachable for containers started before the # unified-veth change (or unix-socket vendor containers, - # rejected earlier): the kernel path needs the veth host end. + # rejected earlier): the kernel path needs an anchor — the + # veth host end, or the persistent TAP of an IOL runner. raise DockerError( - "Adapter {adapter_number} port {port_number} of container '{name}' has no veth " - "interface (it was started before the unified-veth datapath); " + "Adapter {adapter_number} port {port_number} of container '{name}' has no host-side " + "anchor interface (it was started before the unified-veth datapath); " "restart the node to attach a kernel link to it".format( adapter_number=adapter_number, port_number=port_number, name=self.name ) @@ -1894,7 +1895,7 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): if self.ubridge: if isinstance(nio, NIOBridge): if self.status == "started": - host_ifc = self._kernel_veths.get((adapter_number, port_number)) + host_ifc = self._kernel_host_ifc(adapter_number, port_number) if host_ifc is not None: # Incremental marker + filter reconcile on the anchor # — the relay-datapath equivalent of the branch below. @@ -2008,9 +2009,11 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): nio = self._ethernet_adapters[adapter_number].get_nio(port_number) if isinstance(nio, NIOBridge): # Kernel datapath: no relay bridge exists — capture via uBridge's - # AF_PACKET module bound to the veth host end. Single capture per - # uBridge process: a concurrent second one returns EALREADY. - host_ifc = self._kernel_veths.get((adapter_number, port_number)) + # AF_PACKET module bound to the anchor (a veth host end for the + # standard container, a persistent TAP for an IOL runner). + # Single capture per uBridge process: a concurrent second one + # returns EALREADY. + host_ifc = self._kernel_host_ifc(adapter_number, port_number) await self._ubridge_send(f'capture start_kernel {host_ifc} "{output_file}"') return # Relay datapath (even though the adapter interface is a veth, the diff --git a/gns3server/compute/docker/iol_docker_vm.py b/gns3server/compute/docker/iol_docker_vm.py index b8e30ec36..212ef1390 100644 --- a/gns3server/compute/docker/iol_docker_vm.py +++ b/gns3server/compute/docker/iol_docker_vm.py @@ -35,6 +35,17 @@ linked nodes get distinct MACs; starting a node without an allocation is an error, not a fallback — an uncoordinated id could collide with the pool and blackhole traffic as a MAC loop. +Kernel datapath: the unix-socket pair is this container's only physical +layer, so one userspace hop on its leg is irreducible — but the *link +segment* need not ride the relay. Every Ethernet bay/unit owns a persistent +TAP anchor (``gx`` names, created at node start like IOU's), and the +per-port uBridge bridge swaps its topology leg between UDP (relay) and the +anchor (kernel link) through ``bridge add_nio_tap`` / ``bridge +delete_nio_tap`` — stop → delete → add → start, the c-socket binding +surviving the swap. Eligibility needs the compute's uBridge to report both +``ubridge_tap`` (the tap module) and ``ubridge_bridge_tap`` (the swappable +leg); anything less keeps the node relay-only, exactly as before. + This class is selected by the ``GNS3_IOL_RUNNER=1`` environment marker. """ @@ -51,6 +62,11 @@ from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Err from gns3server.compute.docker.vendor_docker_vm import VendorDockerVM from gns3server.compute.iou.utils.iou_export import nvram_export from gns3server.compute.iou.utils.iou_import import nvram_import +from gns3server.compute.kernel_datapath import KernelDatapathMixin +from gns3server.compute.nios.nio_bridge import NIOBridge +from gns3server.compute.ubridge.tc_probe import probe_bridge_tap_support, probe_tap_support +from gns3server.compute.ubridge.ubridge_error import UbridgeError +from gns3server.utils.kernel_anchor import kernel_anchor_name log = logging.getLogger(__name__) @@ -95,6 +111,15 @@ class IOLDockerVM(VendorDockerVM): _startup_config_content: str | None = None _startup_config_dirty = False + def __init__(self, *args, **kwargs): + + super().__init__(*args, **kwargs) + # Kernel datapath (see KernelDatapathMixin): one persistent TAP per + # Ethernet bay/unit, held by uBridge's port bridge while a kernel + # link binds the port (bridge add_nio_tap). + self._kernel_taps = {} + self._tap_datapath = False + def _parse_vendor_environment(self): super()._parse_vendor_environment() @@ -383,3 +408,268 @@ class IOLDockerVM(VendorDockerVM): """ self._permissions_fixed = True + + # ------------------------------------------------------------------ + # Kernel datapath (see KernelDatapathMixin) + # ------------------------------------------------------------------ + + def _kernel_host_ifc(self, adapter_number, port_number=0): + """ + The persistent TAP this Ethernet bay/unit owns (one 4-port unit per + adapter, the IOU model), or None before the node started — or on a + uBridge that cannot serve the anchor lifecycle, in which case the + node runs relay-only. + """ + + return self._kernel_taps.get((adapter_number, port_number)) + + def _kernel_anchors(self): + """ + Every anchor TAP this node currently owns. + """ + + return set(self._kernel_taps.values()) + + def _kernel_error(self, message): + return DockerError(message) + + def _tap_name(self, adapter_number, port_number): + """ + Deterministic anchor TAP name for an Ethernet bay/unit — the shared + utils.kernel_anchor naming contract under the ``iol_docker`` key, so + the controller names a peer's anchor with the same function when an + Ethernet switch absorbs it. + """ + + return kernel_anchor_name("iol_docker", self._id, adapter_number, port_number) + + async def _start_ubridge(self, require_privileged_access=False): + """ + Override: with the control channel up, give every Ethernet bay/unit + its persistent TAP anchor — before the first link can attach (the + start loop calls _add_ubridge_connection right after this). The + anchors exist whether or not any link ever uses them: an anchor born + with the link would leave the switch fast path's deferred join + waiting forever (a link to a stopped node must not fail — and one to + a just-started node must not silently miss its anchor either). + """ + + await super()._start_ubridge(require_privileged_access=require_privileged_access) + await self._prepare_tap_datapath() + + async def _prepare_tap_datapath(self): + """ + Probe uBridge for the commands the anchor lifecycle needs — the tap + module (``tap create``/``tap delete``, the ``ubridge_tap`` + capability) and the bridge module's swappable TAP leg + (``bridge delete_nio_tap``, ``ubridge_bridge_tap``) — and when both + answer, create the persistent TAP every Ethernet bay/unit owns. + A uBridge missing either keeps this node on the relay datapath: + every link rides unix ↔ UDP through the port bridges, exactly as + before the kernel datapath existed. + """ + + self._tap_datapath = False + self._kernel_taps.clear() + + if await probe_bridge_tap_support() is not True: + log.info( + "IOL container '%s': this compute's uBridge cannot swap a bridge's TAP leg " + "(bridge delete_nio_tap); the node runs on the relay datapath and cannot " + "carry kernel links", + self._name, + ) + return + if await probe_tap_support() is not True: + log.info( + "IOL container '%s': this compute's uBridge has no persistent-TAP module; " + "the node runs on the relay datapath and cannot carry kernel links", + self._name, + ) + return + + self._tap_datapath = True + for adapter_number in range(0, len(self._ethernet_adapters)): + for port_number in range(0, self._ethernet_adapters[adapter_number].interfaces): + tap = self._tap_name(adapter_number, port_number) + # A persistent TAP outlives a crash: sweep any leftover of a + # previous run before recreating it (a surviving device would + # also keep its stale tc qdisc). + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._ubridge_send(f'tap create "{tap}"') + # Carrier off until a link attaches (the anchor contract). + await self._ubridge_send(f'link set "{tap}" down') + self._kernel_taps[(adapter_number, port_number)] = tap + log.debug( + "IOL container '%s': anchor TAP %s created for bay %d unit %d", + self._name, + tap, + adapter_number, + port_number, + ) + + async def _ensure_anchor(self, adapter_number, port_number): + """ + The ensure half of the frozen ensure-then-add contract: ``bridge + add_nio_tap`` deliberately opens a *transient* TAP when the name is + absent (cloud's bridge-interface path depends on it), so a persistent + anchor swept away between the node's start and the link's attach + would silently become a device that dies with the fd. ``tap create`` + is strictly create-only (``IFF_TUN_EXCL`` — re-creating an existing + persistent device answers EBUSY), so the existence check is local: + uBridge runs on this host, and /sys/class/net names every device. + The repair create starts DOWN like every anchor at birth, so the + carrier pass in the attach flow is what brings it up. + """ + + tap = self._tap_name(adapter_number, port_number) + if os.path.exists(os.path.join("/sys/class/net", tap)): + return + await self._ubridge_send(f'tap create "{tap}"') + + async def _add_ubridge_connection(self, nio, adapter_number, port_number=0): + """ + Override: the vendor rejection of kernel-datapath NIOs does not + apply here — IOL runner adapters own persistent TAP anchors, so a + NIOBridge wires the port bridge's topology leg to the anchor instead + of raising (_connect_nio routes it; the port bridge and its unix NIO + are ensured inside). + """ + + if isinstance(nio, NIOBridge): + await self._connect_nio(adapter_number, nio, port_number) + return + await super()._add_ubridge_connection(nio, adapter_number, port_number) + + async def _connect_nio(self, adapter_number, nio, port_number=0): + """ + Override: route the NIO onto the port bridge's topology leg. A + kernel link (NIOBridge) swaps that leg to the port's anchor TAP and + runs the shared mixin flow on it; a relay NIO keeps the unix ↔ UDP + shape of the base class (the port bridge and its unix NIO ensured + first, so the base path only adds the UDP half). + """ + + if isinstance(nio, NIOBridge): + await self._attach_kernel_link(adapter_number, port_number, nio) + return + + await self._ensure_unix_port_bridge(adapter_number, port_number) + await super()._connect_nio(adapter_number, nio, port_number) + + async def _attach_kernel_link(self, adapter_number, port_number, nio): + """ + Wire a kernel link's NIO (NIOBridge) on this port: the port bridge's + topology leg becomes the anchor TAP (``bridge add_nio_tap`` — uBridge + opens and holds the fd, relaying the unix-socket guest leg onto the + device), then the anchor is enslaved into the per-link kernel bridge + — the shared mixin flow, with the TAP playing the role Docker's veth + and QEMU's TAP play. The relay starts only once both NIOs are in + (uBridge requires two), which is also what carries the guest frames + onto the anchor. + """ + + anchor = self._kernel_host_ifc(adapter_number, port_number) + if anchor is None: + raise DockerError( + "Bay {bay}/{unit} of IOL container '{name}' has no TAP anchor to carry a kernel " + "link (this compute's uBridge lacks the tap module or bridge delete_nio_tap, " + "or the node was started before the kernel-datapath support); restart the " + "node after upgrading uBridge".format(bay=adapter_number, unit=port_number, name=self._name) + ) + await self._ensure_unix_port_bridge(adapter_number, port_number) + await self._ensure_anchor(adapter_number, port_number) + bridge_name = self._bridge_name(adapter_number, port_number) + await self._ubridge_send(f'bridge add_nio_tap {bridge_name} "{anchor}"') + # Per-link kernel bridge (brctl), capture, markers and impairment + # filters on the anchor — everything keyed on the interface name. + await self._kernel_attach(anchor, nio) + # Two NIOs now: start the [unix ↔ tap] relay. + await self._ubridge_send(f"bridge start {bridge_name}") + # The carrier pass refines the anchor's admin state (a suspended NIO + # sets it back down) — it is also what brings a born-down anchor up. + await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) + + async def _release_port_tap(self, adapter_number, port_number): + """ + The teardown half of the swap contract: stop the port bridge (its + relay threads hold the NIO pointers for their whole life — freeing a + NIO under them is a use-after-free, which is why delete_nio_tap + refuses while running), then release the TAP NIO by name. The anchor + device itself survives (the node owns it, not the link); the c-socket + binding survives too (stop keeps every NIO), so the container's + egress frames queue on the socket until a leg is swapped in again. + """ + + bridge_name = self._bridge_name(adapter_number, port_number) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f"bridge stop {bridge_name}") + tap = self._kernel_taps.get((adapter_number, port_number)) + if tap is not None: + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'bridge delete_nio_tap {bridge_name} "{tap}"') + + async def adapter_remove_nio_binding(self, adapter_number, port_number=0): + """ + Override: a kernel link leaves through the port-leg swap (stop + + delete_nio_tap) before the shared anchor-side teardown — markers, + tc reset, brctl delif and the per-link bridge deletion — runs in the + base path. + """ + + if self.ubridge: + try: + adapter = self._ethernet_adapters[adapter_number] + except IndexError: + adapter = None + if adapter is not None and isinstance(adapter.get_nio(port_number), NIOBridge): + await self._release_port_tap(adapter_number, port_number) + await super().adapter_remove_nio_binding(adapter_number, port_number) + + async def _stop_ubridge(self): + """ + Override: release the kernel datapath's host state while the control + channel is still up, in the order the fd holders dictate — the port + bridges hold the anchor TAP fds (``bridge add_nio_tap``), so they go + first (``bridge delete`` stops their relay threads and frees the + NIOs), then the anchors themselves (``tap delete`` answers EBADFD on + a device another fd still holds), then the per-link kernel bridges. + The port bridges die with the process anyway, but the anchors are + persistent: deleting them here is what keeps a stopped node from + littering the host with ``gx`` devices. The next start sweeps and + re-creates everything. + """ + + if self.ubridge: + # stop() clears _bridges before this point, so derive the port + # bridge names from the adapters (the historical naming is a + # pure function of adapter/port). + for adapter_number in range(0, len(self._ethernet_adapters)): + adapter = self._ethernet_adapters[adapter_number] + for port_number in range(0, adapter.interfaces): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f"bridge delete {self._bridge_name(adapter_number, port_number)}") + for tap in self._kernel_taps.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._remove_kernel_bridges() + self._kernel_taps.clear() + self._tap_datapath = False + self._ubridge_tc_caps = None + await super()._stop_ubridge() + + async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): + """ + Override: refine the vendor no-op. A port with an anchor (the kernel + datapath) toggles the anchor's admin state — the port bridge's TAP + NIO writes answer EIO and its reads fall silent while it is down, + which is the suspend semantics of every other anchored node type. A + port without an anchor (relay datapath on a uBridge without the + swappable leg) keeps the no-op: the unix-socket pair has no carrier, + and the port bridge carries no TAP NIO for the base class to toggle. + """ + + if self._kernel_host_ifc(adapter_number, port_number) is not None: + await KernelDatapathMixin._set_adapter_carrier(self, adapter_number, connected, port_number) + diff --git a/gns3server/compute/docker/vendor_docker_vm.py b/gns3server/compute/docker/vendor_docker_vm.py index c1fac493b..3daa935e5 100644 --- a/gns3server/compute/docker/vendor_docker_vm.py +++ b/gns3server/compute/docker/vendor_docker_vm.py @@ -395,7 +395,28 @@ class VendorDockerVM(DockerVM): Override: with GNS3_UNIX_SOCKET_NIO, bridge the adapter port through the image's AF_UNIX datagram socket pair (raw Ethernet frames) instead of a TAP interface moved into the container's network - namespace. + namespace — see ``_ensure_unix_port_bridge`` for the wiring. + """ + + if not self._unix_socket_nio: + return await super()._add_ubridge_connection(nio, adapter_number, port_number) + + if isinstance(nio, NIOBridge): + raise DockerError( + f"Container '{self._name}' bridges its adapters through AF_UNIX sockets; " + "kernel-datapath links are not supported for this container" + ) + + await self._ensure_unix_port_bridge(adapter_number, port_number) + + if nio: + await self._connect_nio(adapter_number, nio, port_number) + + async def _ensure_unix_port_bridge(self, adapter_number, port_number): + """ + The adapter port's uBridge bridge with its unix-socket NIO — the + guest leg every link topology rides on. Idempotent: an already + created bridge (this start wired it) is left alone. Ports are addressed flat across adapters: interface index = adapter number × ports-per-adapter + port number (single-port adapters @@ -413,15 +434,6 @@ class VendorDockerVM(DockerVM): uses. """ - if not self._unix_socket_nio: - return await super()._add_ubridge_connection(nio, adapter_number, port_number) - - if isinstance(nio, NIOBridge): - raise DockerError( - f"Container '{self._name}' bridges its adapters through AF_UNIX sockets; " - "kernel-datapath links are not supported for this container" - ) - try: adapter = self._ethernet_adapters[adapter_number] except IndexError: @@ -429,6 +441,8 @@ class VendorDockerVM(DockerVM): interface_number = adapter_number * adapter.interfaces + port_number bridge_name = self._bridge_name(adapter_number, port_number) + if bridge_name in self._bridges: + return try: await self._ubridge_send(f"bridge create {bridge_name}") self._bridges.add(bridge_name) @@ -470,9 +484,6 @@ class VendorDockerVM(DockerVM): remote_sock, ) - if nio: - await self._connect_nio(adapter_number, nio, port_number) - async def _set_adapter_carrier(self, adapter_number, connected, port_number=0): """ Override: with GNS3_UNIX_SOCKET_NIO the bridges carry only unix and diff --git a/gns3server/utils/kernel_anchor.py b/gns3server/utils/kernel_anchor.py index 556694097..9cf72b77e 100644 --- a/gns3server/utils/kernel_anchor.py +++ b/gns3server/utils/kernel_anchor.py @@ -20,10 +20,12 @@ The kernel-datapath anchor naming contract, in one place. Every node type whose adapters own a host-side interface names that anchor deterministically: ``g{node_id[:8]}e{adapter}p{port}`` — ``gv`` for a Docker veth host end, ``gq`` for a QEMU TAP, ``gi`` for an IOU port TAP, -``gd`` for a Dynamips port TAP. The prefixes keep anchors out of the -``gns3`` bridge/TAP name space and name-collision-free on a host; the 8 -hex chars of the node id plus adapter/port keep every name unique and -within IFNAMSIZ (15). +``gd`` for a Dynamips port TAP, ``gx`` for an IOL runner container's port +TAP (a Docker node whose guest leg is unix sockets, so its anchor is a TAP +like IOU's, not a veth). The prefixes keep anchors out of the ``gns3`` +bridge/TAP name space and name-collision-free on a host; the 8 hex chars of +the node id plus adapter/port keep every name unique and within IFNAMSIZ +(15). The compute classes create their anchors with these helpers (single source of truth), and so does the controller when a switch link has to name the @@ -40,12 +42,32 @@ log = logging.getLogger(__name__) # Docker veth guest end) is container-local and deliberately absent. ANCHOR_PREFIX = { "docker": "gv", + "iol_docker": "gx", "qemu": "gq", "iou": "gi", "dynamips": "gd", } +def kernel_anchor_type(node_type, environment=None): + """ + The anchor-type key a node of this type and environment names its + anchors under. IOL runner containers are Docker nodes (node_type + "docker") whose anchors are persistent TAPs, not veth host ends — their + own key keeps the ``gv`` veth namespace free of TAPs (stale-interface + sweeps and human eyes both key on the prefix). Every other node type is + its own key unchanged. + """ + + # The marker check mirrors utils.application_id.is_iol_runner_environment + # inline: this module sits below every node module (compute and + # controller alike import it during their own import), so it must not + # depend on anything that pulls the controller package in. + if node_type == "docker" and "GNS3_IOL_RUNNER=" in (environment or ""): + return "iol_docker" + return node_type + + def anchor_suffix(node_id, adapter_number, port_number=0): """ The shared ``{node_id[:8]}e{adapter}p{port}`` name stem (8 hex chars of @@ -59,7 +81,8 @@ def kernel_anchor_name(node_type, node_id, adapter_number, port_number=0): """ The host-side kernel-datapath anchor of one adapter port, or None for a node type without one (the anchor-less types never anchor a kernel - link). + link). Accepts either a raw node type or an anchor-type key from + :func:`kernel_anchor_type`. """ prefix = ANCHOR_PREFIX.get(node_type) diff --git a/tests/compute/docker/test_iol_docker_kernel_datapath.py b/tests/compute/docker/test_iol_docker_kernel_datapath.py new file mode 100644 index 000000000..8c7a4f7db --- /dev/null +++ b/tests/compute/docker/test_iol_docker_kernel_datapath.py @@ -0,0 +1,395 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software and is distributed under the same terms as gns3-server. +# See the LICENSE file for licensing information. +# + +""" +Kernel-datapath tests for IOLDockerVM: the persistent TAP anchors (one per +Ethernet bay/unit), the per-port bridge's swappable topology leg (unix ↔ UDP +relay vs unix ↔ TAP anchor) and the teardown order the fd holders dictate. + +Image-free: everything is asserted against the uBridge command stream and +the anchor naming contract. +""" + +import uuid + +import pytest +import pytest_asyncio + +from unittest.mock import MagicMock, patch + +from tests.utils import AsyncioMagicMock + +from gns3server.compute.docker import Docker +from gns3server.compute.docker.docker_error import DockerError +from gns3server.compute.docker.iol_docker_vm import IOLDockerVM + + +NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" +TAP00 = "gx00010203e0p0" # the vm fixture's node id, bay 0 unit 0 +TAP01 = "gx00010203e0p1" +TAP10 = "gx00010203e1p0" +PORT_BRIDGE = "bridge0" # adapter 0 port 0 (single-port naming) +BRIDGE = "gns3abc123def45" # the per-link kernel bridge carried by the NIO + + +@pytest_asyncio.fixture +async def manager(port_manager): + + m = Docker.instance() + m.port_manager = port_manager + return m + + +@pytest_asyncio.fixture(scope="function") +async def vm(compute_project, manager): + + vm = IOLDockerVM( + "iol-xe-1", + NODE_ID, + compute_project, + manager, + "iol-xe/iol-xe:17-18-02", + environment="GNS3_IOL_RUNNER=1", + adapters=2, + ) + vm._cid = "e90e34656842" + vm.application_id = 700 + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_hypervisor.is_running.return_value = True + vm._ubridge_send = AsyncioMagicMock() + return vm + + +def _capable(vm, monkeypatch): + """Make both anchor-lifecycle probes answer True.""" + + async def probe(timeout=15.0): + return True + + monkeypatch.setattr("gns3server.compute.docker.iol_docker_vm.probe_bridge_tap_support", probe) + monkeypatch.setattr("gns3server.compute.docker.iol_docker_vm.probe_tap_support", probe) + return vm + + +def _commands(vm): + return [str(c.args[0]) for c in vm._ubridge_send.call_args_list] + + +# --------------------------------------------------------------------------- +# Anchor lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_creates_one_tap_per_bay_unit(vm, monkeypatch): + """ + On a capable uBridge every Ethernet bay/unit gets its persistent TAP — + four units per adapter (the IOU model), swept of leftovers first and + born down. No set_owner: uBridge itself holds these fds. + """ + + _capable(vm, monkeypatch) + await vm._prepare_tap_datapath() + + assert vm._tap_datapath is True + assert set(vm._kernel_taps) == {(bay, unit) for bay in range(2) for unit in range(4)} + assert vm._kernel_taps[(0, 0)] == TAP00 + commands = _commands(vm) + assert f'tap delete "{TAP00}"' in commands # the sweep precedes every create + assert commands.index(f'tap delete "{TAP00}"') < commands.index(f'tap create "{TAP00}"') + assert f'link set "{TAP00}" down' in commands + assert not any("set_owner" in c for c in commands) + + +@pytest.mark.asyncio +async def test_prepare_tap_datapath_falls_back_without_either_probe(vm, monkeypatch): + """ + A uBridge without the swappable TAP leg (or without the tap module at + all) keeps the node on the relay datapath — it still starts, it just + cannot carry kernel links. + """ + + async def probe(timeout=15.0): + return None + + monkeypatch.setattr("gns3server.compute.docker.iol_docker_vm.probe_bridge_tap_support", probe) + monkeypatch.setattr("gns3server.compute.docker.iol_docker_vm.probe_tap_support", probe) + await vm._prepare_tap_datapath() + + assert vm._tap_datapath is False + assert vm._kernel_taps == {} + assert not any("tap create" in c for c in _commands(vm)) + + +def test_tap_name_fits_ifnamsiz(vm): + """ + The anchor name has to fit IFNAMSIZ (15) even for a late bay/unit. + """ + + assert len(vm._tap_name(7, 3)) <= 15 + + +# --------------------------------------------------------------------------- +# Link attach / detach (the port bridge's swappable topology leg) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_attach_kernel_link_swaps_the_port_leg_to_the_anchor(vm): + """ + A kernel link ensures the anchor (the frozen ensure-then-add contract — + add_nio_tap on an absent name would open a transient TAP), swaps the + port bridge's topology leg to it, runs the shared mixin flow (per-link + kernel bridge, capture/markers/filters) and starts the relay — the unix + binding untouched throughout. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + vm._bridges.add(PORT_BRIDGE) # the start flow registered the port bridge + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + await vm._attach_kernel_link(0, 0, nio) + + commands = _commands(vm) + assert f'tap create "{TAP00}"' in commands # ensure-then-add + assert commands.index(f'tap create "{TAP00}"') < commands.index(f'bridge add_nio_tap {PORT_BRIDGE} "{TAP00}"') + assert f'brctl create "{BRIDGE}"' in commands + assert f'brctl addif "{BRIDGE}" "{TAP00}"' in commands + assert commands.index(f'brctl addif "{BRIDGE}" "{TAP00}"') < commands.index(f"bridge start {PORT_BRIDGE}") + assert f'link set "{TAP00}" up' in commands # the carrier pass brings the anchor up + assert not any("add_nio_udp" in c for c in commands) + + +@pytest.mark.asyncio +async def test_ensure_anchor_creates_only_when_missing(vm, monkeypatch): + """ + tap create is strictly create-only (IFF_TUN_EXCL — an existing device + answers EBUSY), so the ensure checks existence first: a present anchor + costs no command, a swept one is recreated (the repair half of the + ensure-then-add contract). + """ + + vm._kernel_taps[(0, 0)] = TAP00 + + with monkeypatch.context() as missing: + missing.setattr("os.path.exists", lambda p: False) + await vm._ensure_anchor(0, 0) + assert f'tap create "{TAP00}"' in _commands(vm) + + vm._ubridge_send.reset_mock() + with monkeypatch.context() as present: + present.setattr("os.path.exists", lambda p: True) + await vm._ensure_anchor(0, 0) + vm._ubridge_send.assert_not_called() + + +@pytest.mark.asyncio +async def test_attach_kernel_link_without_anchor_raises(vm): + """ + A kernel link on a node that never anchored (relay-only uBridge, or a + start that predates the kernel datapath) is an error naming the relay + datapath, not a silent fallback. + """ + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + with pytest.raises(DockerError, match="no TAP anchor"): + await vm._attach_kernel_link(0, 0, nio) + + +@pytest.mark.asyncio +async def test_add_ubridge_connection_routes_a_kernel_nio_to_the_anchor(vm): + """ + The start-flow entry (also the restart-with-attached-link path) accepts + a kernel NIO where the vendor class rejects it: the port bridge is + ensured, then the same attach flow runs. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + vm._bridges.add(PORT_BRIDGE) + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + + await vm._add_ubridge_connection(nio, 0, 0) + + assert f'bridge add_nio_tap {PORT_BRIDGE} "{TAP00}"' in _commands(vm) + + +@pytest.mark.asyncio +async def test_add_nio_binding_relay_keeps_the_unix_udp_shape(vm): + """ + A relay link on a running node wires only the UDP half — the port bridge + and its unix NIO already exist from the start flow. + """ + + vm.status = "started" + vm._bridges.add(PORT_BRIDGE) + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + + await vm.adapter_add_nio_binding(0, nio, 0) + + commands = _commands(vm) + assert "bridge add_nio_udp bridge0 20000 127.0.0.1 20001" in commands + assert "bridge start bridge0" in commands + assert not any("add_nio_tap" in c for c in commands) + + +@pytest.mark.asyncio +async def test_update_nio_binding_kernel_reapplies_on_the_anchor(vm): + """ + A filter update on an attached kernel link reconciles on the anchor (tc) + without re-swapping the port leg or re-enslaving the TAP. + """ + + vm.status = "started" + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.filters = {"delay": [10]} + + await vm.adapter_update_nio_binding(0, nio, 0) + + commands = _commands(vm) + assert f'tc reset "{TAP00}"' in commands + assert f'tc netem set "{TAP00}" delay 10' in commands + assert not any("add_nio_tap" in c for c in commands) + assert not any("brctl addif" in c for c in commands) + + +@pytest.mark.asyncio +async def test_update_nio_binding_kernel_suspend_flaps_the_anchor(vm): + """ + Suspending a kernel link admin-downs the anchor: the port bridge's TAP + writes fail EIO (dropped) and nothing comes back — a genuinely dead + link, where the relay datapath used the synthetic filter instead. + """ + + vm.status = "started" + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + nio.suspend = True + + await vm.adapter_update_nio_binding(0, nio, 0) + + assert f'link set "{TAP00}" down' in _commands(vm) + + +@pytest.mark.asyncio +async def test_remove_nio_binding_kernel_releases_the_port_leg_first(vm): + """ + Removing a kernel link stops the port bridge (its relay threads hold the + NIO pointers for their whole life — delete_nio_tap refuses while + running), releases the TAP NIO by name, then the shared anchor-side + teardown runs: qdisc reset, brctl delif, per-link bridge deletion. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.adapter_remove_nio_binding(0, 0) + + commands = _commands(vm) + assert f"bridge stop {PORT_BRIDGE}" in commands + assert f'bridge delete_nio_tap {PORT_BRIDGE} "{TAP00}"' in commands + assert commands.index(f"bridge stop {PORT_BRIDGE}") < commands.index(f'bridge delete_nio_tap {PORT_BRIDGE} "{TAP00}"') + assert f'tc reset "{TAP00}"' in commands + assert f'brctl delif "{BRIDGE}" "{TAP00}"' in commands + assert f'brctl delete "{BRIDGE}"' in commands + assert commands.index(f'bridge delete_nio_tap {PORT_BRIDGE} "{TAP00}"') < commands.index(f'tc reset "{TAP00}"') + + +@pytest.mark.asyncio +async def test_remove_nio_binding_relay_keeps_the_udp_teardown(vm): + """ + Removing a relay link keeps the historical teardown: stop + UDP NIO + removal, the unix NIO and its binding untouched. + """ + + vm.status = "started" + vm._bridges.add(PORT_BRIDGE) + nio = vm.manager.create_nio({"type": "nio_udp", "lport": 20000, "rhost": "127.0.0.1", "rport": 20001}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.adapter_remove_nio_binding(0, 0) + + commands = _commands(vm) + assert f"bridge stop {PORT_BRIDGE}" in commands + assert "bridge remove_nio_udp bridge0 20000 127.0.0.1 20001" in commands + assert not any("delete_nio_tap" in c for c in commands) + + +# --------------------------------------------------------------------------- +# Stop order (the port bridges hold the anchor fds) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_stop_ubridge_releases_fds_before_deleting_taps(vm): + """ + The port bridges hold the anchor TAP fds (bridge add_nio_tap), so they + are deleted first — closing the fds — before the taps are unpersisted + (tap delete answers EBADFD on a held device), and the per-link kernel + bridges still need the live control channel. All before the hypervisor + itself stops. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + vm._kernel_taps[(1, 0)] = TAP10 + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio) + + with patch("gns3server.compute.docker.docker_vm.DockerVM._stop_ubridge", new=AsyncioMagicMock()) as stop: + await vm._stop_ubridge() + + stop.assert_called_once() + commands = _commands(vm) + assert f"bridge delete {PORT_BRIDGE}" in commands + assert "bridge delete bridge1" in commands # bay 1 unit 0 (port 0 keeps the historical name) + assert commands.index(f"bridge delete {PORT_BRIDGE}") < commands.index(f'tap delete "{TAP00}"') + assert commands.index("bridge delete bridge1") < commands.index(f'tap delete "{TAP10}"') + assert commands.index(f'tap delete "{TAP00}"') < commands.index(f'brctl delete "{BRIDGE}"') + assert vm._kernel_taps == {} + assert vm._tap_datapath is False + + +@pytest.mark.asyncio +async def test_start_ubridge_prepares_the_datapath_before_any_link(vm): + """ + The anchors are born with the node (uBridge control channel up, before + the start loop attaches links): an anchor that appears only with its + link would leave the switch fast path's deferred join waiting forever. + """ + + with ( + patch("gns3server.compute.base_node.BaseNode._start_ubridge", new=AsyncioMagicMock()) as start, + patch.object(IOLDockerVM, "_prepare_tap_datapath", new=AsyncioMagicMock()) as prepare, + ): + await vm._start_ubridge(require_privileged_access=True) + + start.assert_called_once() + prepare.assert_called_once() + + +# --------------------------------------------------------------------------- +# Anchor naming (the shared contract) +# --------------------------------------------------------------------------- + + +def test_anchor_names_follow_the_iol_docker_key(): + """ + The controller names an IOL runner's anchor for switch absorption with + the same function the compute creates it under — the iol_docker key, + never the docker veth namespace. + """ + + from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_anchor_type + + environment = "GNS3_IOL_RUNNER=1\nGNS3_IOL_STARTUP_CONFIG=cfg.txt" + assert kernel_anchor_type("docker", environment) == "iol_docker" + assert kernel_anchor_type("docker", "GNS3_UNIX_SOCKET_NIO=1") == "docker" + assert kernel_anchor_name(kernel_anchor_type("docker", environment), NODE_ID, 0, 1) == TAP01 diff --git a/tests/compute/docker/test_iol_docker_vm.py b/tests/compute/docker/test_iol_docker_vm.py index 23fcb0e36..49b1952b5 100644 --- a/tests/compute/docker/test_iol_docker_vm.py +++ b/tests/compute/docker/test_iol_docker_vm.py @@ -650,8 +650,8 @@ async def test_generic_unix_socket_dir_honored_in_wiring(compute_project, manage async def test_link_operations_never_send_tap_carrier(compute_project, manager): """ uBridge rejects set_nio_tap_carrier on a bridge without a TAP NIO - ("bridge has no TAP NIO"), and unix-socket NIO bridges never have one: - every link create/update/delete on a running node would fail. The + ("bridge has no TAP NIO"), and unix-socket NIO relay bridges never have + one: every link create/update/delete on a running node would fail. The vendor override must keep the carrier command out of all three paths. """ @@ -659,6 +659,10 @@ async def test_link_operations_never_send_tap_carrier(compute_project, manager): vm._ubridge_hypervisor = MagicMock() # truthy ubridge, like a started node vm._ubridge_send = AsyncioMagicMock() vm.status = "started" + # A started node's port bridge exists and is registered by the start + # flow — record it, or the relay attach would run the ensure path + # (bridge create + socket wait) this fake never set up. + vm._bridges.add(vm._bridge_name(0, 0)) nio = manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) await vm.adapter_add_nio_binding(0, nio, 0) diff --git a/tests/utils/test_kernel_anchor.py b/tests/utils/test_kernel_anchor.py index 7b88ffd9c..7f5558d1a 100644 --- a/tests/utils/test_kernel_anchor.py +++ b/tests/utils/test_kernel_anchor.py @@ -21,7 +21,7 @@ Ethernet-switch link), plus the cascade veth naming for switch-to-switch links. """ -from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name, kernel_cascade_names +from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name, kernel_anchor_type, kernel_cascade_names NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" LINK_ID = "1a2b3c4d-5e6f-4a5b-8c9d-0e1f2a3b4c5d" @@ -32,6 +32,22 @@ def test_every_node_type_has_its_own_prefix(): assert kernel_anchor_name("qemu", NODE_ID, 1, 2) == "gq00010203e1p2" assert kernel_anchor_name("iou", NODE_ID, 0, 3) == "gi00010203e0p3" assert kernel_anchor_name("dynamips", NODE_ID, 6, 48) == "gd00010203e6p48" + assert kernel_anchor_name("iol_docker", NODE_ID, 0, 0) == "gx00010203e0p0" + + +def test_anchor_type_keys_iol_runner_containers_apart(): + """ + IOL runner containers are Docker nodes whose anchors are persistent TAPs, + not veth host ends — their own key keeps the gv namespace free of TAPs. + The marker check must ignore a mere mention of the knob and survive a + multi-line environment. + """ + + assert kernel_anchor_type("docker", "GNS3_IOL_RUNNER=1") == "iol_docker" + assert kernel_anchor_type("docker", "GNS3_IOL_STARTUP_CONFIG=cfg.txt\nGNS3_IOL_RUNNER=1") == "iol_docker" + assert kernel_anchor_type("docker") == "docker" + assert kernel_anchor_type("docker", "GNS3_UNIX_SOCKET_NIO=1") == "docker" + assert kernel_anchor_type("qemu") == "qemu" def test_unknown_node_types_have_no_anchor(): @@ -62,7 +78,7 @@ def test_cascade_names_fit_ifnamsiz(): def test_cascade_names_stay_out_of_the_node_anchor_namespace(): - # "gs" cannot collide with any node anchor prefix (gv/gq/gi/gd), so a + # "gs" cannot collide with any node anchor prefix (gv/gq/gi/gd/gx), so a # cascade end is never mistaken for another node's anchor for end in kernel_cascade_names(LINK_ID): - assert not any(end.startswith(prefix) for prefix in ("gv", "gq", "gi", "gd")) + assert not any(end.startswith(prefix) for prefix in ("gv", "gq", "gi", "gd", "gx")) From 13c1ffb951b7c32b36459c412618ca359970dadf Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 13:30:42 +0800 Subject: [PATCH 046/101] feat(controller): let IOL runner containers take the kernel datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _is_unix_socket_docker no longer excludes the IOL runner marker: those containers anchor on persistent TAPs now (compute side), so they graduate through _kernel_endpoint_ready's gate instead — both ubridge_bridge_tap (the swappable leg) and ubridge_tap (the tap module) strict True, an unreported capability keeping the link on the relay where it always works. The generic GNS3_UNIX_SOCKET_NIO exclusion keeps its shape (an IOL runner environment wins the compute's class selection, so the exclusion must not fire when both markers are set). The switch fast path names an IOL runner peer's anchor through kernel_anchor_type — the iol_docker key, never the gv veth namespace. The 307d676cb regression tests move with the semantics: the capability matrix replaces the unconditional relay, and the switch-path test also asserts the gx naming. Its original concern stands — a compute that cannot serve the anchor lifecycle must never be handed a kernel link (the switch's deferred join would make it a silent dead cable). --- gns3server/controller/udp_link.py | 42 +++++++---- tests/controller/test_kernel_datapath_link.py | 72 +++++++++++++------ 2 files changed, 80 insertions(+), 34 deletions(-) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index cc8e3b356..391bb04e3 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -21,7 +21,7 @@ import logging from gns3server.config import Config from gns3server.utils.application_id import is_iol_runner_environment -from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_cascade_names +from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_anchor_type, kernel_cascade_names from gns3server.utils.packet_filter_validation import ( KERNEL_UNSUPPORTED_FILTERS, split_kernel_only_features, @@ -57,18 +57,21 @@ log = logging.getLogger(__name__) def _is_unix_socket_docker(node): """ Best-effort detection of vendor containers bridged through AF_UNIX - socket pairs: they have no host-side interface to enslave into a kernel - bridge. Two ways in — the generic GNS3_UNIX_SOCKET_NIO knob, or the IOL - runner marker, whose class (IOLDockerVM, selected on GNS3_IOL_RUNNER) - forces the unix-socket wiring whatever the knob says. The compute side - independently rejects kernel-datapath NIOs on these containers, so a - missed detection surfaces as a clearer-late error — except on the switch - fast path, where an absent anchor is deferred forever by design (a link - to a stopped node must not fail), making the miss a silently dead cable. + socket pairs with no anchor of their own: the generic + GNS3_UNIX_SOCKET_NIO knob. IOL runner containers also bridge through + unix sockets (their class forces the wiring), but their anchors are + persistent TAPs — they graduate through _kernel_endpoint_ready's + capability gate instead of this exclusion. When both markers are present + the IOL runner one wins the compute's class selection, so the exclusion + must not fire for it. The compute side independently rejects + kernel-datapath NIOs on anchor-less containers, so a missed detection + surfaces as a clearer-late error — except on the switch fast path, where + an absent anchor is deferred forever by design (a link to a stopped node + must not fail), making the miss a silently dead cable. """ environment = (node.properties or {}).get("environment") or "" - return "GNS3_UNIX_SOCKET_NIO" in environment or is_iol_runner_environment(environment) + return "GNS3_UNIX_SOCKET_NIO" in environment and not is_iol_runner_environment(environment) class UDPLink(Link): @@ -196,8 +199,12 @@ class UDPLink(Link): anchor (brctl) or attaches the relay to it (add_nio_ethernet) without touching the interfaces the node itself uses. Both endpoints must be on the same compute (there is no kernel link across hosts) and both - must be able to anchor: Docker adapters are born as veth pairs, - QEMU adapters as persistent TAPs, which need uBridge's tap module — + must be able to anchor: Docker adapters are born as veth pairs, and + IOL runner containers — whose guest leg is unix sockets, not a veth — + anchor on persistent TAPs reached through the bridge module's + swappable TAP leg (its own capability pair, see + _kernel_endpoint_ready). QEMU adapters are persistent TAPs, which + need uBridge's tap module — asked of that compute, since an old uBridge leaves QEMU on the relay datapath and cannot carry a kernel link at all. IOU's Ethernet bays anchor on persistent TAPs too, bound to its IOL fabric @@ -253,6 +260,15 @@ class UDPLink(Link): """ if node.node_type == "docker": + environment = (node.properties or {}).get("environment") or "" + if is_iol_runner_environment(environment): + # IOL runner containers anchor on persistent TAPs like IOU, + # through the bridge module's swappable TAP leg — both + # capabilities strict True: an unreported one (old uBridge, + # failed probe) keeps the link on the relay, where it always + # works. + capabilities = node.compute.capabilities or {} + return capabilities.get("ubridge_bridge_tap") is True and capabilities.get("ubridge_tap") is True return True capabilities = node.compute.capabilities or {} if node.node_type == "qemu": @@ -354,7 +370,7 @@ class UDPLink(Link): peer_index = 1 - index peer = self._nodes[peer_index]["node"] peer_anchor = kernel_anchor_name( - peer.node_type, + kernel_anchor_type(peer.node_type, (peer.properties or {}).get("environment")), peer.id, self._nodes[peer_index]["adapter_number"], self._nodes[peer_index]["port_number"], diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 834172dde..9c0ababfe 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -431,43 +431,73 @@ async def test_kernel_datapath_not_eligible_for_unix_socket_containers(project): @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_for_iol_runner_containers(project): +async def test_iol_runner_containers_gate_on_the_bridge_tap_capabilities(project): """ - The IOL runner marker is the other way into the unix-socket datapath: the - compute selects IOLDockerVM on GNS3_IOL_RUNNER and that class wires the - adapters through AF_UNIX socket pairs whatever the generic - GNS3_UNIX_SOCKET_NIO knob says — and the documented environment for - those nodes carries only the marker. Matching the knob alone let them - through to a kernel link their compute cannot serve. + The IOL runner marker selects a container whose guest leg is unix + sockets: it can still anchor kernel links (persistent TAPs through the + bridge module's swappable TAP leg), but only when its compute reports + both ubridge_bridge_tap and ubridge_tap — an unreported capability (old + uBridge, failed probe) keeps the link on the relay, where it always + works. This is also the regression guard for the compute that cannot + serve the anchor lifecycle being handed a kernel link anyway. """ - link, node1, node2 = await _kernel_link(project) - node2._properties = {"environment": "GNS3_IOL_RUNNER=1"} - assert link._kernel_datapath_eligible(node1, node2) is False + def iol_capable_compute(bridge_tap, tap): + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_bridge_tap": bridge_tap, "ubridge_tap": tap} + return compute - # the marker travels in a multi-line environment like any other knob - node2._properties = {"environment": "GNS3_IOL_RUNNER=1\nGNS3_IOL_STARTUP_CONFIG=cfg.txt"} - assert link._kernel_datapath_eligible(node1, node2) is False + for environment in ("GNS3_IOL_RUNNER=1", "GNS3_IOL_RUNNER=1\nGNS3_IOL_STARTUP_CONFIG=cfg.txt"): + compute = iol_capable_compute(True, True) + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "iol1", environment=environment) + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True, environment + + for bridge_tap, tap in ((None, True), (True, None), (False, True), (True, False)): + compute = iol_capable_compute(bridge_tap, tap) + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "iol1", environment=environment) + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False, (environment, bridge_tap, tap) @pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_for_an_iol_runner_container_on_a_switch(project): +async def test_iol_runner_container_on_a_switch_gates_and_names_the_iol_docker_anchor(project): """ - The switch fast path absorbs the peer's anchor, and a container bridged - through unix sockets has none — nor will it ever. The switch defers a - join against a stopped peer by design (it must not fail the link), so - such a cable would be silently dead instead of merely slow: the gate has - to run before the switch branch, keeping the link on the relay the - switch's own port TAP serves. + On the switch fast path an IOL runner peer gates on the same capability + pair — and when it passes, the switch absorbs an anchor named under the + ``iol_docker`` key (a persistent TAP, not the ``gv`` veth namespace). + Without the capabilities the link must stay on the relay: the switch + defers a join against a missing anchor by design (a link to a stopped + node must not fail), so a gate miss here is a silently dead cable, not + a late error. """ + from gns3server.utils.kernel_anchor import kernel_anchor_name + + compute = MagicMock() + compute.id = "compute-1" + compute.capabilities = {"ubridge_bridge_tap": True, "ubridge_tap": True} + switch = _node(project, compute, "sw1", node_type="ethernet_switch") + peer = _node(project, compute, "iol1", environment="GNS3_IOL_RUNNER=1") + link = await _link(project, switch, peer) + + assert link._kernel_datapath_eligible(switch, peer) is True + entries = await link._prepare() + switch_nio = entries[0][3] + assert switch_nio["type"] == "nio_anchor" + assert switch_nio["anchor"] == kernel_anchor_name("iol_docker", peer.id, 0, 0) + assert switch_nio["anchor"].startswith("gx") + + # a compute that cannot serve the swappable leg keeps the link relay compute = MagicMock() compute.id = "compute-1" compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} switch = _node(project, compute, "sw1", node_type="ethernet_switch") peer = _node(project, compute, "iol1", environment="GNS3_IOL_RUNNER=1") link = await _link(project, switch, peer) - assert link._kernel_datapath_eligible(switch, peer) is False From 72aa379e8778e4510f4b91498875e39573b7e40a Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 13:30:51 +0800 Subject: [PATCH 047/101] test(e2e): live IOL runner container kernel datapath suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real iol-xe containers (Cisco CML iol-runner images, real IOS-XE on the telnet console) against an isolated instance, the real uBridge with the swappable bridge TAP leg: * kernel link reports kernel_datapath; anchors exist from node start and the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [unix <-> tap] port-bridge relay; delay 100 lands as netem on both anchors with the measured RTT growing >= 150 ms and returning < 50 ms when cleared; * suspend admin-downs the anchor and kills the traffic; resume restores; AF_PACKET capture writes a real pcap; * link delete/re-create swaps the port leg out and back in with the ping returning; node stop leaves no gx devices and a start recreates the whole wiring from the NIO; project delete cleans everything; * the relay control on a relay-configured instance keeps the link off the kernel and still pings (anchors may exist idle — the same semantics as QEMU's and Dynamips' anchors: the server flag decides attachment, not anchor birth). Harness: create_iol_router (the appliance's template properties), docker_images, iol_anchor_name, stage_iol_base_config (the shipped iol-xe-base.txt copied into the isolated instance's configs dir — the GNS3_IOL_STARTUP_CONFIG lookup path), boot_wait(straight_prompt=True) for images that boot from NVRAM straight into the CLI with no 'Press RETURN' sentinel, and GNS3_E2E_DEBUG=1 to run the isolated instance with debug logging (the uBridge command stream lands in server.log — how the ensure-anchor EBUSY was pinned down). --- tests/e2e/README.md | 12 + tests/e2e/harness.py | 60 ++++- tests/e2e/test_iol_docker_kernel_datapath.py | 249 +++++++++++++++++++ 3 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/test_iol_docker_kernel_datapath.py diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 1addb9d2f..f3adf7016 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -44,3 +44,15 @@ Current scenarios: (anchors, per-link bridge, netem delay, suspend, capture, delete/re-create, stop/start) plus a serial link staying on the relay, and a negative control on an isolated relay-configured instance. +* `test_ethernet_switch_kernel_datapath.py` — a real Ethernet switch + (kernel bridge) between two c7200 routers: deferred joins on boot, VLAN + isolation, absorbed-anchor filters/suspend/capture, VPCS peers staying on + the relay, and a relay negative control. +* `test_iol_docker_kernel_datapath.py` — two real iol-xe containers (Cisco + CML iol-runner images): kernel link through the port bridge's swappable + TAP leg (anchors at start, netem delay, suspend, capture, link + delete/re-create swap, node stop/start rewiring) and a relay negative + control. + +Set `GNS3_E2E_DEBUG=1` to run the isolated instance with debug logging (the +uBridge command stream lands in its `server.log`). diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 239ce9bb8..aef68c624 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -126,6 +126,26 @@ class Compute: {"compute_id": "local", "name": name, "node_type": "dynamips", "properties": properties}, ) + def create_iol_router(self, project_id, name, image, adapters=2): + """An IOL runner container (iol-runner images), the appliance's + template properties.""" + return self.call( + "POST", + f"/projects/{project_id}/nodes", + { + "compute_id": "local", + "name": name, + "node_type": "docker", + "properties": { + "image": image, + "adapters": adapters, + "console_type": "telnet", + "environment": "GNS3_IOL_RUNNER=1\nGNS3_IOL_STARTUP_CONFIG=iol-xe-base.txt", + "extra_volumes": ["/config"], + }, + }, + ) + def create_link(self, project_id, a, b): return self.call( "POST", @@ -150,6 +170,9 @@ class Compute: def dynamips_images(self): return [image["filename"] for image in self.call("GET", "/computes/local/dynamips/images")] + def docker_images(self): + return [image["image"] for image in self.call("GET", "/computes/local/docker/images")] + # --------------------------------------------------------------------------- # Server lifecycle @@ -247,8 +270,11 @@ def _start_isolated(kernel): log = open(os.path.join(tmpdir, "server.log"), "w") # S603: fixed argv, this interpreter, no shell involvement + argv = [sys.executable, "-m", "gns3server", "--config", config_path] + if os.environ.get("GNS3_E2E_DEBUG"): + argv.append("-d") # debug logging — the uBridge command stream lands in server.log process = subprocess.Popen( # noqa: S603 - [sys.executable, "-m", "gns3server", "--config", config_path], + argv, cwd=REPO_ROOT, stdout=log, stderr=subprocess.STDOUT, @@ -370,10 +396,14 @@ class IOSConsole: out, self.buf = self.buf, "" return out - def boot_wait(self, timeout=360): + def boot_wait(self, timeout=360, straight_prompt=False): """Wait until the IOS prompt is ready, answering the initial config dialog if it appears. The prompt only counts once the stream around - it goes quiet — a boot banner still printing is not a prompt.""" + it goes quiet — a boot banner still printing is not a prompt. + + ``straight_prompt`` accepts a quiet prompt without the "Press RETURN" + sentinel — images that boot straight from a startup config into the + CLI (IOL from its NVRAM) never print it.""" answered = False deadline = time.time() + timeout self.buf = "" @@ -385,7 +415,8 @@ class IOSConsole: self.sock.sendall(b"no\r") answered = True last_growth = time.time() - prompt_seen = ("Press RETURN" in self.buf or answered) and PROMPT_RE.search(self.buf) + sentinel = straight_prompt or "Press RETURN" in self.buf or answered + prompt_seen = sentinel and PROMPT_RE.search(self.buf) if prompt_seen and time.time() - last_growth > 1.2: self.buf = "" return @@ -492,6 +523,27 @@ def anchor_name(node_id, adapter, port): return "gd" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" +def iol_anchor_name(node_id, adapter, port): + """The IOL runner container's anchor TAP name (see IOLDockerVM._tap_name).""" + return "gx" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" + + +def stage_iol_base_config(server): + """ + Copy the shipped iol-xe base config into an isolated instance's configs + directory (its GNS3_IOL_STARTUP_CONFIG lookup path) so nodes boot past + the setup dialog. A no-op for an external server — there the operator's + install provides it. + """ + + if not server.isolated: + return + src = os.path.join(REPO_ROOT, "gns3server", "configs", "iol-xe-base.txt") + configs_dir = os.path.join(server.tmpdir, "configs") + os.makedirs(configs_dir, exist_ok=True) + shutil.copy(src, os.path.join(configs_dir, "iol-xe-base.txt")) + + def qdiscs(dev): """The tc qdisc listing of a host interface ('' when tc is missing).""" if not TC: diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py new file mode 100644 index 000000000..594ea2cae --- /dev/null +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -0,0 +1,249 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software and is redistributed under the same terms as gns3-server. +# See the LICENSE file for licensing information. +# + +""" +Live end-to-end test for the IOL runner container's kernel datapath (pytest +marker ``e2e``): two real iol-xe containers (Cisco CML's iol-runner images, +the real IOS-XE CLI on PID 1 stdio), driven through the controller REST API +and their telnet consoles. The guest leg stays the runner's unix-socket pair +— the kernel datapath buys the *link segment*: anchor TAPs, a per-link Linux +bridge, kernel filters, suspend and capture. + +* two containers on one compute report ``kernel_datapath`` when the compute's + uBridge has both the tap module and the swappable bridge TAP leg + (``ubridge_bridge_tap``); the anchors (``gx`` TAPs) exist from node start, + before any link — and the per-link bridge enslaves exactly the two of them; +* real ICMP crosses the [unix ↔ TAP] port bridge relay; +* a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; +* suspend admin-downs the anchor and kills the traffic; resume restores; +* link delete releases the port bridge's TAP leg (stop → delete_nio_tap) and + the anchor survives — re-creating the link swaps the leg back in (the + c-socket binding never re-binds) and the traffic returns; +* node stop tears the anchors down (uBridge holds their fds — port bridges + first, then tap delete); a restart recreates the whole wiring from the NIO; +* deleting the project leaves no anchors and no bridges behind. + +``test_iol_docker_relay_control`` is the negative control on an isolated +relay-configured instance: the same topology rides unix ↔ UDP through the +uBridge relay (no anchors ever exist) and still pings. +""" + +import re +import time + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# one adapter = one 4-port unit; Ethernet0/0 is (adapter 0, port 0) +ETH = (0, 0) +BOOT_TIMEOUT = 360 + + +def _pick_image(server): + images = server.compute.docker_images() + image = next((name for name in images if name.startswith("iol-xe/iol-xe:")), None) + if not image: + pytest.skip(f"no iol-xe/iol-xe image available on this server (images: {images[:5]})") + return image + + +def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): + node = server.compute.node(project_id, node_id) + console = harness.IOSConsole(node["console_host"], node["console"]) + # straight_prompt: IOL boots from its NVRAM startup config straight into + # the CLI — no "Press RETURN" sentinel on this console + console.boot_wait(timeout=timeout, straight_prompt=True) + return console + + +def _topology(compute, pid, image, name): + n1 = compute.create_iol_router(pid, f"{name}-1", image) + n2 = compute.create_iol_router(pid, f"{name}-2", image) + return n1, n2 + + +def test_iol_docker_kernel_datapath(): + server = harness.live_server(kernel=True) + compute = server.compute + caps = compute.capabilities() + if caps.get("ubridge_bridge_tap") is not True or caps.get("ubridge_tap") is not True: + pytest.skip( + f"this compute's uBridge cannot serve the IOL container anchor lifecycle " + f"(ubridge_bridge_tap={caps.get('ubridge_bridge_tap')}, ubridge_tap={caps.get('ubridge_tap')})" + ) + image = _pick_image(server) + harness.stage_iol_base_config(server) + + project = compute.create_project("iol-e2e") + pid = project["project_id"] + a1 = a2 = None + try: + n1, n2 = _topology(compute, pid, image, "E2E") + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.iol_anchor_name(n1_id, *ETH) + a2 = harness.iol_anchor_name(n2_id, *ETH) + + # The link is created while both containers are stopped: the kernel + # NIO is bound with no anchors in existence yet and wired by node + # start (the deferred-wiring path). + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert not harness.tap_exists(a1), "anchors must not exist before start" + + print(".. starting containers and waiting for IOS-XE boots") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{n2_id}/start") + c1 = _boot(server, pid, n1_id) + c2 = _boot(server, pid, n2_id) + + # Kernel objects: anchors exist (born with the node, before the link + # was ever wired) and the per-link bridge has exactly the two of them. + assert harness.tap_exists(a1) and harness.tap_exists(a2), (a1, a2) + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + out = c1.run("show ip int brief") + assert re.search(r"Ethernet0/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + print(".. both routers configured, pinging through the port-bridge swap") + baseline = harness.wait_ping(c1, "10.1.1.2") + assert baseline["success"] == 100, baseline["raw"] + + # delay 100: netem on both anchors (each impairs one direction), + # one-way ~100 ms => RTT grows by ~200 ms + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(a1) and "netem" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + delayed = harness.wait_ping(c1, "10.1.1.2") + assert delayed["success"] == 100, delayed["raw"] + assert delayed["avg"] >= 150, (baseline, delayed) + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) + fast = harness.wait_ping(c1, "10.1.1.2") + assert fast["success"] == 100 and fast["avg"] < 50, fast + + # suspend: anchor admin-down (the port bridge's TAP writes fail EIO, + # nothing comes back); resume restores + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) + assert not harness.tap_up(a1) + dead = harness.ping(c1, "10.1.1.2", repeat=3) + assert dead["success"] == 0, dead["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) + assert harness.tap_up(a1) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + + # capture: AF_PACKET on the anchor writes a real pcap + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.ping(c1, "10.1.1.2") + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # delete / re-create the kernel link: the port bridge's TAP leg is + # released (stop -> delete_nio_tap), the anchor survives (the node + # owns it), and re-creating swaps the leg back in — the unix binding + # never re-binds across the churn. + compute.call("DELETE", f"/projects/{pid}/links/{lid}") + assert harness.bridge_members(bridge) is None + assert harness.tap_exists(a1) and harness.tap_exists(a2) + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + # node stop: uBridge holds the anchor fds, so the port bridges go + # first and the taps with them (a stopped container must not litter + # the host with gx devices); a restart recreates the whole wiring + # from the NIO that outlived it. + print(".. stopping and restarting container 1") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/stop") + assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 + # the per-link bridge lost this end's port; it may be gone entirely + # (both endpoints delete it, last one wins) + assert harness.bridge_members(bridge) in (None, [a2]), harness.bridge_members(bridge) + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + c1.close() + c1 = _boot(server, pid, n1_id) + assert harness.tap_exists(a1) + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closed the nodes: anchors and bridges go with them + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(a2), timeout=15), ( + a1, + a2, + ) + + +def test_iol_docker_relay_control(): + """ + Negative control: the same topology with ``enable_kernel_datapath = + false`` (an isolated instance) wires nothing into the kernel — no + per-link bridge, no enslavement — and still pings over the unix ↔ UDP + relay. + + Anchors DO exist here (an IOL container creates its port TAPs at start + whenever the uBridge tap module and the swappable leg are available); + the server configuration only decides whether links are attached to + them (the same semantics as QEMU's and Dynamips' anchors). + """ + + server = harness.live_server(kernel=False) + compute = server.compute + image = _pick_image(server) + harness.stage_iol_base_config(server) + + project = compute.create_project("iol-relay-e2e") + pid = project["project_id"] + n1_id = n2_id = a1 = None + try: + n1, n2 = _topology(compute, pid, image, "E2E") + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.iol_anchor_name(n1_id, *ETH) + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + assert link["kernel_datapath"] is False, link + + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{n2_id}/start") + c1 = _boot(server, pid, n1_id) + c2 = _boot(server, pid, n2_id) + + # no kernel wiring: the anchor (if born) is nobody's bridge port + assert harness.bridge_members(harness.link_bridge_name(link["link_id"])) is None + if harness.tap_exists(a1): + assert not harness.tap_up(a1), a1 + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + relay_ping = harness.wait_ping(c1, "10.1.1.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) From 02340f2ed330481a5a23088f57ec781ef224c1ae Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 13:30:59 +0800 Subject: [PATCH 048/101] docs(iol): record the IOL runner container kernel datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feature doc covers the architecture (one irreducible userspace hop on the unix-socket leg; the link segment kernelized through the port bridge's swappable TAP leg), the capability gate, the anchor lifecycle, the swap flow, the inherited capture/markers/filters, the relay fallback and the live verification — including the ensure-then-add lesson (tap create is strictly create-only). The other datapath docs' roadmaps catch up: the IOL runner container has landed (and the ethernet-switch doc's stale 'cascade deferred' exclusion from d2f8aba58 is corrected while there); the docker doc's adapter table no longer rejects IOLDockerVM. --- docs/features/docker-kernel-datapath.md | 2 +- docs/features/dynamips-kernel-datapath.md | 4 +- .../ethernet-switch-kernel-datapath.md | 20 +- docs/features/iol-docker-kernel-datapath.md | 172 ++++++++++++++++++ docs/features/iou-kernel-datapath.md | 11 +- docs/features/qemu-kernel-datapath.md | 7 +- 6 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 docs/features/iol-docker-kernel-datapath.md diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 42ebb763f..7de504a59 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -85,7 +85,7 @@ markers): |---|---|---|---| | `DockerVM` | default | veth pair | yes | | `VendorDockerVM` (non-unix-socket) | `console_type=docker_exec`, `GNS3_SKIP_INIT`, … (XRd, SR Linux, …) | veth pair (same path via `super()`) | yes | -| `IOLDockerVM` | `GNS3_IOL_RUNNER=1` | AF_UNIX datagram socket pairs (`sNN`/`cNN`) — the container netns is unused | rejected (no host-side interface) | +| `IOLDockerVM` | `GNS3_IOL_RUNNER=1` | AF_UNIX datagram socket pairs (`sNN`/`cNN`) — the container netns is unused; kernel links ride persistent `gx` TAP anchors through the port bridge's swappable TAP leg (see `iol-docker-kernel-datapath.md`) | yes (gated on `ubridge_bridge_tap` + `ubridge_tap`) | | `VendorDockerVM` (unix-socket) | `GNS3_UNIX_SOCKET_NIO=1` | same socket contract, generic capability for vendor NOS images | rejected | Unix-socket containers are detected both in the controller diff --git a/docs/features/dynamips-kernel-datapath.md b/docs/features/dynamips-kernel-datapath.md index 64e18ad02..4d731c7b9 100644 --- a/docs/features/dynamips-kernel-datapath.md +++ b/docs/features/dynamips-kernel-datapath.md @@ -151,5 +151,7 @@ enable_kernel_datapath = True The Ethernet switch has landed (see `ethernet-switch-kernel-datapath.md` — it absorbs the peer's anchor into its own kernel bridge; switch-to-switch -cascades and the IOL Docker node are next). Cross-compute kernel links need +cascades and the IOL runner container have landed too — see +`ethernet-switch-kernel-datapath.md` and +`iol-docker-kernel-datapath.md`). Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index 2c309707a..7808129ee 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -28,15 +28,17 @@ end / TAP becomes the switch port; frames cross ## Datapath selection A link with an `ethernet_switch` endpoint rides the fast path when the -**peer** can anchor (same compute; Docker always, QEMU/IOU/Dynamips per +**peer** can anchor (same compute; Docker always — including IOL runner +containers, per their own capability gate — QEMU/IOU/Dynamips per their capabilities) — the switch side needs no anchoring capability of its own, its bridge and brctl exist by construction. Exclusions, all staying on the relay: -* **switch ↔ switch**: a kernel interface can belong to exactly one - bridge, so cascading two switch bridges needs a veth pair between them — - a separate mechanism, deferred; -* anchor-less peers (VPCS, cloud, the Dynamips-hosted hub); +* **switch ↔ switch** cascades ride the kernel too — through a veth pair + joining the two bridges (one end per switch, each side applying its own + port mode to its own end); see the wire format below; +* anchor-less peers (VPCS, cloud, the Dynamips-hosted hub, generic + `GNS3_UNIX_SOCKET_NIO` containers); * cross-compute links, non-Ethernet ports: the common rules. ## The wire format: one NIO names the other end's interface @@ -153,6 +155,8 @@ enable_kernel_datapath = True ## Roadmap Switch-to-switch links (a veth pair joining the two kernel bridges, with -each side's port mode applied at its own end) and the IOL Docker node -(`bridge add_nio_tap`); cross-compute kernel links need VXLAN/GENEVE -encapsulation, deferred until every node type is kernelized. +each side's port mode applied at its own end) have landed (see above), and +so has the IOL runner container (`iol-docker-kernel-datapath.md` — its +`iol_docker` anchors absorb like any peer's). Cross-compute kernel links +need VXLAN/GENEVE encapsulation, deferred until every node type is +kernelized. diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md new file mode 100644 index 000000000..91d3a4b08 --- /dev/null +++ b/docs/features/iol-docker-kernel-datapath.md @@ -0,0 +1,172 @@ + + +> This documentation is AI-generated with reference to actual code and verified +> against real-kernel testing. AI can make mistakes — please verify against +> the source code when in doubt. + +# IOL runner container kernel datapath + +## Overview + +IOL runner containers (`IOLDockerVM`, selected by the `GNS3_IOL_RUNNER` +environment marker — the Cisco CML `iol-runner` images such as +`iol-xe/iol-xe:17-18-02`) have no container-namespace networking at all: +their guest leg is the runner's per-interface AF_UNIX datagram socket pairs. +A unix socket is not a kernel interface, so until now every link of such a +container rode the uBridge userspace relay (unix ↔ UDP). + +The kernel datapath gives every Ethernet bay/unit a **persistent TAP anchor** +(`gx` names, born at node start exactly like IOU's) and wires the *link +segment* through the kernel. One userspace hop on the container leg is +irreducible — the socket pair is the only physical layer the runner exposes — +but the anchor → per-link bridge → peer anchor crossing, the kernel filters, +the suspend carrier and the AF_PACKET capture/markers are all native now. + +``` + container netns (iol-runner) root namespace + ┌────────────┐ unix datagram ┌────────────────────────────────┐ + │ netiomux │◄─ sNN/cNN.sock ──►│ uBridge port bridge │ + │ (sNN/cNN) │ (host-side dir) │ [nio_unix, nio_tap] relay │ + └────────────┘ │ │ holds the TAP fd │ + │ ▼ │ + │ gx{id}e{bay}p{unit} ─────────┼─► gns3{link_id} + │ (persistent TAP anchor) │ (per-link kernel + └────────────────────────────────┘ bridge port) +``` + +The port bridge is the deployment shape ubridge's +`doc/gns3server-integration.md` § *bridge — generic NIO relay with a +swappable TAP leg* froze: a per-node bridge created at start, the unix NIO in +the first slot for the node's whole life, the topology leg in the second — +`add_nio_udp` (relay) or `add_nio_tap` (kernel link) — swapped as links are +attached, suspended, deleted or the project reopens, through +stop → delete → add → start. The c-socket binding never re-binds across a +swap: frames the container sends while the bridge is stopped queue on the +socket and relay after `start` (regression-tested in ubridge's bridge suite). + +## Datapath selection + +`UDPLink._kernel_datapath_eligible` treats an IOL runner container like any +Docker node (same compute, Ethernet port) with one gate of its own in +`_kernel_endpoint_ready`: the compute must report both + +* `ubridge_tap` — the tap module (`tap create`/`tap delete`), the same + capability QEMU and Dynamips gate on, and +* `ubridge_bridge_tap` — `bridge delete_nio_tap`, the swappable-leg command + (probed with one command against a bridge that cannot exist: 202 = old + build, 214 = new; no scratch objects, no capabilities, cached by binary + identity like every other probe). + +Both strict `True`; anything unreported (old uBridge, failed probe) keeps the +link on the relay, where it always works. The generic `GNS3_UNIX_SOCKET_NIO` +containers stay excluded as before — that marker's semantics are +image-specific, while `GNS3_IOL_RUNNER` selects a class whose anchors this +server owns. + +## Anchor lifecycle + +* **Birth (node start, before any link)** — `_start_ubridge` runs + `_prepare_tap_datapath` right after the hypervisor connects: probe both + capabilities, then for every (bay, unit 0-3) sweep a leftover + (`tap delete`, suppressed), `tap create` (hardened, starts DOWN) and + `link set down` — carrier off until a link attaches. The anchors exist + whether or not any link ever uses them: an anchor born with its link would + leave the Ethernet switch fast path's deferred join waiting forever. + No `tap set_owner`: uBridge itself holds these fds. +* **Life** — the anchor is never recreated. Link create/delete/switch, + suspend, capture and markers only change what is attached to it, and the + ensure-then-add contract guards the one window where the device could have + been swept (`/sys/class/net` existence check — `tap create` is strictly + create-only, `IFF_TUN_EXCL`, and refuses an existing device with EBUSY). +* **Death (node stop)** — the port bridges hold the anchor fds + (`bridge add_nio_tap`), so they are deleted first (`bridge delete` stops + their relay threads and frees the NIOs), then the anchors themselves + (`tap delete` answers EBADFD on a device another fd still holds), then the + per-link kernel bridges — all while the control channel lives, before the + hypervisor stops. A restart recreates everything from the NIO. + +## Link operations + +* **Kernel link create** (`_attach_kernel_link`): ensure the port bridge and + its unix NIO → ensure the anchor → `bridge add_nio_tap` (uBridge opens and + holds the fd) → the shared mixin flow (`brctl create`/`addif` the per-link + bridge, capture, markers, tc netem/cls_bpf/eBPF filters on the anchor) → + `bridge start` (the [unix ↔ tap] relay) → the carrier pass (which is also + what brings a born-down anchor up; a suspended NIO sets it back down). +* **Link delete** (`_release_port_tap`): `bridge stop` (delete_nio_tap + refuses while running — the relay threads hold the NIO pointers for their + whole life; freeing under them is a use-after-free) → + `bridge delete_nio_tap ""` (matched on the kernel-resolved + name) → the mixin teardown (markers, tc reset, `brctl delif`, per-link + bridge deletion). The anchor survives — the node owns it, not the link. +* **Relay link** — unchanged: the port bridge's second leg is `add_nio_udp`, + userspace filters at the port bridge. The unix NIO stays in its slot + across every datapath switch. + +## Capture, markers, filters + +Inherited from `KernelDatapathMixin` unchanged — everything is keyed on the +anchor interface name, which is why a TAP and a veth host end are +interchangeable there: `capture start_kernel` / `marker add_kernel` +(AF_PACKET), one tc netem qdisc (the full netem surface plus the +extensions), `bpf` as cls_bpf match-drop, `frequency_drop`/`quota`/ +`window_drop` as the eBPF stateful classifier — all capability-gated per +compute exactly like the Docker veth datapath. + +Suspend semantics: anchor admin-down. The port bridge's TAP writes fail EIO +(tolerated by the delivered uBridge hardening) and its reads fall silent — +traffic stops both ways. The runner itself is unaware of the link state +(netiomux has no carrier signal), exactly as on the relay datapath where +suspend rode the synthetic frequency_drop filter instead. + +## Relay fallback + +Missing either capability, the node runs relay-only: links ride +unix ↔ UDP, `kernel_datapath` stays false, and anchors are simply never +created (or, with `enable_kernel_datapath` off server-wide, created but +never attached — the same semantics as QEMU's and Dynamips' anchors). + +## Verified + +Live on the real server (`tests/e2e/test_iol_docker_kernel_datapath.py`, +isolated instance, real `iol-xe/iol-xe:17-18-02` images, real uBridge with +`bridge delete_nio_tap`): + +* kernel link reports `kernel_datapath`, anchors exist from node start, the + per-link bridge enslaves exactly the two of them; +* real ICMP crosses the [unix ↔ tap] port-bridge relay (100 % ping); +* `delay 100` lands as netem on both anchors, measured RTT grows ≥ 150 ms + and returns < 50 ms when cleared; +* suspend admin-downs the anchor and kills the traffic; resume restores; +* AF_PACKET capture writes a real pcap of the ICMP exchange; +* link delete/re-create swaps the port bridge's TAP leg out and back in + (unix binding never re-binds) with the ping returning; +* node stop tears the anchors down (no `gx` devices left), a restart + recreates the whole wiring from the NIO; +* project delete leaves no anchors and no bridges; the relay control on a + relay-configured instance keeps the link off the kernel and still pings. + +Also caught and fixed live: `tap create` is strictly create-only +(`IFF_TUN_EXCL`) — the ensure-then-add guard must check existence first +(`/sys/class/net`), not re-create unconditionally (EBUSY on the device the +start loop just made). + +## Configuration + +``` +[Server] +enable_kernel_datapath = True # the global switch (link attachment) +``` + +The anchor lifecycle needs no configuration of its own — the capability +probes decide per compute, and an IOL runner container without them simply +never anchors. + +## Roadmap + +Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until +every node type is kernelized; with the IOL runner container anchored, that +condition is one project away from met. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 17be06e3c..c1b531a98 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -172,8 +172,9 @@ enable_kernel_datapath = True Dynamips has landed (see `dynamips-kernel-datapath.md` — the hypervisor opens uBridge-created TAPs, the same shape QEMU uses), and so has the Ethernet switch (see `ethernet-switch-kernel-datapath.md` — it absorbs the -peer's anchor into its own kernel bridge). What remains: switch-to-switch -cascades (a veth pair between the two kernel bridges) and the IOL Docker -node (same `bridge add_nio_tap` shape as this one). Cross-compute kernel -links need VXLAN/GENEVE encapsulation — deferred until every node type is -kernelized. +peer's anchor into its own kernel bridge). Switch-to-switch cascades (a veth +pair between the two kernel bridges) have landed, and so has the IOL runner +container — see `iol-docker-kernel-datapath.md` (same persistent-TAP shape, +through the generic bridge module's swappable TAP leg instead of the IOL +bridge). Cross-compute kernel links need VXLAN/GENEVE encapsulation — +deferred until every node type is kernelized. diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 91ee4fc8d..0c9441209 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -182,5 +182,8 @@ hypervisor opens uBridge-created TAPs with `nio create_tap`: the same external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay). **The Ethernet switch has landed** — see `docs/features/ethernet-switch-kernel-datapath.md` (the switch absorbs a -peer's anchor into its own kernel bridge; switch-to-switch cascades stay -relay for now). +peer's anchor into its own kernel bridge; switch-to-switch cascades too). +**The IOL runner container has landed** — see +`docs/features/iol-docker-kernel-datapath.md` (persistent TAP anchors +through the generic bridge module's swappable TAP leg; the unix-socket guest +leg keeps one userspace hop, like IOU's fabric). From 30cb7119e555037d62eef8c9286168e609a5b8d7 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 14:36:52 +0800 Subject: [PATCH 049/101] test(e2e): assert the L2-anchor spec's server-side guarantees The spec's gns3-server alignment section promised three assertions the suites never carried; they land in harness.py and run in every kernel scenario (plus the relay controls' anchor checks): - E.1 assert_pure_l2: anchors and bridges carry no L3 identity (addrgenmode none, no address in /proc/net/if_inet6, no ip -o -4 addr) -- the kernel scenarios' anchors and per-link/switch bridges, relay-mode anchors and the switch's relay TAPs alike (hardening is a creation-time property, not a datapath one); - E.2 assert_idle_silence (Dynamips suite): guest interfaces shut to silence IOS's own CDP/keepalives, settle 2.5 s, then a 5 s window of zero traffic over both anchors and the per-link bridge; - assert_forwarding: every attached port reports brport/state == 3 -- a bridge left DOWN keeps its ports disabled and forwards nothing, with no error anywhere; the assertion is the only signal. harness.l2only_supported() probes a throwaway uBridge once (link l2only against a nonexistent device: 208 = command present, 202-Unknown command = old build); without the command the assertions skip -- the spec's degradation contract, mirroring the tc-capability skip. The first live run caught a fifth-creator gap: the switch's relay port TAPs (minted implicitly by bridge add_nio_tap) carried a live fe80::. Delivered uBridge-side per the spec (fb72758: the transient TAP is hardened at creation, attaches untouched); the server needed no change, and the relay-control E.1 check is that creator's end-to-end coverage. --- tests/e2e/README.md | 11 +- tests/e2e/harness.py | 136 ++++++++++++++++++ tests/e2e/test_dynamips_kernel_datapath.py | 36 +++++ .../test_ethernet_switch_kernel_datapath.py | 18 +++ tests/e2e/test_iol_docker_kernel_datapath.py | 16 +++ 5 files changed, 215 insertions(+), 2 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index f3adf7016..b7d2b7e06 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -35,8 +35,8 @@ and print where to look at it. `harness.py` holds the shared pieces — REST client, server lifecycle, the IOS console driver, host-side kernel inspections (bridge membership, tap -admin state, tc qdiscs) — so a new node type adds only its own scenario -module. +admin state, tc qdiscs, the L2-anchor spec §E assertions) — so a new node +type adds only its own scenario module. Current scenarios: @@ -54,5 +54,12 @@ Current scenarios: delete/re-create swap, node stop/start rewiring) and a relay negative control. +Every kernel scenario also asserts the L2-anchor spec's server-side +guarantees (`docs/design/ubridge-l2-anchor-spec.md` §E): anchors and bridges +carry no L3 identity (`assert_pure_l2`), joined ports are FORWARDING +(`assert_forwarding`), and — in the Dynamips suite — an idle link with the +guests silenced stays silent for 5 s (`assert_idle_silence`). These skip on +a uBridge without `link l2only` (probed once by `harness.l2only_supported()`). + Set `GNS3_E2E_DEBUG=1` to run the isolated instance with debug logging (the uBridge command stream lands in its `server.log`). diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index aef68c624..0cdd585c8 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -43,6 +43,7 @@ Environment knobs: test fails, and print their ids, for inspection. """ +import contextlib import json import os import re @@ -62,6 +63,9 @@ REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__fi # tc lives in /usr/sbin on most distributions; subprocesses do not always # inherit a PATH that has it. TC = shutil.which("tc", path=os.environ.get("PATH", "") + ":/usr/sbin:/sbin") +# Reading addresses with iproute2 needs no privileges; the L2 assertions +# below refuse to pass silently when it is missing. +IP = shutil.which("ip", path=os.environ.get("PATH", "") + ":/usr/sbin:/sbin") # IOS prompt as the last thing on the stream: any hostname/mode shape. PROMPT_RE = re.compile(r"[A-Za-z0-9\-_.()/]+[>#]\s*$") @@ -512,6 +516,138 @@ def bridge_members(bridge): return sorted(os.listdir(f"/sys/class/net/{bridge}/brif")) +# --------------------------------------------------------------------------- +# L2-anchor spec assertions (docs/design/ubridge-l2-anchor-spec.md §E) +# --------------------------------------------------------------------------- + +# uBridge's creators harden every host-side datapath device with +# ``link l2only``; these assertions verify the result from the server side. +# They need the probe below to confirm the command exists — the spec's +# degradation contract is a skip on old builds, where anchors keep the +# kernel-default L3 behaviour (measured: 6 frames of MLD/DAD noise per 2 s). + + +def l2only_supported(): + """Whether this host's uBridge knows ``link l2only`` (probed once). + + Sends the command for a nonexistent device to a throwaway uBridge: a + 208 reply means the command exists, ``202-Unknown command`` an old + build. Anything unreadable also degrades to False — the L2 assertions + skip rather than fail, mirroring the tc-capability degradation.""" + global _l2only_support + if _l2only_support is None: + _l2only_support = _probe_l2only() + return _l2only_support + + +_l2only_support = None + + +def _probe_l2only(): + ubridge = shutil.which("ubridge") + if not ubridge: + return False + port = _free_port() + with tempfile.TemporaryDirectory(prefix="gns3-e2e-l2probe-") as workdir: + # S603: resolved binary, fixed argv, no shell + proc = subprocess.Popen( # noqa: S603 + [ubridge, "-H", f"127.0.0.1:{port}"], + cwd=workdir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + sock = None + deadline = time.time() + 10 + while time.time() < deadline and sock is None: + if proc.poll() is not None: + return False + try: + sock = socket.create_connection(("127.0.0.1", port), timeout=1) + except OSError: + time.sleep(0.1) + if sock is None: + return False + with sock: + sock.settimeout(3) + sock.sendall(b"link l2only e2el2probe0\n") + buf = b"" + deadline = time.time() + 5 + while time.time() < deadline: + try: + chunk = sock.recv(4096) + except TimeoutError: + break + if not chunk: + break + buf += chunk + if re.match(rb"\d{3}-", buf.split(b"\n")[0]): + break + reply = buf.decode(errors="replace").splitlines()[0] if buf else "" + # 208 = the command ran and refused the nonexistent device + return reply.startswith("208-") + finally: + proc.terminate() + with contextlib.suppress(Exception): + proc.wait(timeout=5) + + +def assert_pure_l2(name): + """§E.1: a host-side datapath device (anchor or bridge) carries no L3 + identity — no IPv4/IPv6 address, and addrgenmode none so none can + self-provision (no DAD, MLD or RS is ever generated from it).""" + mode_path = f"/sys/class/net/{name}/addr_gen_mode" + mode = sysfs(mode_path) if os.path.exists(mode_path) else None + assert mode in (None, "1"), f"{name}: addr_gen_mode is {mode!r}, expected none ('1')" + if os.path.exists("/proc/net/if_inet6"): + with open("/proc/net/if_inet6") as f: + assigned = [line.strip() for line in f if line.split()[-1] == name] + assert not assigned, f"{name}: unexpected IPv6 address: {assigned}" + if not IP: + raise AssertionError("ip binary not found — cannot verify the anchor has no IPv4 address") + # S603: resolved binary, fixed argv, no shell + v4 = subprocess.run( # noqa: S603 + [IP, "-o", "-4", "addr", "show", "dev", name], capture_output=True, text=True, check=False + ).stdout.strip() + assert not v4, f"{name}: unexpected IPv4 address: {v4}" + + +def brport_state(name): + """The interface's bridge-port STP state (None when not enslaved); the + kernel reports 3 for forwarding.""" + path = f"/sys/class/net/{name}/brport/state" + return sysfs(path) if os.path.exists(path) else None + + +def assert_forwarding(name): + """Silent-failure guard found while validating the L2 spec: an attached + bridge port must be FORWARDING. A bridge device left DOWN keeps its + ports disabled and forwards nothing — with no error anywhere, this + assertion is the only signal.""" + state = brport_state(name) + assert state == "3", f"{name}: bridge port state {state!r}, expected forwarding (3)" + + +def packet_counters(name): + """(rx, tx) packet counters of a host interface.""" + return ( + int(sysfs(f"/sys/class/net/{name}/statistics/rx_packets")), + int(sysfs(f"/sys/class/net/{name}/statistics/tx_packets")), + ) + + +def assert_idle_silence(*names, settle=2.5, window=5.0): + """§E.2: after the one-shot enslavement burst settles (the IGMP/MLD + membership reports of the bridge are accepted residual, see §0), an + idle device in its normal role shows no traffic for *window* seconds. + The baseline without the hardening: 6 frames / 2 s, continuous.""" + time.sleep(settle) + before = {name: packet_counters(name) for name in names} + time.sleep(window) + noisy = {name: (before[name], packet_counters(name)) for name in names if packet_counters(name) != before[name]} + assert not noisy, f"idle devices saw traffic (before -> after): {noisy}" + + def link_bridge_name(link_id): """The per-link kernel bridge name the controller derives from a link id (see UDPLink._prepare).""" diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py index deb8305d4..0f8b82021 100644 --- a/tests/e2e/test_dynamips_kernel_datapath.py +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -71,6 +71,13 @@ def _boot(server, project_id, node_id, timeout=360): return console +def _eth_shutdown(console, shut): + console.run("conf t") + console.run("interface f1/0") + console.run("shutdown" if shut else "no shutdown") + console.run("end") + + def test_dynamips_kernel_datapath(): server = harness.live_server(kernel=True) compute = server.compute @@ -108,6 +115,19 @@ def test_dynamips_kernel_datapath(): assert harness.tap_exists(t1) and harness.tap_exists(t2), (t1, t2) assert harness.bridge_members(bridge) == sorted([t1, t2]), harness.bridge_members(bridge) + # L2-anchor spec §E.1: anchors and the per-link bridge carry no L3 + # identity (skipped on a uBridge without `link l2only` — the spec's + # degradation), and both ports are FORWARDING — the silent-failure + # guard against a bridge device left DOWN. + if harness.l2only_supported(): + harness.assert_pure_l2(t1) + harness.assert_pure_l2(t2) + harness.assert_pure_l2(bridge) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(t1) + harness.assert_forwarding(t2) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") out = c1.run("show ip int brief") @@ -116,6 +136,18 @@ def test_dynamips_kernel_datapath(): baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + # L2-anchor spec §E.2: shut the guest interfaces (silencing IOS's own + # CDP/keepalives — the only legitimate speakers on the segment) and + # demand the host side stays silent for 5 s: without the hardening + # the host's own MLD/DAD noise floods the link (6 frames / 2 s). + if harness.l2only_supported(): + for console in (c1, c2): + _eth_shutdown(console, shut=True) + harness.assert_idle_silence(t1, t2, bridge) + for console in (c1, c2): + _eth_shutdown(console, shut=False) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + # delay 100: netem on both anchors (each impairs one direction), # one-way ~100 ms => RTT grows by ~200 ms compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) @@ -237,6 +269,10 @@ def test_dynamips_relay_control(): assert harness.bridge_members(harness.link_bridge_name(lid)) is None if harness.tap_exists(t1): assert not harness.tap_up(t1), "an unattached relay anchor should sit down" + if harness.l2only_supported(): + # the L2 hardening is a creation-time property, not a + # datapath one: relay anchors are pure L2 too + harness.assert_pure_l2(t1) harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index 57a2b8f25..0e4a8fbfd 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -139,6 +139,18 @@ def test_ethernet_switch_kernel_fast_path(): baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + # L2-anchor spec §E.1: the absorbed anchors and the switch bridge + # carry no L3 identity (skipped on a uBridge without `link l2only`), + # and the joined ports are FORWARDING — the silent-failure guard. + if harness.l2only_supported(): + harness.assert_pure_l2(a1) + harness.assert_pure_l2(a2) + harness.assert_pure_l2(sw_bridge) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(a1) + harness.assert_forwarding(a2) + # VLAN programming on absorbed anchors: different access VLANs # isolate, the same VLAN bridges again. compute.call( @@ -274,6 +286,12 @@ def test_ethernet_switch_relay_control(): members = harness.bridge_members(sw_bridge) assert members == sorted([f"{sw_bridge}-0", f"{sw_bridge}-1"]), members assert a1 not in members + if harness.l2only_supported(): + # creation-time hardening applies on the relay too: the switch's + # own port TAPs and its bridge are pure L2 + for member in members: + harness.assert_pure_l2(member) + harness.assert_pure_l2(sw_bridge) harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py index 594ea2cae..b39484e6e 100644 --- a/tests/e2e/test_iol_docker_kernel_datapath.py +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -110,6 +110,18 @@ def test_iol_docker_kernel_datapath(): assert harness.tap_exists(a1) and harness.tap_exists(a2), (a1, a2) assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + # L2-anchor spec §E.1: anchors and the per-link bridge carry no L3 + # identity (skipped on a uBridge without `link l2only`), and both + # ports are FORWARDING — the silent-failure guard. + if harness.l2only_supported(): + harness.assert_pure_l2(a1) + harness.assert_pure_l2(a2) + harness.assert_pure_l2(bridge) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(a1) + harness.assert_forwarding(a2) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") out = c1.run("show ip int brief") @@ -236,6 +248,10 @@ def test_iol_docker_relay_control(): assert harness.bridge_members(harness.link_bridge_name(link["link_id"])) is None if harness.tap_exists(a1): assert not harness.tap_up(a1), a1 + if harness.l2only_supported(): + # the L2 hardening is a creation-time property, not a + # datapath one: relay anchors are pure L2 too + harness.assert_pure_l2(a1) harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") relay_ping = harness.wait_ping(c1, "10.1.1.2") From f13da59f235610ae5952c8e7e1c0f3b822eeac67 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 14:36:58 +0800 Subject: [PATCH 050/101] docs(ubridge): the fifth L2-only creator, and the call site that used it The spec's B table grows its fifth row: bridge add_nio_tap mints a transient TAP when the name is free and hardens it at creation, while an attach to an existing device (possibly a user-owned TAP with addresses) is left untouched. The rule paragraph spells out attach-is-not-a-creation. Delivery: uBridge fb72758, closing the gap the new e2e assertions found (live: the switch's gns3*-0 relay TAP carried fe80::). The server issues no link l2only of its own -- the switch call site now documents that the TAP is minted-and-hardened by add_nio_tap, and the ESW feature doc describes the uBridge-side close instead of a server workaround. --- docs/design/ubridge-l2-anchor-spec.md | 55 ++++++++++++++----- .../ethernet-switch-kernel-datapath.md | 9 +++ .../compute/builtin/nodes/ethernet_switch.py | 5 +- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/docs/design/ubridge-l2-anchor-spec.md b/docs/design/ubridge-l2-anchor-spec.md index 8c6afad71..279e5ebba 100644 --- a/docs/design/ubridge-l2-anchor-spec.md +++ b/docs/design/ubridge-l2-anchor-spec.md @@ -4,10 +4,11 @@ See LICENSE file for licensing information. --> > Frozen requirements spec for the **uBridge** project. Status: **delivered** -> (uBridge `e2e3155` — `link l2only` plus the creators that apply it) and -> verified from the server side: anchors and per-link bridges report -> `addrgenmode none`, `ip -4/-6 addr` are empty, and an idle anchor is silent -> once the one-shot membership burst of §0 has settled. +> (uBridge `e2e3155` — `link l2only` plus the creators that apply it; +> `fb72758` — the fifth creator: the TAP `bridge add_nio_tap` creates for a +> free name) and verified from the server side: anchors and per-link bridges +> report `addrgenmode none`, `ip -4/-6 addr` are empty, and an idle anchor is +> silent once the one-shot membership burst of §0 has settled. > > **Residual, accepted (measured after delivery):** enslaving a port still > makes the *bridge device* announce its multicast memberships — one IGMPv3 @@ -141,10 +142,16 @@ directly, and a caller cannot forget it: | `docker create_veth ` | the **host** end only | the guest end moves into a container netns and is deliberately untouched | | `link veth ` | both ends | both ends are host-side | | `brctl create ` | the bridge device itself | the fabric the anchors are enslaved to; the bridge's own link-local floods to every port | +| `bridge add_nio_tap
` | the TAP it **creates** (name free) | by-name `TUNSETIFF` creates a transient device when the name is free — cloud's bridge interfaces, and the relay swap's create-if-missing path. Hardened at creation; an attach to an existing device is left untouched (the caller may name a user-owned TAP that has addresses). | Rule for future creators: any command that creates a host-side device for the data plane hardens it, and the acceptance checks in §E apply to it. +An **attach** is not a creation: `bridge add_nio_tap` (and every other open +of a TAP that already exists) must leave that device's addresses alone. The +transient TAP `bridge add_nio_tap` creates for a free name is a creation and +is hardened like the rest — best-effort, before the NIO is handed back. + ## C. Implementation notes (netlink, not `/proc`) - Use `RTM_SETLINK` with `IFLA_AF_SPEC{ AF_INET6, IFLA_INET6_ADDR_GEN_MODE = @@ -198,17 +205,35 @@ data plane hardens it, and the acceptance checks in §E apply to it. ## gns3-server alignment (informational — not uBridge scope) -- **Call sites unchanged**: the four creators harden the devices; the server does - not issue `link l2only`, and no NIO/JSON schema, filter or MCP tool description - changes. This is host-side hygiene only. -- **e2e assertions to add** (kernel-datapath suites, alongside the existing - capture/marker/filter checks): - - anchors and per-link bridges have no IPv4/IPv6 addresses (§E.1); - - an idle anchor captures zero frames in 5 s (§E.2); +- **Call sites**: the creators harden the devices; the server issues no + `link l2only` itself, and no NIO/JSON schema, filter or MCP tool + description changes. This is host-side hygiene only. +- **The fifth creator was found by these servers' e2e assertions**: on a + build predating `fb72758`, the Ethernet switch's relay port TAPs + (`gns3{id}-N`, created implicitly by `bridge add_nio_tap`) carried a live + `fe80::` — the §E.1 assertion failed on exactly that device, live. The + close is uBridge-side (§B table row 5): the transient TAP is hardened at + creation, attaches stay untouched. The server side needed no change for + it — the assertion alone caught it, and it now guards the fixed build. +- **e2e assertions** (landed in `tests/e2e/harness.py`, asserted by the + kernel-datapath suites alongside the existing capture/marker/filter checks): + - anchors and per-link bridges have no IPv4/IPv6 addresses, addrgenmode + `none` (`assert_pure_l2`, §E.1) — on the kernel scenarios' anchors and + bridges, and on relay-mode anchors/switch TAPs alike (the hardening is a + creation-time property, not a datapath one). The switch relay-TAP + assertion is what exercises the fifth creator end-to-end (the same + uBridge creation path cloud's transient TAPs take); + - an idle link stays silent for 5 s after the settle window + (`assert_idle_silence`, §E.2) — measured with the guest interfaces shut + (silencing IOS's own CDP/keepalives) over the anchors and the bridge; - a silent-failure guard found while validating this spec: after attach, every - bridge port reports `brport/state == 3` (forwarding). A bridge device left - DOWN keeps its ports `DISABLED` (`state == 0`) and forwards nothing — with - no error anywhere, so the assertion is the only signal. + bridge port reports `brport/state == 3` (forwarding) (`assert_forwarding`). + A bridge device left DOWN keeps its ports `DISABLED` (`state == 0`) and + forwards nothing — with no error anywhere, so the assertion is the only + signal. - **Rollout**: with a uBridge that lacks the command the anchors keep today's behaviour (noise present). The e2e assertions skip when `link l2only` answers - `202-Unknown command`, mirroring the existing tc-capability degradation. + `202-Unknown command` (`harness.l2only_supported()` probes a throwaway + uBridge once), mirroring the existing tc-capability degradation. A build + that has the command but predates `fb72758` fails the switch relay-TAP + assertion instead of skipping: the gap is exactly what it exists to catch. diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index 7808129ee..406088b2e 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -41,6 +41,15 @@ on the relay: `GNS3_UNIX_SOCKET_NIO` containers); * cross-compute links, non-Ethernet ports: the common rules. +**Relay port TAPs.** The switch's own relay-path port TAPs (the +`gns3{id}-N` devices) are created *implicitly* by uBridge's +`bridge add_nio_tap` — a name that is free is minted as a transient TAP, +and uBridge hardens it L2-only at creation (the fifth creator of +`docs/design/ubridge-l2-anchor-spec.md` §B, uBridge `fb72758`). The server +issues nothing extra: without that hardening the device would take a +kernel-default IPv6 identity whose DAD/MLD noise floods the segment, which +is exactly what the e2e's relay-control §E.1 assertion guards against. + ## The wire format: one NIO names the other end's interface ``` diff --git a/gns3server/compute/builtin/nodes/ethernet_switch.py b/gns3server/compute/builtin/nodes/ethernet_switch.py index 48188bde1..d0fd0a478 100644 --- a/gns3server/compute/builtin/nodes/ethernet_switch.py +++ b/gns3server/compute/builtin/nodes/ethernet_switch.py @@ -386,7 +386,10 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): ubridge_bridge = self._ubridge_bridge_name(port_number) tap = self._tap_name(port_number) - # per-port uBridge relay -- uBridge holds the TAP fd + # per-port uBridge relay -- uBridge holds the TAP fd. Nothing created + # this TAP before: add_nio_tap mints it (transient, a free name) and + # hardens it L2-only at creation (ubridge-l2-anchor-spec §B, the fifth + # creator — the server issues no link l2only of its own). await self._ubridge_send(f"bridge create {ubridge_bridge}") await self._ubridge_send(f'bridge add_nio_tap {ubridge_bridge} "{tap}"') # enslave the same TAP to the kernel bridge (the cloud.py::_add_linux_ethernet move) From e0329ceee60670fb9faa7cb247640400a388b0c1 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 15:06:52 +0800 Subject: [PATCH 051/101] test(e2e): give the dynamips routers a detected idle-PC Without one every c7200 busy-waits a full CPU core for the whole scenario -- the tests create raw nodes and no template supplies a value (measured: constant ~100 % per router, up to ~6 minutes per dynamips scenario). harness.dynamips_idlepc() detects a value through the controller's auto_idlepc endpoint (the GUI idle-PC finder path: a throwaway project boots the image, candidate values are validated by measuring actual CPU usage) and caches it in ~/.cache/gns3-e2e/idlepc.json keyed by the image checksum, so the cost (~100 s here) is paid once per image, not per run. GNS3_E2E_IDLEPC overrides detection; any failure returns None and the scenarios run without one rather than fail on a CPU optimisation. All four scenarios that boot c7200 routers (dynamips kernel/relay, ethernet switch kernel/relay) pass the value at creation. Live: detected 0x60189234 for the M11 image (238 s first run including detection); the routers drop from ~100 % during boot to 3-8 % steady (sampled per process), and a re-run reports the cached value with no detection. --- tests/e2e/README.md | 9 +++ tests/e2e/harness.py | 80 ++++++++++++++++++- tests/e2e/test_dynamips_kernel_datapath.py | 10 ++- .../test_ethernet_switch_kernel_datapath.py | 12 +-- 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index b7d2b7e06..c5fcfc4b6 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -33,6 +33,15 @@ Either way each test creates its own project and deletes it again. Set `GNS3_E2E_KEEP=1` to keep the project (and a started instance) on failure and print where to look at it. +The dynamips scenarios give their routers an idle-PC: detected once per +image through the controller's `auto_idlepc` (a throwaway project and a +real CPU-usage measurement — the GUI's idle-PC finder path) and cached in +`~/.cache/gns3-e2e/idlepc.json` keyed by the image checksum. Without one, +each c7200 burns a **full CPU core** for the whole scenario — the e2e +creates raw nodes, so no template supplies a value. Set `GNS3_E2E_IDLEPC` +to skip the detection and use the given value (detection failing is never +fatal: the routers simply run without one). + `harness.py` holds the shared pieces — REST client, server lifecycle, the IOS console driver, host-side kernel inspections (bridge membership, tap admin state, tc qdiscs, the L2-anchor spec §E assertions) — so a new node diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 0cdd585c8..07391a45a 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -39,6 +39,9 @@ Environment knobs: * ``GNS3_E2E_USER`` / ``GNS3_E2E_PASSWORD`` — credentials for the target. * ``GNS3_E2E_IMAGES`` — images directory the isolated instance uses (default ``~/GNS3/images``); also where the dynamips test finds its image. +* ``GNS3_E2E_IDLEPC`` — use this idle-PC value for the dynamips routers + instead of detecting one (see ``dynamips_idlepc``; without an idle-PC each + router burns a full CPU core for the whole scenario). * ``GNS3_E2E_KEEP=1`` — keep the project (and the isolated instance) when a test fails, and print their ids, for inspection. """ @@ -121,8 +124,10 @@ class Compute: def create_project(self, name="e2e"): return self.call("POST", "/projects", {"name": f"{name}-{int(time.time())}"}) - def create_dynamips_router(self, project_id, name, image, slots, platform="c7200", ram=512): + def create_dynamips_router(self, project_id, name, image, slots, platform="c7200", ram=512, idlepc=None): properties = {"platform": platform, "image": image, "ram": ram} + if idlepc: + properties["idlepc"] = idlepc properties.update(slots) return self.call( "POST", @@ -350,6 +355,79 @@ def release(server, project_id, failed=False): server.stop() +# --------------------------------------------------------------------------- +# Dynamips idle-PC (CPU) +# --------------------------------------------------------------------------- + +# Detected values, keyed by image identity; the detection needs a throwaway +# boot, so it is worth caching across runs. +IDLEPC_CACHE_PATH = os.path.join(os.path.expanduser("~/.cache/gns3-e2e"), "idlepc.json") + + +def dynamips_idlepc(server, image, platform="c7200", ram=512): + """The idle-PC value the scenarios' dynamips routers should be created + with — detected once per image and cached. + + Without one, dynamips busy-waits and burns a *full CPU core per router* + for the whole scenario: the e2e creates raw nodes, and no template + supplies an idle-PC. Detection goes through the controller's + ``auto_idlepc`` endpoint — the GUI's idle-PC finder path: a throwaway + project boots the image, candidate values are validated by measuring the + actual process CPU (first candidate under 70 % wins) — and the result is + cached in ``~/.cache/gns3-e2e/idlepc.json`` keyed by the image's + checksum, so the cost is paid once per image, not once per run. + + ``GNS3_E2E_IDLEPC`` overrides detection and cache entirely. Any failure + returns None: the scenarios then run without an idle-PC (and burn the + cores) rather than fail on a CPU optimisation. + """ + override = os.environ.get("GNS3_E2E_IDLEPC") + if override: + return override + + checksum = None + for entry in server.compute.call("GET", "/computes/local/dynamips/images"): + if entry["filename"] == image: + checksum = entry.get("md5sum") + break + key = f"{image}:{checksum}" if checksum else image + + cache = {} + try: + with open(IDLEPC_CACHE_PATH) as f: + cache = json.load(f) + except (OSError, ValueError): + pass + if cache.get(key): + print(f".. idle-PC {cache[key]} for {image} (cached)") + return cache[key] + + print(f".. detecting an idle-PC for {image} (once per image — this takes a few minutes)") + try: + result = server.compute.call( + "POST", + "/computes/local/dynamips/auto_idlepc", + {"platform": platform, "image": image, "ram": ram}, + timeout=600, + ) + except Exception as e: + print(f".. idle-PC detection failed ({e}); the routers will burn a core each") + return None + idlepc = (result or {}).get("idlepc") + if not idlepc: + print(f".. no idle-PC value found for {image}; the routers will burn a core each") + return None + cache[key] = idlepc + try: + os.makedirs(os.path.dirname(IDLEPC_CACHE_PATH), exist_ok=True) + with open(IDLEPC_CACHE_PATH, "w") as f: + json.dump(cache, f, indent=2, sort_keys=True) + except OSError as e: + print(f".. could not write the idle-PC cache {IDLEPC_CACHE_PATH}: {e}") + print(f".. idle-PC {idlepc} for {image} (cached for later runs)") + return idlepc + + # --------------------------------------------------------------------------- # IOS console (dynamips telnet console) # --------------------------------------------------------------------------- diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py index 0f8b82021..5e41d10eb 100644 --- a/tests/e2e/test_dynamips_kernel_datapath.py +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -84,13 +84,14 @@ def test_dynamips_kernel_datapath(): if compute.capabilities().get("ubridge_tap") is not True: pytest.skip("this compute's uBridge cannot create persistent TAPs (ubridge_tap != True)") image = _pick_image(server) + idlepc = harness.dynamips_idlepc(server, image) project = compute.create_project("dyn-e2e") pid = project["project_id"] t1 = t2 = None try: - r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS) - r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS) + r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS, idlepc=idlepc) + r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS, idlepc=idlepc) r1_id, r2_id = r1["node_id"], r2["node_id"] t1 = harness.anchor_name(r1_id, *ETH) t2 = harness.anchor_name(r2_id, *ETH) @@ -244,13 +245,14 @@ def test_dynamips_relay_control(): server = harness.live_server(kernel=False) compute = server.compute image = _pick_image(server) + idlepc = harness.dynamips_idlepc(server, image) project = compute.create_project("dyn-e2e-relay") pid = project["project_id"] t1 = None try: - r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS) - r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS) + r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS, idlepc=idlepc) + r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS, idlepc=idlepc) r1_id, r2_id = r1["node_id"], r2["node_id"] t1 = harness.anchor_name(r1_id, *ETH) diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index 0e4a8fbfd..2cf98905d 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -67,7 +67,7 @@ def _switch_bridge_name(switch_id): return "gns3" + switch_id.replace("-", "")[:6] -def _create_topology(compute, pid, image, name): +def _create_topology(compute, pid, image, name, idlepc=None): switch = compute.call( "POST", f"/projects/{pid}/nodes", @@ -83,8 +83,8 @@ def _create_topology(compute, pid, image, name): }, }, ) - r1 = compute.create_dynamips_router(pid, f"{name}-R1", image, SLOTS) - r2 = compute.create_dynamips_router(pid, f"{name}-R2", image, SLOTS) + r1 = compute.create_dynamips_router(pid, f"{name}-R1", image, SLOTS, idlepc=idlepc) + r2 = compute.create_dynamips_router(pid, f"{name}-R2", image, SLOTS, idlepc=idlepc) return switch, r1, r2 @@ -100,12 +100,13 @@ def test_ethernet_switch_kernel_fast_path(): if compute.capabilities().get("ubridge_tap") is not True: pytest.skip("this compute's uBridge cannot create persistent TAPs (ubridge_tap != True)") image = _pick_image(server) + idlepc = harness.dynamips_idlepc(server, image) project = compute.create_project("esw-e2e") pid = project["project_id"] sw_bridge = None try: - switch, r1, r2 = _create_topology(compute, pid, image, "E2E") + switch, r1, r2 = _create_topology(compute, pid, image, "E2E", idlepc) switch_id = switch["node_id"] r1_id, r2_id = r1["node_id"], r2["node_id"] sw_bridge = _switch_bridge_name(switch_id) @@ -262,12 +263,13 @@ def test_ethernet_switch_relay_control(): server = harness.live_server(kernel=False) compute = server.compute image = _pick_image(server) + idlepc = harness.dynamips_idlepc(server, image) project = compute.create_project("esw-e2e-relay") pid = project["project_id"] sw_bridge = None try: - switch, r1, r2 = _create_topology(compute, pid, image, "E2E") + switch, r1, r2 = _create_topology(compute, pid, image, "E2E", idlepc) sw_bridge = _switch_bridge_name(switch["node_id"]) r1_id, r2_id = r1["node_id"], r2["node_id"] a1 = harness.anchor_name(r1_id, *ETH) From 897f50e4306ae892f0d19c6aa935ab7795bd2f50 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 16:23:14 +0800 Subject: [PATCH 052/101] test(e2e): live coverage for the standard Docker node kernel datapath Two real Alpine containers on an isolated instance, addressed the way a GNS3 user does it (the node's persistent /etc/network/interfaces, applied by init.sh) and driven through the Docker daemon socket the server itself uses. Four scenarios: * fast path: the per-link kernel bridge enslaves exactly the two veth host ends, real ICMP crosses, the L2-anchor spec E.1/E.2 assertions hold (idle silence measured with the guests' links down -- a container's own boot-time IGMP report re-fires ~10 s later, so silencing is not just IPv6), netem delay, suspend/resume, capture, link delete/re-create, container stop/start (the veth host ends die with the adapter and are rebuilt from the NIO), zero residue on project delete; * filter matrix, one type at a time, each asserted through real traffic and its tc state: netem core (delay, jitter+distribution, loss, corrupt, duplicate), the extensions (reorder+correl+gap, seed+limit, gemodel, rate), cls_bpf match-drop including a byte-size discriminator, the eBPF classifier (frequency_drop every-Nth and -1, quota byte-cap cutoff, window_drop single outage with per-seq pass/drop/pass evidence), plus available_filters offering the kernel-only types; * relay control: no per-link bridge, the kernel-only types hidden; * relay->kernel reopen upgrade: the datapath choice is flipped across a server restart and the project reopened -- the link is rebuilt on the kernel datapath and forwards. The harness gains a Docker guest driver (a stdlib Engine API client over the daemon socket: exec with stream demux, container id by deterministic name -- the controller's node payload does not carry it, image inspect), a digest-pinned image helper (cache hit = no network, miss = pull through the server's own route, neither = skip) and Server.restart() for the reopen scenario (which also needs the local compute's capabilities to be up before a project open is accepted). Live: 4/4 scenarios, 142 s for the file; full e2e suite 10/10 (1503 s); unit suite 2149 passed; ruff clean. The matrix measures loss with 20 packets at 0.2 s spacing (143 s -> 58 s) and the jitter RTT assertion keeps statistical margin (per-direction N(50,20); 10 packets, >= 65 ms). --- tests/e2e/README.md | 27 +- tests/e2e/harness.py | 322 +++++++++++++- tests/e2e/test_docker_kernel_datapath.py | 544 +++++++++++++++++++++++ 3 files changed, 885 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/test_docker_kernel_datapath.py diff --git a/tests/e2e/README.md b/tests/e2e/README.md index c5fcfc4b6..fd5a6f6ff 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -27,7 +27,8 @@ Two run modes, chosen per test through `harness.live_server(kernel=...)`: process on a free port with its own config directory (the controller DB and every path follow the config), so nothing on the machine is touched. It needs `ubridge`, `dynamips` and an images directory - (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally. + (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally; the + Docker scenarios additionally need the Docker daemon the server talks to. Either way each test creates its own project and deletes it again. Set `GNS3_E2E_KEEP=1` to keep the project (and a started instance) on failure @@ -43,9 +44,11 @@ to skip the detection and use the given value (detection failing is never fatal: the routers simply run without one). `harness.py` holds the shared pieces — REST client, server lifecycle, the -IOS console driver, host-side kernel inspections (bridge membership, tap -admin state, tc qdiscs, the L2-anchor spec §E assertions) — so a new node -type adds only its own scenario module. +IOS console driver, the Docker guest driver (a tiny Docker Engine API +client over the daemon socket, plus the digest-pinned image helper), +host-side kernel inspections (bridge membership, tap admin state, tc +qdiscs, the L2-anchor spec §E assertions) — so a new node type adds only +its own scenario module. Current scenarios: @@ -62,6 +65,22 @@ Current scenarios: TAP leg (anchors at start, netem delay, suspend, capture, link delete/re-create swap, node stop/start rewiring) and a relay negative control. +* `test_docker_kernel_datapath.py` — two real Alpine containers (the plain + veth datapath): the per-link kernel bridge enslaves exactly the two veth + host ends, real ICMP crosses, the filter matrix runs one type at a time + (netem core and extensions, cls_bpf match-drop, the eBPF classifier — + each measured through real traffic), plus a relay negative control and a + relay→kernel reopen upgrade (the server's datapath choice is flipped + across a restart). + +The Docker scenarios run a digest-pinned `alpine:3`, so every developer +tests the same bytes: a cached copy in the local Docker daemon costs no +network, a cache miss pulls through the server's own image-pull route, and +with neither cache nor registry the scenario skips (`GNS3_E2E_DOCKER_IMAGE` +overrides the reference). Their guests are addressed the way a GNS3 user +does it — the node's persistent `/etc/network/interfaces`, which the +container's `init.sh` applies at boot — and driven through the Docker +daemon socket the server itself uses. Every kernel scenario also asserts the L2-anchor spec's server-side guarantees (`docs/design/ubridge-l2-anchor-spec.md` §E): anchors and bridges diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 07391a45a..b3872f0f5 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -42,11 +42,18 @@ Environment knobs: * ``GNS3_E2E_IDLEPC`` — use this idle-PC value for the dynamips routers instead of detecting one (see ``dynamips_idlepc``; without an idle-PC each router burns a full CPU core for the whole scenario). +* ``GNS3_E2E_DOCKER_IMAGE`` — the image the Docker scenarios run (default + ``alpine:3``, pinned by digest and pulled on a cache miss; see + ``ensure_docker_image``). +* ``GNS3_E2E_DOCKER_SOCKET`` — the Docker daemon socket the Docker guest + driver talks to (default ``/var/run/docker.sock`` — the same daemon the + server uses). * ``GNS3_E2E_KEEP=1`` — keep the project (and the isolated instance) when a test fails, and print their ids, for inspection. """ import contextlib +import http.client import json import os import re @@ -57,6 +64,7 @@ import sys import tempfile import time import urllib.error +import urllib.parse import urllib.request import pytest @@ -135,6 +143,20 @@ class Compute: {"compute_id": "local", "name": name, "node_type": "dynamips", "properties": properties}, ) + def create_docker_node(self, project_id, name, image, adapters=1): + """A standard Docker node (the veth datapath): the scenario addresses + its eth0 from the host and pings through it.""" + return self.call( + "POST", + f"/projects/{project_id}/nodes", + { + "compute_id": "local", + "name": name, + "node_type": "docker", + "properties": {"image": image, "adapters": adapters, "console_type": "telnet"}, + }, + ) + def create_iol_router(self, project_id, name, image, adapters=2): """An IOL runner container (iol-runner images), the appliance's template properties.""" @@ -244,7 +266,7 @@ class Server: with open(path) as f: return "".join(f.readlines()[-lines:]) - def stop(self): + def _stop_process(self): if self.process and self.process.poll() is None: self.process.terminate() try: @@ -252,11 +274,30 @@ class Server: except subprocess.TimeoutExpired: self.process.kill() self.process.wait(timeout=10) + + def stop(self): + self._stop_process() if self.tmpdir: shutil.rmtree(self.tmpdir, ignore_errors=True) + def restart(self, kernel=True): + """Restart an isolated instance in place: stop the process — the + temporary directory (config, controller DB, projects) survives — and + start a new one on the same paths with the given kernel-datapath + setting and a fresh port. This is how the reopen-upgrade scenario + flips the datapath choice and reopens a project created on the + relay.""" + if not self.isolated: + raise AssertionError("restart() needs an isolated instance (the harness did not start one)") + self._stop_process() + self.process, self.compute = _spawn_isolated(self.tmpdir, kernel) + return self -def _start_isolated(kernel): + +def _spawn_isolated(tmpdir, kernel): + """Start one isolated instance inside *tmpdir* (its config is written + there; a fresh free port each boot) and return ``(process, compute)``, + logged in. Shared by the first boot and by ``Server.restart``.""" ubridge = shutil.which("ubridge") dynamips = shutil.which("dynamips") if not ubridge: @@ -267,7 +308,6 @@ def _start_isolated(kernel): if not os.path.isdir(images): pytest.skip(f"images directory not found: {images} (set GNS3_E2E_IMAGES)") - tmpdir = tempfile.mkdtemp(prefix="gns3-e2e-") os.makedirs(os.path.join(tmpdir, "projects"), exist_ok=True) port = _free_port() config = ISOLATED_CONFIG.format( @@ -277,7 +317,7 @@ def _start_isolated(kernel): with open(config_path, "w") as f: f.write(config) - log = open(os.path.join(tmpdir, "server.log"), "w") + log = open(os.path.join(tmpdir, "server.log"), "a") # S603: fixed argv, this interpreter, no shell involvement argv = [sys.executable, "-m", "gns3server", "--config", config_path] if os.environ.get("GNS3_E2E_DEBUG"): @@ -307,6 +347,17 @@ def _start_isolated(kernel): raise AssertionError("isolated server did not come up within 60s") compute.login() + # The controller's connection to its own local compute comes up + # asynchronously; until it reports capabilities the compute counts as + # disconnected and a project open is refused. Same wait as live_server. + if not wait_until(lambda: (compute.capabilities() or {}).get("version"), timeout=45): + raise AssertionError(f"{compute.url}: the controller's local compute did not report capabilities within 45s") + return process, compute + + +def _start_isolated(kernel): + tmpdir = tempfile.mkdtemp(prefix="gns3-e2e-") + process, compute = _spawn_isolated(tmpdir, kernel) return Server(compute, process, tmpdir) @@ -566,6 +617,262 @@ def wait_ping(console, target, attempts=4, repeat=3, timeout=1): return result +# --------------------------------------------------------------------------- +# Docker guest driver (Docker Engine API over the daemon socket) +# --------------------------------------------------------------------------- + +# The e2e needs two things the server's REST surface does not expose: the +# identity of the image a container runs (to verify a cached image is the +# pinned build) and a way to run commands inside a booted container (address +# the guest, drive real ICMP from it). Both are plain Engine API calls over +# the same socket the server itself uses. + +DOCKER_SOCKET = os.environ.get("GNS3_E2E_DOCKER_SOCKET", "/var/run/docker.sock") + +# The image the Docker scenarios run. Both the registry digest (what to pull +# on a cache miss; a rolling tag resolved once and then pinned forever) and +# the repo-digest it must resolve to locally are pinned, so every developer +# tests the same bytes. The Docker daemon is the cache: a hit costs no +# network, a miss pulls through the server's own route (the WebUI's path), +# and no cache + no registry skips the scenario. +DOCKER_IMAGE = "alpine:3" +DOCKER_IMAGE_DIGEST = "sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + + +class _UnixHTTPConnection(http.client.HTTPConnection): + """An HTTP connection speaking over a Unix socket (stdlib only).""" + + def __init__(self, socket_path, timeout): + super().__init__("localhost", timeout=timeout) + self._socket_path = socket_path + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect(self._socket_path) + + +class DockerDaemon: + """Minimal Docker Engine API client over the daemon socket.""" + + def __init__(self, socket_path=DOCKER_SOCKET, timeout=60): + self.socket_path = socket_path + self.timeout = timeout + + def _request(self, method, path, body=None, timeout=None): + conn = _UnixHTTPConnection(self.socket_path, timeout or self.timeout) + try: + data = None + headers = {} + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + conn.request(method, path, body=data, headers=headers) + response = conn.getresponse() + return response.status, response.read() + except (OSError, http.client.HTTPException) as e: + raise AssertionError(f"docker {method} {path} failed: {e}") from None + finally: + conn.close() + + def container_id(self, name): + """The id of the container called *name* (any state), or None. The + controller's node payload does not carry a Docker node's container + id (a compute-side read-only field); the daemon knows the container + by the deterministic name the server gives it (DockerVM.docker_name: + ``GNS3..``).""" + filters = urllib.parse.quote(json.dumps({"name": [name]}), safe="") + status, payload = self._request("GET", f"/containers/json?all=1&filters={filters}") + assert status == 200, f"docker container list -> HTTP {status}: {payload[:300]!r}" + for container in json.loads(payload): + if name in [entry.lstrip("/") for entry in container.get("Names") or []]: + return container["Id"] + return None + + def image_inspect(self, reference): + """The image record stored under *reference*, or None.""" + status, payload = self._request("GET", "/images/" + urllib.parse.quote(reference, safe="") + "/json") + if status == 404: + return None + assert status == 200, f"docker image inspect {reference} -> HTTP {status}: {payload[:300]!r}" + return json.loads(payload) + + def pull(self, reference, timeout=600): + """Pull *reference* (digest-pinned when the caller wants + reproducibility). Raises AssertionError on the daemon's error + stream — offline, unknown manifest, ...""" + status, payload = self._request( + "POST", "/images/create?fromImage=" + urllib.parse.quote(reference, safe=""), timeout=timeout + ) + if status != 200: + raise AssertionError(f"docker pull {reference} -> HTTP {status}: {payload[:300]!r}") + # the body is a stream of JSON progress objects; any object carrying + # an error key (in either shape) is a failure + text = payload.decode(errors="replace").strip() + decoder = json.JSONDecoder() + while text: + try: + obj, index = decoder.raw_decode(text) + except ValueError: + break + error = obj.get("error") or (obj.get("errorDetail") or {}).get("message") + if error: + raise AssertionError(f"docker pull {reference} failed: {error}") + text = text[index:].strip() + + def exec(self, container_id, cmd, timeout=60): + """Run *cmd* inside a container; returns (exit_code, combined + output).""" + status, payload = self._request( + "POST", + f"/containers/{container_id}/exec", + {"AttachStdout": True, "AttachStderr": True, "Tty": False, "Cmd": list(cmd)}, + ) + assert status == 201, f"docker exec create in {container_id[:12]} -> HTTP {status}: {payload[:300]!r}" + exec_id = json.loads(payload)["Id"] + try: + status, payload = self._request( + "POST", f"/exec/{exec_id}/start", {"Detach": False, "Tty": False}, timeout=timeout + ) + except AssertionError as e: + # A stalled stream is diagnosable: was the process still running + # (stream open, nothing flushed) or already gone? + raise AssertionError(f"{e} (exec {exec_id[:12]} still running: {self._exec_running(exec_id)})") from None + assert status == 200, f"docker exec start {exec_id[:12]} -> HTTP {status}: {payload[:300]!r}" + # the hijacked stream multiplexes stdout/stderr: 8-byte frame headers + # (stream id + big-endian length) — demultiplex into one buffer + output = bytearray() + index = 0 + while index + 8 <= len(payload): + length = int.from_bytes(payload[index + 4 : index + 8], "big") + output += payload[index + 8 : index + 8 + length] + index += 8 + length + status, payload = self._request("GET", f"/exec/{exec_id}/json") + assert status == 200, f"docker exec inspect {exec_id[:12]} -> HTTP {status}: {payload[:300]!r}" + return json.loads(payload)["ExitCode"], output.decode(errors="replace") + + def _exec_running(self, exec_id): + """Whether an exec is still running (diagnostic only, never raises).""" + try: + status, payload = self._request("GET", f"/exec/{exec_id}/json", timeout=10) + return json.loads(payload).get("Running") if status == 200 else f"HTTP {status}" + except (AssertionError, ValueError): + return "unknown" + + +def ensure_docker_image(server): + """The Docker image the scenarios run, ensured present locally. + + The pinned reference (``DOCKER_IMAGE``, digest ``DOCKER_IMAGE_DIGEST``) + is pulled through the server's own pull route on a cache miss and + verified against the pinned repo digest afterwards. A cached image that + resolves to another digest is used with a warning if the pull fails + (offline developer), but with no cached image and no registry the + scenario skips. ``GNS3_E2E_DOCKER_IMAGE`` overrides the reference + entirely (the operator then owns its content).""" + + override = os.environ.get("GNS3_E2E_DOCKER_IMAGE") + daemon = DockerDaemon() + if override: + info = daemon.image_inspect(override) + if info is None: + pytest.skip(f"GNS3_E2E_DOCKER_IMAGE={override} is not present on the Docker daemon") + print(f".. using GNS3_E2E_DOCKER_IMAGE={override} ({info['Id'][:19]}…)") + return override + + reference = f"{DOCKER_IMAGE}@{DOCKER_IMAGE_DIGEST}" + info = daemon.image_inspect(DOCKER_IMAGE) + if info is not None and any(DOCKER_IMAGE_DIGEST in d for d in (info.get("RepoDigests") or [])): + print(f".. Docker image {DOCKER_IMAGE} is cached at the pinned digest ({info['Id'][:19]}…)") + return DOCKER_IMAGE + if info is None: + print(f".. Docker image {DOCKER_IMAGE} is not cached — pulling the pinned digest") + else: + print( + f".. cached {DOCKER_IMAGE} is {info.get('RepoDigests') or info['Id'][:19]}…, not the pinned digest — pulling" + ) + try: + server.compute.call("POST", "/computes/local/docker/images/pull", {"image": reference}, timeout=600) + except AssertionError as e: + if info is None: + pytest.skip(f"Docker image {DOCKER_IMAGE} is not cached and could not be pulled: {e}") + print(f".. pull failed ({e}); using the cached {DOCKER_IMAGE} — not the pinned build") + return DOCKER_IMAGE + print(f".. pulled {reference}") + return DOCKER_IMAGE + + +def configure_docker_interfaces(node, address, netmask="255.255.255.0", adapter=0): + """Give the container's eth{adapter} a static address the way a GNS3 + user does: write the persistent ``/etc/network/interfaces`` the server + bind-mounts into the container and init.sh applies with busybox ifup at + boot (CIDR is not understood — separate address/netmask lines). Call + before starting the node.""" + directory = node.get("node_directory") + if not directory or not os.path.isdir(directory): + pytest.skip(f"cannot reach the node directory {directory!r} — this scenario needs the compute on this host") + path = os.path.join(directory, "etc", "network", "interfaces") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(f"auto eth{adapter}\niface eth{adapter} inet static\n\taddress {address}\n\tnetmask {netmask}\n") + + +def docker_wait_address(daemon, container_id, address, timeout=30): + """Wait until the container's eth0 carries *address* — init.sh applies + the interfaces file (busybox ifup) a moment after container start.""" + + def present(): + _code, output = daemon.exec(container_id, ["ip", "-4", "-o", "addr", "show", "dev", "eth0"]) + return address in output + + if not wait_until(present, timeout=timeout, interval=0.5): + raise AssertionError(f"container {container_id[:12]} did not configure {address} within {timeout}s") + + +_BUSYBOX_PING_LOSS_RE = re.compile(r"(\d+)% packet loss") +_BUSYBOX_PING_RTT_RE = re.compile(r"round-trip min/avg/max = [\d.]+/([\d.]+)/[\d.]+ ms") + + +def docker_ping(daemon, container_id, target, count=3, timeout=1, interval=0.2, size=None): + """Run a real busybox ping inside a container; returns + ``{loss, avg, raw, exit}`` (loss in percent, avg in ms). The default + 0.2 s interval packs more samples into loss measurements; loss-based + checks pass a higher count. + + The ping is wrapped in busybox `timeout` with a generous cap so a + misbehaving guest can never hold the exec stream open past it — the + harness then reports a normal failed ping instead of a socket timeout. + """ + cap = int(count * interval + timeout + 15) + cmd = ["timeout", str(cap), "ping", "-c", str(count), "-W", str(timeout)] + if interval is not None: + cmd += ["-i", str(interval)] + if size is not None: + cmd += ["-s", str(size)] + cmd.append(target) + code, output = daemon.exec(container_id, cmd, timeout=cap + 10) + loss = _BUSYBOX_PING_LOSS_RE.search(output) + rtt = _BUSYBOX_PING_RTT_RE.search(output) + return { + "loss": int(loss.group(1)) if loss else 100, + "avg": float(rtt.group(1)) if rtt else None, + "raw": output, + "exit": code, + } + + +def docker_wait_ping(daemon, container_id, target, attempts=4, **kwargs): + """ping until it fully succeeds (a fresh veth/bridge needs a beat for + carrier and ARP); returns the last result either way.""" + result = None + for _ in range(attempts): + result = docker_ping(daemon, container_id, target, **kwargs) + if result["loss"] == 0: + return result + time.sleep(1.5) + return result + + # --------------------------------------------------------------------------- # Host-side inspection (same machine as the compute) # --------------------------------------------------------------------------- @@ -737,6 +1044,13 @@ def anchor_name(node_id, adapter, port): return "gd" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" +def docker_anchor_name(node_id, adapter, port=0): + """The standard Docker node's anchor: the veth host end the server keeps + in the root namespace (see DockerKernelDatapathMixin._veth_names and + utils.kernel_anchor).""" + return "gv" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" + + def iol_anchor_name(node_id, adapter, port): """The IOL runner container's anchor TAP name (see IOLDockerVM._tap_name).""" return "gx" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" diff --git a/tests/e2e/test_docker_kernel_datapath.py b/tests/e2e/test_docker_kernel_datapath.py new file mode 100644 index 000000000..a56101a91 --- /dev/null +++ b/tests/e2e/test_docker_kernel_datapath.py @@ -0,0 +1,544 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Live end-to-end tests for the standard Docker node's kernel datapath (pytest +marker ``e2e``): two real Alpine containers, addressed from the host the way +a GNS3 user does (the node's persistent ``/etc/network/interfaces``, applied +by the container's init.sh) and driven through the Docker daemon socket the +server itself uses. The scenarios cover: + +* ``test_docker_kernel_fast_path`` — the veth datapath itself: the link + reports ``kernel_datapath``, the per-link kernel bridge enslaves exactly + the two veth host ends, real ICMP crosses kernel-only, the L2-anchor spec + §E assertions hold (no L3 identity, FORWARDING ports, idle silence), a + ``delay 100`` filter lands as netem on both host ends, suspend kills the + traffic at the carrier, capture writes a real pcap, link delete/re-create + and container stop/start leave the wiring consistent, and deleting the + project leaves no veth or bridge behind. +* ``test_docker_kernel_filter_matrix`` — one filter type at a time on a live + kernel link, each asserted by its effect on real traffic (RTT, loss, + per-seq outage evidence) and its tc state, from netem core and the netem + extensions to the cls_bpf and eBPF classifiers; the kernel-only types are + offered by ``available_filters`` and everything restores cleanly. +* ``test_docker_relay_control`` — the negative control on an isolated + relay-configured instance: no per-link kernel bridge, the wire is the + uBridge relay, and the kernel-only filter types are hidden. +* ``test_docker_relay_to_kernel_reopen_upgrade`` — the server's datapath + choice is flipped (relay → kernel) across a restart and the project is + reopened: the link is rebuilt on the kernel datapath and forwards. + +The image is pinned by digest (``harness.ensure_docker_image``): a cached +copy costs no network, a miss pulls through the server's own pull route, +and no cache + no registry skips the scenario. +""" + +import os +import re +import time + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# every standard Docker adapter has a single port +ETH = (0, 0) +R1_IP = "10.1.1.1" +R2_IP = "10.1.1.2" + + +def _resolve_container(daemon, node): + """Fill in the node's container id from the daemon (the controller's + node payload does not carry the compute-side id), by the deterministic + name the server gives the container.""" + container_id = daemon.container_id(f"GNS3.{node['name']}.{node['project_id']}") + assert container_id, f"container for {node['name']} not found on the Docker daemon" + node["container_id"] = container_id + return node + + +def _create_topology(server, pid, image, name): + """Two containers with addressed eth0s. The harness reaches their guests + through the Docker daemon socket — the same socket the server uses.""" + compute = server.compute + daemon = harness.DockerDaemon() + nodes = [] + for suffix, address in (("1", R1_IP), ("2", R2_IP)): + node = compute.create_docker_node(pid, f"{name}-{suffix}", image) + harness.configure_docker_interfaces(node, address) + nodes.append(_resolve_container(daemon, node)) + return nodes[0], nodes[1], daemon + + +def _start_addresses(compute, pid, daemon, nodes, addresses): + """Start the containers and wait for init.sh to have applied their + addresses (busybox ifup runs a moment after container start).""" + for node in nodes: + compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start") + for node, address in zip(nodes, addresses, strict=False): + harness.docker_wait_address(daemon, node["container_id"], address) + + +def _put_filters(compute, pid, lid, filters): + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": filters}) + + +def _clear_filters(compute, pid, lid, *anchors): + """Clear the link's filters and assert the teardown reached the kernel: + no netem qdisc and no clsact left on the anchors.""" + _put_filters(compute, pid, lid, {}) + for anchor in anchors: + assert "netem" not in harness.qdiscs(anchor), harness.qdiscs(anchor) + assert "clsact" not in harness.qdiscs(anchor), harness.qdiscs(anchor) + + +def _filter_types(compute, pid, lid): + return {entry["type"] for entry in compute.call("GET", f"/projects/{pid}/links/{lid}/available_filters")} + + +def _silence_guest(daemon, container_id): + """Silence the guest for the idle-silence measurement — the docker + analogue of shutting the IOS interfaces in the dynamips scenario. The + link goes down first (a guest's own chatter is not just IPv6: Linux + re-sends its boot-time IGMP membership report seconds later), then its + IPv6 stack is switched off so the re-up does not re-fire DAD/MLD. The + IPv4 address stays configured and answers once the link is back.""" + code, output = daemon.exec( + container_id, + ["sh", "-c", "ip link set eth0 down && echo 1 > /proc/sys/net/ipv6/conf/eth0/disable_ipv6"], + ) + assert code == 0, output + + +def _unsilence_guest(daemon, container_id): + code, output = daemon.exec(container_id, ["ip", "link", "set", "eth0", "up"]) + assert code == 0, output + + +def test_docker_kernel_fast_path(): + server = harness.live_server(kernel=True) + compute = server.compute + image = harness.ensure_docker_image(server) + + project = compute.create_project("docker-e2e") + pid = project["project_id"] + a1 = a2 = bridge = None + try: + n1, n2, daemon = _create_topology(server, pid, image, "E2E") + n1_id, n2_id = n1["node_id"], n2["node_id"] + c1, c2 = n1["container_id"], n2["container_id"] + a1 = harness.docker_anchor_name(n1_id, *ETH) + a2 = harness.docker_anchor_name(n2_id, *ETH) + + # The link is created while both containers are stopped: the kernel + # NIO is bound with no anchors in existence yet and attached by + # container start (the deferred-wiring path). + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert not harness.tap_exists(a1), "the veth pair is born at container start" + + print(".. starting containers") + _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + + # Kernel objects: the veth host ends exist and the per-link bridge + # enslaves exactly the two of them. + assert harness.tap_exists(a1) and harness.tap_exists(a2), (a1, a2) + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + + # L2-anchor spec §E.1 (anchors and bridge carry no L3 identity; + # skipped on a uBridge without `link l2only`) and the forwarding + # guard — a DOWN bridge silently forwards nothing. + if harness.l2only_supported(): + harness.assert_pure_l2(a1) + harness.assert_pure_l2(a2) + harness.assert_pure_l2(bridge) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(a1) + harness.assert_forwarding(a2) + + print(".. pinging between the containers over the kernel link") + baseline = harness.docker_wait_ping(daemon, c1, R2_IP) + assert baseline["loss"] == 0, baseline["raw"] + assert baseline["avg"] is not None and baseline["avg"] < 50, baseline + + # §E.2: with the guests silenced the segment is silent for 5 s — + # the host anchors do not generate the MLD/DAD flood an unhardened + # device produces (baseline: 6 frames / 2 s). + if harness.l2only_supported(): + _silence_guest(daemon, c1) + _silence_guest(daemon, c2) + harness.assert_idle_silence(a1, a2, bridge) + _unsilence_guest(daemon, c1) + _unsilence_guest(daemon, c2) + assert harness.docker_wait_ping(daemon, c1, R2_IP, attempts=5)["loss"] == 0 + + # delay 100: netem on both host ends (each impairs one direction), + # one-way ~100 ms => RTT grows by ~200 ms + _put_filters(compute, pid, lid, {"delay": [100]}) + assert "netem" in harness.qdiscs(a1) and "netem" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + delayed = harness.docker_wait_ping(daemon, c1, R2_IP) + assert delayed["loss"] == 0 and delayed["avg"] >= 150, (baseline, delayed) + _put_filters(compute, pid, lid, {}) + assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) + fast = harness.docker_wait_ping(daemon, c1, R2_IP) + assert fast["loss"] == 0 and fast["avg"] < 50, fast + + # suspend: the host end admin-downs (carrier loss into the + # container), the link is dead; resume restores + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) + assert not harness.tap_up(a1) + dead = harness.docker_ping(daemon, c1, R2_IP, count=3, timeout=1) + assert dead["loss"] == 100, dead["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) + assert harness.tap_up(a1) + assert harness.docker_wait_ping(daemon, c1, R2_IP)["loss"] == 0 + + # capture: AF_PACKET on the anchor writes a real pcap + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.docker_ping(daemon, c1, R2_IP) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # link delete detaches both ends (the veths survive — they are the + # adapters); re-creating enslaves them again and the traffic returns + compute.call("DELETE", f"/projects/{pid}/links/{lid}") + assert harness.bridge_members(bridge) is None + assert harness.tap_exists(a1) and harness.tap_exists(a2) + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert harness.docker_wait_ping(daemon, c1, R2_IP, attempts=5)["loss"] == 0 + + # container stop: the veth host ends are deleted explicitly (a veth + # outlives its container's netns) and the bridge goes with them; a + # restart rebuilds the whole wiring from the NIO + print(".. stopping and restarting container 1") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/stop") + assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 + assert harness.bridge_members(bridge) in (None, [a2]), harness.bridge_members(bridge) + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + harness.docker_wait_address(daemon, c1, R1_IP) + assert harness.tap_exists(a1) + assert harness.wait_until(lambda: harness.bridge_members(bridge) == sorted([a1, a2]), timeout=15) + assert harness.docker_wait_ping(daemon, c1, R2_IP, attempts=5)["loss"] == 0 + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closed the containers: veths and bridges go + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(a2), timeout=15), ( + a1, + a2, + ) + assert harness.wait_until(lambda: harness.bridge_members(bridge) is None, timeout=10), bridge + + +def test_docker_kernel_filter_matrix(): + """One filter type at a time on a live kernel link, each asserted by its + effect on real traffic and its tc state. Types the uBridge on this + compute cannot serve (bpf without cbpf, the eBPF modes without ebpf) are + skipped rather than failed — the capability chain is ``available_filters``' + contract, and a compute reporting nothing keeps the full list.""" + server = harness.live_server(kernel=True) + compute = server.compute + caps = (compute.capabilities() or {}).get("ubridge_tc") or {} + image = harness.ensure_docker_image(server) + + project = compute.create_project("docker-filter-e2e") + pid = project["project_id"] + try: + n1, n2, daemon = _create_topology(server, pid, image, "E2E") + n1_id, n2_id = n1["node_id"], n2["node_id"] + c1 = n1["container_id"] + a1 = harness.docker_anchor_name(n1_id, *ETH) + a2 = harness.docker_anchor_name(n2_id, *ETH) + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + assert link["kernel_datapath"] is True, link + + _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + baseline = harness.docker_wait_ping(daemon, c1, R2_IP) + assert baseline["loss"] == 0, baseline["raw"] + + # The kernel link offers the kernel-only types the compute's uBridge + # reports it can run (all of them on this uBridge). + offered = _filter_types(compute, pid, lid) + assert {"delay", "packet_loss", "corrupt", "duplicate", "bpf"} <= offered, offered + if not {"rate", "reorder", "gemodel", "seed", "limit"} & offered: + pytest.skip(f"this compute's uBridge reports no netem extensions: {sorted(offered)}") + + # -- netem core ---------------------------------------------------- + print(".. delay 100") + _put_filters(compute, pid, lid, {"delay": [100]}) + assert "netem" in harness.qdiscs(a1) and "netem" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + delayed = harness.docker_ping(daemon, c1, R2_IP) + print(f".. {delayed['avg']} ms RTT") + assert delayed["loss"] == 0 and delayed["avg"] >= 150, (baseline, delayed) + + print(".. delay 50 jitter 20 distribution normal") + _put_filters(compute, pid, lid, {"delay": [50, 20, "normal"]}) + # the jitter parameter lands verbatim...; the sampled RTT mean is a + # random variable (per-direction ~N(50, 20)), so only the elevation + # is asserted, with margin — 10 packets put the sample mean's sigma + # around 9 ms + assert re.search(r"delay 50ms\s+20ms", harness.qdiscs(a1)), harness.qdiscs(a1) + jittered = harness.docker_ping(daemon, c1, R2_IP, count=10) + print(f".. {jittered['avg']} ms RTT") + assert jittered["loss"] == 0 and jittered["avg"] >= 65, jittered + _clear_filters(compute, pid, lid, a1, a2) + + print(".. packet_loss 30") + _put_filters(compute, pid, lid, {"packet_loss": [30]}) + lossy = harness.docker_ping(daemon, c1, R2_IP, count=20) + print(f".. {lossy['loss']} % round-trip loss") + # round trip = 1 - 0.7^2 ~ 51 %; the stochastic draw must land + # somewhere between "clearly lossy" and "not a black hole" + assert 20 <= lossy["loss"] <= 80, lossy["raw"] + _clear_filters(compute, pid, lid, a1, a2) + + print(".. corrupt 30") + _put_filters(compute, pid, lid, {"corrupt": [30]}) + assert "corrupt 30%" in harness.qdiscs(a1), harness.qdiscs(a1) + # a corrupted frame fails its checksum and is dropped: the same + # round-trip arithmetic as packet_loss (1 - 0.7^2) + corrupt = harness.docker_ping(daemon, c1, R2_IP, count=20) + print(f".. {corrupt['loss']} % round-trip loss") + assert 20 <= corrupt["loss"] <= 80, corrupt["raw"] + _clear_filters(compute, pid, lid, a1, a2) + + print(".. duplicate 50") + _put_filters(compute, pid, lid, {"duplicate": [50]}) + assert "dup" in harness.qdiscs(a1), harness.qdiscs(a1) + # duplicated frames are absorbed by the stack: no loss, traffic flows + dups = harness.docker_ping(daemon, c1, R2_IP, count=5) + assert dups["loss"] == 0, dups["raw"] + _clear_filters(compute, pid, lid, a1, a2) + + # -- netem extensions (kernel-only) -------------------------------- + print(".. delay 50 + reorder 25 correl 50 gap 5") + _put_filters(compute, pid, lid, {"delay": [50], "reorder": [25, 50, 5]}) + assert "reorder" in harness.qdiscs(a1), harness.qdiscs(a1) + reordered = harness.docker_ping(daemon, c1, R2_IP, count=10) + assert reordered["loss"] <= 60, reordered["raw"] + _clear_filters(compute, pid, lid, a1, a2) + + print(".. delay 50 + seed 42 + limit 5000") + _put_filters(compute, pid, lid, {"delay": [50], "seed": [42], "limit": [5000]}) + assert "limit 5000" in harness.qdiscs(a1), harness.qdiscs(a1) + seeded = harness.docker_ping(daemon, c1, R2_IP) + assert seeded["loss"] == 0 and seeded["avg"] >= 80, seeded + _clear_filters(compute, pid, lid, a1, a2) + + print(".. gemodel 100/0/30 (steady-state bad, 30 % loss)") + _put_filters(compute, pid, lid, {"gemodel": [100, 0, 30]}) + assert "gemodel" in harness.qdiscs(a1), harness.qdiscs(a1) + gemodel = harness.docker_ping(daemon, c1, R2_IP, count=20) + print(f".. {gemodel['loss']} % round-trip loss") + assert 20 <= gemodel["loss"] <= 90, gemodel["raw"] + _clear_filters(compute, pid, lid, a1, a2) + + print(".. rate 512kbit (1400-byte pings)") + _put_filters(compute, pid, lid, {"rate": ["512kbit"]}) + assert "rate" in harness.qdiscs(a1).lower(), harness.qdiscs(a1) + # 1400+28 bytes at 512 kbit ≈ 22 ms serialization per direction + rated = harness.docker_ping(daemon, c1, R2_IP, size=1400) + print(f".. {rated['avg']} ms RTT") + assert rated["loss"] == 0 and rated["avg"] >= 30, (baseline, rated) + _clear_filters(compute, pid, lid, a1, a2) + + # -- cls_bpf match-drop -------------------------------------------- + if caps.get("cbpf"): + print(".. bpf 'icmp' drops everything") + _put_filters(compute, pid, lid, {"bpf": ["icmp"]}) + assert "clsact" in harness.qdiscs(a1), harness.qdiscs(a1) + dropped = harness.docker_ping(daemon, c1, R2_IP, count=4) + assert dropped["loss"] == 100, dropped["raw"] + + print(".. bpf 'greater 150' discriminates by frame size") + _put_filters(compute, pid, lid, {"bpf": ["greater 150"]}) + small = harness.docker_ping(daemon, c1, R2_IP, count=4) + assert small["loss"] == 0, small["raw"] + big = harness.docker_ping(daemon, c1, R2_IP, count=4, size=300) + assert big["loss"] == 100, big["raw"] + _clear_filters(compute, pid, lid, a1, a2) + else: + print(".. uBridge reports no cbpf: skipping the bpf filter checks") + + # -- eBPF stateful classifier -------------------------------------- + modes = caps.get("ebpf_modes") or [] + if "nth" in modes: + print(".. frequency_drop 3 (every 3rd packet per direction)") + _put_filters(compute, pid, lid, {"frequency_drop": [3]}) + nth = harness.docker_ping(daemon, c1, R2_IP, count=20) + print(f".. {nth['loss']} % round-trip loss") + # kernel counts per direction: 1 - (2/3)^2 = 55.6 % round trip + assert 30 <= nth["loss"] <= 80, nth["raw"] + _put_filters(compute, pid, lid, {"frequency_drop": [-1]}) + all_dropped = harness.docker_ping(daemon, c1, R2_IP, count=4) + assert all_dropped["loss"] == 100, all_dropped["raw"] + _clear_filters(compute, pid, lid, a1, a2) + if "quota" in modes: + print(".. quota 3000 bytes hard cutoff") + _put_filters(compute, pid, lid, {"quota": [3000, 100]}) + # 1400-byte pings: the per-direction byte caps run out after + # ~2 packets, then nothing passes until the quota is reset + cut = harness.docker_ping(daemon, c1, R2_IP, count=5, size=1400) + assert cut["loss"] >= 40, cut["raw"] + after = harness.docker_ping(daemon, c1, R2_IP, count=4) + assert after["loss"] == 100, after["raw"] + _clear_filters(compute, pid, lid, a1, a2) + assert harness.docker_ping(daemon, c1, R2_IP, count=4)["loss"] == 0 + if "window" in modes: + print(".. window_drop [1000, 2000, 100] — one outage with passes on both sides") + _put_filters(compute, pid, lid, {"window_drop": [1000, 2000, 100]}) + # 25 pings at 5/s: seqs before t=1 s and after t=3 s answer, the + # middle is gone — per-sequence evidence of a bounded outage + code, output = daemon.exec(c1, ["ping", "-c", "25", "-i", "0.2", "-W", "1", R2_IP], timeout=30) + replied = {int(m.group(1)) for m in re.finditer(r"seq=(\d+)", output)} + missing = set(range(25)) - replied + assert 0 in replied and 24 in replied, (sorted(replied), output) + assert missing and max(missing) - min(missing) >= 5, (sorted(replied), output) + _clear_filters(compute, pid, lid, a1, a2) + assert harness.docker_ping(daemon, c1, R2_IP, count=4)["loss"] == 0 + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(a2), timeout=15), ( + a1, + a2, + ) + + +def test_docker_relay_control(): + """Negative control: the same topology on an isolated relay-configured + instance wires nothing into the kernel — no per-link bridge, no + enslavement — and still pings over the uBridge UDP relay.""" + server = harness.live_server(kernel=False) + compute = server.compute + image = harness.ensure_docker_image(server) + + project = compute.create_project("docker-relay-e2e") + pid = project["project_id"] + a1 = None + try: + n1, n2, daemon = _create_topology(server, pid, image, "E2E") + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.docker_anchor_name(n1_id, *ETH) + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + assert link["kernel_datapath"] is False, link + + _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + + # No kernel wiring: no per-link bridge, the veth host end is just a + # relay endpoint (up, pure L2 — the hardening is a creation-time + # property, not a datapath one) + assert harness.bridge_members(harness.link_bridge_name(link["link_id"])) is None + if harness.tap_exists(a1): + assert harness.tap_up(a1), a1 + if harness.l2only_supported(): + harness.assert_pure_l2(a1) + + relay_ping = harness.docker_wait_ping(daemon, n1["container_id"], R2_IP) + assert relay_ping["loss"] == 0, relay_ping["raw"] + + # available_filters hides the kernel-only types on the relay + offered = _filter_types(compute, pid, link["link_id"]) + assert "delay" in offered and "frequency_drop" in offered, offered + assert not (offered & {"rate", "reorder", "gemodel", "duplicate", "seed", "limit", "quota", "window_drop"}), ( + offered + ) + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 + + +def test_docker_relay_to_kernel_reopen_upgrade(): + """The server's datapath choice is a live decision: with the project + created on a relay instance, flipping ``enable_kernel_datapath`` across + a server restart and reopening the project rebuilds the link on the + kernel datapath (the upgrade path a deployment takes) and it forwards.""" + if os.environ.get("GNS3_E2E_URL"): + pytest.skip("this scenario restarts the server — it needs an isolated instance") + server = harness.live_server(kernel=False) + compute = server.compute + image = harness.ensure_docker_image(server) + + project = compute.create_project("docker-upgrade-e2e") + pid = project["project_id"] + a1 = a2 = bridge = None + try: + n1, n2, daemon = _create_topology(server, pid, image, "E2E") + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.docker_anchor_name(n1_id, *ETH) + a2 = harness.docker_anchor_name(n2_id, *ETH) + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + assert link["kernel_datapath"] is False, link + bridge = harness.link_bridge_name(link["link_id"]) + + _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + assert harness.bridge_members(bridge) is None + assert harness.docker_wait_ping(daemon, n1["container_id"], R2_IP)["loss"] == 0 + + # Flip the datapath choice and reopen the project: every link + # re-runs _prepare, which re-evaluates eligibility live. + print(".. restarting the server with enable_kernel_datapath=True") + server.restart(kernel=True) + compute = server.compute + compute.call("POST", f"/projects/{pid}/open") + n1 = _resolve_container(daemon, compute.node(pid, n1_id)) + n2 = _resolve_container(daemon, compute.node(pid, n2_id)) + link = compute.call("GET", f"/projects/{pid}/links/{link['link_id']}") + assert link["kernel_datapath"] is True, link + assert not harness.tap_exists(a1), "containers are stopped after the reopen" + + _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + assert harness.wait_until(lambda: harness.bridge_members(bridge) == sorted([a1, a2]), timeout=15) + assert harness.docker_wait_ping(daemon, n1["container_id"], R2_IP, attempts=5)["loss"] == 0 + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(a2), timeout=15), ( + a1, + a2, + ) + assert harness.wait_until(lambda: harness.bridge_members(bridge) is None, timeout=10), bridge From 765d706ec44b5c688e97fe79db5d3753739a5c65 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 16:23:19 +0800 Subject: [PATCH 053/101] docs(docker): record the in-repo e2e coverage The feature doc's Verification section now points at tests/e2e/test_docker_kernel_datapath.py and what the suite pins down (anchors/bridges, the filter matrix through real traffic, suspend, capture, the stop/start and delete/re-create beats, the relay control and the relay->kernel reopen upgrade), including the digest-pinned image the scenarios share. --- docs/features/docker-kernel-datapath.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 7de504a59..2d400adea 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -458,6 +458,19 @@ links and a relay link both hides it and 409s it; teardown leaves no filter or qdisc behind. The P6b e2e was re-run as a regression on the same uBridge build (13/13 — nth/quota unaffected by the de-looping below). +In-repo live coverage: `tests/e2e/test_docker_kernel_datapath.py` (pytest +marker `e2e`) runs two real Alpine containers on an isolated instance — the +per-link bridge enslaving exactly the two veth host ends, real ICMP, the +L2-anchor spec §E assertions (no L3 identity, forwarding ports, idle +silence with the guests' links down), the filter matrix one type at a time +(netem core and extensions, cls_bpf match-drop, the eBPF classifier, each +asserted through real traffic), suspend, capture, link delete/re-create, +container stop/start, a relay negative control, and the relay→kernel +upgrade on project reopen (the datapath choice is flipped across a server +restart). The container image is digest-pinned and pulled through the +server's own pull route on a cache miss, so collaborators test the same +bytes. + The uBridge-side window work took two fixes that are worth recording, both found by *this* integration rather than by unit tests: the delivered catch-up loop tripped the verifier's 8192 **jump-sequence** budget — on the From aaabf0409d7c89cc8ab7b8ad0caaf743df8c4a38 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 19:30:26 +0800 Subject: [PATCH 054/101] test(e2e): live coverage for the IOU node kernel datapath Two real IOU routers on a real L3 image (the image's own IOL fabric, the real IOS CLI on the server's telnet console), driving the anchor TAPs the kernel datapath builds on: anchors born with the node (one per Ethernet bay/unit), real ICMP through the [fabric <-> TAP] port bridge, netem delay, suspend, capture, link delete/re-create, node stop/start rewiring, the serial link staying on the relay (serial bays never anchor) and a relay negative control on an isolated instance. Every scenario ends with no anchors, bridges or tmpdirs left behind. harness: create_iou_node/iou_images/iou_anchor_name, and the IOS console helper is now ios_console (dynamips, IOU and the IOL containers share it). --- tests/e2e/README.md | 8 +- tests/e2e/harness.py | 39 ++- tests/e2e/test_dynamips_kernel_datapath.py | 2 +- .../test_ethernet_switch_kernel_datapath.py | 2 +- tests/e2e/test_iou_kernel_datapath.py | 292 ++++++++++++++++++ 5 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/test_iou_kernel_datapath.py diff --git a/tests/e2e/README.md b/tests/e2e/README.md index fd5a6f6ff..f47d65e6f 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -27,7 +27,8 @@ Two run modes, chosen per test through `harness.live_server(kernel=...)`: process on a free port with its own config directory (the controller DB and every path follow the config), so nothing on the machine is touched. It needs `ubridge`, `dynamips` and an images directory - (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally; the + (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally — the IOU + scenario takes its L3 image from that directory's `IOU` subfolder; the Docker scenarios additionally need the Docker daemon the server talks to. Either way each test creates its own project and deletes it again. Set @@ -65,6 +66,11 @@ Current scenarios: TAP leg (anchors at start, netem delay, suspend, capture, link delete/re-create swap, node stop/start rewiring) and a relay negative control. +* `test_iou_kernel_datapath.py` — two real IOU routers (a real IOU image + from the images directory, the real IOS CLI on the server's telnet + console): kernel link through the [IOL fabric ↔ TAP] port bridge (same + lifecycle as the iol-xe containers), a serial link staying on the relay + (serial bays never anchor), and a relay negative control. * `test_docker_kernel_datapath.py` — two real Alpine containers (the plain veth datapath): the per-link kernel bridge enslaves exactly the two veth host ends, real ICMP crosses, the filter matrix runs one type at a time diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index b3872f0f5..6c3267e85 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -157,6 +157,31 @@ class Compute: }, ) + def create_iou_node(self, project_id, name, image, ethernet_adapters=2, serial_adapters=2, ram=1024, nvram=256): + """An IOU node: the real IOU binary running on this host, its IOS on + the server's telnet console. The kernel scenario picks the image + through ``iou_images`` (an L3 image — an L2 switching image does not + take addresses on its switchports) and addresses Ethernet0/0.""" + return self.call( + "POST", + f"/projects/{project_id}/nodes", + { + "compute_id": "local", + "name": name, + "node_type": "iou", + "properties": { + "path": image, + "ethernet_adapters": ethernet_adapters, + "serial_adapters": serial_adapters, + "ram": ram, + "nvram": nvram, + # explicit ram/nvram, like the IOU appliance template + "use_default_iou_values": False, + "console_type": "telnet", + }, + }, + ) + def create_iol_router(self, project_id, name, image, adapters=2): """An IOL runner container (iol-runner images), the appliance's template properties.""" @@ -204,6 +229,9 @@ class Compute: def docker_images(self): return [image["image"] for image in self.call("GET", "/computes/local/docker/images")] + def iou_images(self): + return [image["filename"] for image in self.call("GET", "/computes/local/iou/images")] + # --------------------------------------------------------------------------- # Server lifecycle @@ -565,7 +593,10 @@ class IOSConsole: self.sock.close() -def dynamips_console(server, project_id, node_id): +def ios_console(server, project_id, node_id): + """The telnet console of a node running IOS (dynamips, IOU, IOL + container): the server-side telnet endpoint around the emulator's own + stdio — same driver for all of them.""" node = server.compute.node(project_id, node_id) host = node["console_host"] if node["console_host"] not in ("0.0.0.0", "::") else "127.0.0.1" return IOSConsole(host, node["console"]) @@ -1056,6 +1087,12 @@ def iol_anchor_name(node_id, adapter, port): return "gx" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" +def iou_anchor_name(node_id, adapter, port=0): + """The IOU node's anchor TAP name: one persistent TAP per Ethernet + bay/unit (see IOUVM._tap_name and utils.kernel_anchor).""" + return "gi" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" + + def stage_iol_base_config(server): """ Copy the shipped iol-xe base config into an isolated instance's configs diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py index 5e41d10eb..64e61e331 100644 --- a/tests/e2e/test_dynamips_kernel_datapath.py +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -66,7 +66,7 @@ def _pick_image(server): def _boot(server, project_id, node_id, timeout=360): - console = harness.dynamips_console(server, project_id, node_id) + console = harness.ios_console(server, project_id, node_id) console.boot_wait(timeout=timeout) return console diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index 2cf98905d..a20a92fbf 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -89,7 +89,7 @@ def _create_topology(compute, pid, image, name, idlepc=None): def _boot(server, project_id, node_id, timeout=360): - console = harness.dynamips_console(server, project_id, node_id) + console = harness.ios_console(server, project_id, node_id) console.boot_wait(timeout=timeout) return console diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py new file mode 100644 index 000000000..55b80f82c --- /dev/null +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -0,0 +1,292 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software and is redistributed under the same terms as gns3-server. +# See the LICENSE file for licensing information. +# + +""" +Live end-to-end test for the IOU node's kernel datapath (pytest marker +``e2e``): two real IOU routers (a real IOU image executed by the local +compute, the real IOS CLI behind the server's telnet console), driven +through the controller REST API. IOU has no host interface of its own: the +guest leg is the IOL fabric unix socket, and the kernel datapath buys the +*link segment* — one persistent anchor TAP per Ethernet bay/unit, a per-link +Linux bridge, kernel filters, suspend and capture. + +* two IOU nodes on one compute report ``kernel_datapath`` when the compute's + uBridge can terminate an IOL port on a persistent TAP + (``ubridge_iol_tap``); the anchors (``gi`` TAPs, one per Ethernet + bay/unit) exist from node start, before any link — and the per-link + bridge enslaves exactly the two of them; +* real ICMP crosses the [IOL fabric ↔ TAP] port bridge; +* a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; +* suspend admin-downs the anchor and kills the traffic; resume restores; +* a serial link between the same nodes never anchors (per-port exclusion): + it stays on the relay, ``kernel_datapath`` is False, and real traffic + still crosses it; +* link delete releases the port bridge's TAP leg and the anchor survives — + re-creating the link swaps the leg back in and the traffic returns; +* node stop tears the anchors down (uBridge holds their fds — the IOL + bridge goes first, then ``tap delete``); a restart recreates the whole + wiring from the NIO; +* deleting the project leaves no anchors and no bridges behind. + +``test_iou_relay_control`` is the negative control on an isolated +relay-configured instance: the same topology rides the fabric socket ↔ UDP +relay (no per-link bridge, nothing enslaved) and still pings. +""" + +import re +import time + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# Ethernet0/0 is (bay 0, unit 0) — one adapter = four units. The adapter +# index runs over Ethernet and serial bays together, so with two Ethernet +# adapters the serial bays are 2 and 3: Serial2/0 is (2, 0). +ETH = (0, 0) +SERIAL = (2, 0) +BOOT_TIMEOUT = 360 + +# L3 images only: an L2 switching image does not take an address on its +# switchports. The first is the image the kernel datapath was validated +# with, the second the matching 17.15 build. +IMAGE_PREFERENCE = ("x86_64_crb_linux-adventerprisek9-ms.iol", "l3-adventerprisek9-ms-17.15.01.bin") + + +def _pick_image(server): + images = server.compute.iou_images() + for name in IMAGE_PREFERENCE: + if name in images: + return name + l3 = next((name for name in images if "adventerprisek9" in name and "l2" not in name.lower()), None) + if not l3: + pytest.skip(f"no L3 IOU image available on this server (images: {images[:5]})") + return l3 + + +def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): + console = harness.ios_console(server, project_id, node_id) + console.boot_wait(timeout=timeout) + return console + + +def test_iou_kernel_datapath(): + server = harness.live_server(kernel=True) + compute = server.compute + caps = compute.capabilities() + if caps.get("ubridge_iol_tap") is not True or caps.get("ubridge_tap") is not True: + pytest.skip( + f"this compute's uBridge cannot serve the IOU anchor lifecycle " + f"(ubridge_iol_tap={caps.get('ubridge_iol_tap')}, ubridge_tap={caps.get('ubridge_tap')})" + ) + image = _pick_image(server) + + project = compute.create_project("iou-e2e") + pid = project["project_id"] + a1 = a2 = None + try: + n1 = compute.create_iou_node(pid, "E2E-R1", image) + n2 = compute.create_iou_node(pid, "E2E-R2", image) + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.iou_anchor_name(n1_id, *ETH) + a2 = harness.iou_anchor_name(n2_id, *ETH) + + # The link is created while both nodes are stopped: the kernel NIO + # is bound with no anchors in existence yet and wired by node start + # (the deferred-wiring path). + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert not harness.tap_exists(a1), "anchors must not exist before start" + + print(f".. starting routers on {image} and waiting for IOS boots") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{n2_id}/start") + c1 = _boot(server, pid, n1_id) + c2 = _boot(server, pid, n2_id) + + # Kernel objects: the anchors are one persistent TAP per Ethernet + # bay/unit, born with the node (all eight — the IOU port model), + # and the per-link bridge has exactly the two addressed ports. + for adapter in range(2): + for port in range(4): + name = harness.iou_anchor_name(n1_id, adapter, port) + assert harness.tap_exists(name), name + assert harness.tap_exists(a2), a2 + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + + # L2-anchor spec §E.1: anchors and the per-link bridge carry no L3 + # identity (skipped on a uBridge without `link l2only`), and both + # ports are FORWARDING — the silent-failure guard. + if harness.l2only_supported(): + harness.assert_pure_l2(a1) + harness.assert_pure_l2(a2) + harness.assert_pure_l2(bridge) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(a1) + harness.assert_forwarding(a2) + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + out = c1.run("show ip int brief") + assert re.search(r"Ethernet0/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + print(".. both routers configured, pinging through the [IOL fabric <-> TAP] port bridge") + baseline = harness.wait_ping(c1, "10.1.1.2") + assert baseline["success"] == 100, baseline["raw"] + + # delay 100: netem on both anchors (each impairs one direction), + # one-way ~100 ms => RTT grows by ~200 ms + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(a1) and "netem" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + delayed = harness.wait_ping(c1, "10.1.1.2") + assert delayed["success"] == 100, delayed["raw"] + assert delayed["avg"] >= 150, (baseline, delayed) + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) + fast = harness.wait_ping(c1, "10.1.1.2") + assert fast["success"] == 100 and fast["avg"] < 50, fast + + # suspend: anchor admin-down (the port bridge's TAP writes fail EIO, + # nothing comes back); resume restores + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) + assert not harness.tap_up(a1) + dead = harness.ping(c1, "10.1.1.2", repeat=3) + assert dead["success"] == 0, dead["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) + assert harness.tap_up(a1) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + + # capture: AF_PACKET on the anchor writes a real pcap + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.ping(c1, "10.1.1.2") + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # serial link between the same nodes: per-port exclusion keeps it on + # the relay, where the real traffic still crosses (IOU-specific — + # serial bays never anchor) + serial = compute.create_link(pid, (n1_id, *SERIAL), (n2_id, *SERIAL)) + assert serial["kernel_datapath"] is False, serial + harness.configure_ios(c1, "R1", serial_ip="10.2.2.1", serial_if="Serial2/0", clock=True) + harness.configure_ios(c2, "R2", serial_ip="10.2.2.2", serial_if="Serial2/0") + relay_ping = harness.wait_ping(c1, "10.2.2.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + compute.call("DELETE", f"/projects/{pid}/links/{serial['link_id']}") + + # delete / re-create the kernel link: the port bridge's TAP leg is + # released (stop -> delete_nio_tap), the anchor survives (the node + # owns it), and re-creating swaps the leg back in — the fabric + # socket binding never re-binds across the churn. + compute.call("DELETE", f"/projects/{pid}/links/{lid}") + assert harness.bridge_members(bridge) is None + assert harness.tap_exists(a1) and harness.tap_exists(a2) + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + # node stop: uBridge holds the anchor fds, so the IOL bridge goes + # first and the taps with it (a stopped node must not litter the + # host with gi devices); a restart recreates the whole wiring from + # the NIO that outlived it. + print(".. stopping and restarting router 1") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/stop") + assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 + # the per-link bridge lost this end's port; it may be gone entirely + # (both endpoints delete it, last one wins) + assert harness.bridge_members(bridge) in (None, [a2]), harness.bridge_members(bridge) + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + c1.close() + c1 = _boot(server, pid, n1_id) + assert harness.tap_exists(a1) + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closed the nodes: anchors and bridges go with them + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(a2), timeout=15), ( + a1, + a2, + ) + + +def test_iou_relay_control(): + """ + Negative control: the same topology with ``enable_kernel_datapath = + false`` (an isolated instance) wires nothing into the kernel — no + per-link bridge, no enslavement — and still pings over the fabric + socket ↔ UDP relay. + + Anchors DO exist here (an IOU node creates its port TAPs at start + whenever the uBridge can terminate an IOL port on one); the server + configuration only decides whether links are attached to them (the same + semantics as QEMU's and Dynamips' anchors). + """ + + server = harness.live_server(kernel=False) + compute = server.compute + image = _pick_image(server) + + project = compute.create_project("iou-relay-e2e") + pid = project["project_id"] + a1 = None + try: + n1 = compute.create_iou_node(pid, "E2E-R1", image) + n2 = compute.create_iou_node(pid, "E2E-R2", image) + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.iou_anchor_name(n1_id, *ETH) + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + assert link["kernel_datapath"] is False, link + + print(".. starting routers and waiting for IOS boots (relay)") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{n2_id}/start") + c1 = _boot(server, pid, n1_id) + c2 = _boot(server, pid, n2_id) + + # no kernel wiring: the anchor (if born) is nobody's bridge port + assert harness.bridge_members(harness.link_bridge_name(link["link_id"])) is None + if harness.tap_exists(a1): + assert not harness.tap_up(a1), a1 + if harness.l2only_supported(): + # the L2 hardening is a creation-time property, not a + # datapath one: relay anchors are pure L2 too + harness.assert_pure_l2(a1) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + relay_ping = harness.wait_ping(c1, "10.1.1.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closed the nodes: the port TAPs go with them + assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 From a0d76b30c0435bf9af8643b5382c79590192c2cc Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 19:30:29 +0800 Subject: [PATCH 055/101] docs(iou): record the in-repo e2e coverage The Verified section only described the /tmp driver with a fake IOU image; the repository suite now carries the same scenario with real routers on a real L3 image. --- docs/features/iou-kernel-datapath.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index c1b531a98..a0fc2df66 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -142,6 +142,14 @@ qdiscs, userspace `delay` at 100.7 ms, suspend via the synthetic frequency_drop, `iol_bridge start_capture`), which is also the regression net for the port-coordinate capture restore this work touched. +The repository's live e2e suite carries the same scenario with **real +IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker +`e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on +the server's telnet console, driving anchors-born-with-the-node, real ICMP +through the port bridge, netem delay, suspend, capture, link +delete/re-create, node stop/start rewiring, the serial link staying on the +relay, and a relay negative control. + The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the fork), including the DOWN-anchor resilience and kernel-bridge interop this design depends on. From 1e999d916da6cc1d1f9309ecddde55e6d3621db7 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 19:44:10 +0800 Subject: [PATCH 056/101] fix(controller): keep port links across a port-list rebuild MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node._list_ports() rebuilds every Port object, and it runs on each node update — including every compute node.updated notification (a node reporting started is one). The links are unaffected by that rebuild, but the fresh port objects started out with link=None, so after any node update the "Port is already used" guard in Link.add_node stopped matching: a second link on an occupied port reached the compute, where the port's NIO was replaced before the attach failed on the already-bound anchor, leaving the link half-torn-down (on IOU, deleting the project afterwards left the node's anchor TAPs and a per-link bridge on the host). Carry the link back-references over by (adapter_number, port_number), first match wins, mirroring get_port; a port that still exists keeps its link. Regression tests pin both the reference (test_node) and the guard (test_link: the duplicate link is refused after a node update). --- gns3server/controller/node.py | 26 ++++++++++++++++++++++++++ tests/controller/test_link.py | 29 +++++++++++++++++++++++++++++ tests/controller/test_node.py | 17 +++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/gns3server/controller/node.py b/gns3server/controller/node.py index 3500cb2db..c8d20f10a 100644 --- a/gns3server/controller/node.py +++ b/gns3server/controller/node.py @@ -1148,6 +1148,32 @@ class Node: return None def _list_ports(self): + """ + Rebuild the port list, preserving the link back-references. + + The list is rebuilt on every node update (a property change or a + compute ``node.updated`` notification), which creates fresh Port + objects — but the links themselves are unaffected by that rebuild, + so a port that still exists must keep pointing at its link. Without + this the "Port is already used" guard in ``Link.add_node`` silently + stops matching after any node update and a second link gets created + on an occupied port (the compute then fails on the already-bound + NIO). Links are carried over by (adapter, port), first match wins, + mirroring ``get_port``. + """ + preserved = {} + for port in self._ports or []: + if port.link is not None: + preserved.setdefault((port.adapter_number, port.port_number), port.link) + + self._rebuild_ports() + + for port in self._ports: + link = preserved.pop((port.adapter_number, port.port_number), None) + if link is not None and port.link is None: + port.link = link + + def _rebuild_ports(self): """ Generate the list of port display in the client if the compute has sent a list we return it (use by diff --git a/tests/controller/test_link.py b/tests/controller/test_link.py index 186e4c332..23e2d8f35 100644 --- a/tests/controller/test_link.py +++ b/tests/controller/test_link.py @@ -112,6 +112,35 @@ async def test_add_node_already_connected(project, compute): await link2.add_node(node1, 0, 4) +@pytest.mark.asyncio +async def test_add_node_already_connected_after_node_update(project, compute): + """ + A node update (any compute node.updated notification) rebuilds its port + objects; the port must still count as used afterwards, or the duplicate + link reaches the compute and fails on the already-bound NIO instead of + being refused here. + """ + + project.dump = AsyncioMagicMock() + + node1 = Node(project, compute, "node1", node_type="qemu") + node2 = Node(project, compute, "node2", node_type="qemu") + + link = Link(project) + link.create = AsyncioMagicMock() + link.node_updated = AsyncioMagicMock() + link._project.emit_notification = MagicMock() + await link.add_node(node1, 0, 0) + await link.add_node(node2, 0, 0) + + await node1.parse_node_response({"status": "started"}) + + link2 = Link(project) + link2.create = AsyncioMagicMock() + with pytest.raises(ControllerError, match="already used"): + await link2.add_node(node1, 0, 0) + + @pytest.mark.asyncio async def test_add_node_cloud(project, compute): diff --git a/tests/controller/test_node.py b/tests/controller/test_node.py index 2c634c8f9..788b560bb 100644 --- a/tests/controller/test_node.py +++ b/tests/controller/test_node.py @@ -1191,3 +1191,20 @@ async def test_parse_node_response(node): node.add_link(link) await node.parse_node_response({"status": "started"}) assert link.node_updated.called + + +@pytest.mark.asyncio +async def test_port_link_survives_port_rebuild(node): + """ + A node update rebuilds the port objects (every compute node.updated + notification does); the link back-references must survive the rebuild, + or the "Port is already used" guard silently stops matching. + """ + + port = node.get_port(0, 0) + link = MagicMock() + port.link = link + await node.parse_node_response({"status": "started"}) + rebuilt = node.get_port(0, 0) + assert rebuilt is not port # the rebuild really does replace the objects + assert rebuilt.link is link From 928757542e09a7632a37940b93049e3ac39b1fd0 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 20:56:58 +0800 Subject: [PATCH 057/101] fix(controller): bind and release link members on their live ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Link.delete cleared the port link through the port object stored at add_node time; a node update since then rebuilt the port list, so that object was stale and the live port kept its link — the next link on the port was refused with "Port is already used". Masked until now by the older rebuild bug (c2d68a65d): links never survived a rebuild, so the guard never fired and the stale clear never mattered. --- gns3server/controller/link.py | 28 ++++++++++++++++++++++------ gns3server/controller/project.py | 8 ++------ tests/controller/test_link.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 013b7e509..909792ac8 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -438,9 +438,7 @@ class Link: if len(self._nodes) == 2 and not batch: await self.create() - for n in self._nodes: - n["node"].add_link(self) - n["port"].link = self + self._bind_members() self._created = True self._project.emit_notification("link.created", self.asdict()) @@ -471,14 +469,32 @@ class Link: """ raise NotImplementedError + def _bind_members(self): + """ + Wire the back-references on every member: the node's link set and the + port's link. The port is resolved live, through the node's current + port list — the object stored at add_node time goes stale whenever a + node update rebuilds that list (``parse_node_response``), and a + back-reference set on the stale object is invisible to the "Port is + already used" guard in add_node, which reads the live one. + """ + + for n in self._nodes: + n["node"].add_link(self) + port = n["node"].get_port(n["adapter_number"], n["port_number"]) or n["port"] + port.link = self + async def delete(self): """ Delete the link """ for n in self._nodes: - # It could be different of self if we rollback an already existing link - if n["port"].link == self: - n["port"].link = None + # The live port, for the same staleness reason as _bind_members; + # the link could be different from self if we rollback an + # already existing link + port = n["node"].get_port(n["adapter_number"], n["port_number"]) or n["port"] + if port.link == self: + port.link = None n["node"].remove_link(self) async def reset(self): diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index c5aedf731..e2def78c9 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -987,9 +987,7 @@ class Project: # an image is missing. Keep the link on the controller (so the # topology is preserved) but defer the NIO creation until the # missing image is resolved. - for n in link._nodes: - n["node"].add_link(link) - n["port"].link = link + link._bind_members() link._deferred = True log.info( "Project '%s' [%s]: deferring link %s until missing image(s) are resolved", @@ -1990,9 +1988,7 @@ class Project: # Finalise every link: wire node/port back-references, mark created, # notify clients, and apply project-level marker definitions. for link, _entries in valid: - for n in link._nodes: - n["node"].add_link(link) - n["port"].link = link + link._bind_members() link._created = True self.emit_notification("link.created", link.asdict()) log.info("Project '%s' [%s]: created %d links", self._name, self._id, len(valid)) diff --git a/tests/controller/test_link.py b/tests/controller/test_link.py index 23e2d8f35..8f932ae28 100644 --- a/tests/controller/test_link.py +++ b/tests/controller/test_link.py @@ -141,6 +141,37 @@ async def test_add_node_already_connected_after_node_update(project, compute): await link2.add_node(node1, 0, 0) +@pytest.mark.asyncio +async def test_link_delete_after_node_update_releases_port(project, compute): + """ + Link.delete drops the port's link reference through the port object it + stored at add_node time; a node update since then rebuilt the port list, + so that stored object is stale. The delete must release the *live* port, + or the next link on it is refused with "Port is already used". + """ + + project.dump = AsyncioMagicMock() + + node1 = Node(project, compute, "node1", node_type="qemu") + node2 = Node(project, compute, "node2", node_type="qemu") + + link = Link(project) + link.create = AsyncioMagicMock() + link.node_updated = AsyncioMagicMock() + link._project.emit_notification = MagicMock() + await link.add_node(node1, 0, 0) + await link.add_node(node2, 0, 0) + + # a node update (e.g. the status change of a start) rebuilds the ports + await node1.parse_node_response({"status": "started"}) + + await link.delete() + + link2 = Link(project) + link2.create = AsyncioMagicMock() + await link2.add_node(node1, 0, 0) # must not raise "Port is already used" + + @pytest.mark.asyncio async def test_add_node_cloud(project, compute): From 4fd7e1aa07b7a424964401e4ff9d476c8f39c775 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 20:57:10 +0800 Subject: [PATCH 058/101] test(e2e): live coverage for the QEMU node kernel datapath --- tests/e2e/README.md | 12 +- tests/e2e/harness.py | 41 ++++ tests/e2e/test_qemu_kernel_datapath.py | 293 +++++++++++++++++++++++++ 3 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/test_qemu_kernel_datapath.py diff --git a/tests/e2e/README.md b/tests/e2e/README.md index f47d65e6f..35e21df9b 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -28,8 +28,9 @@ Two run modes, chosen per test through `harness.live_server(kernel=...)`: and every path follow the config), so nothing on the machine is touched. It needs `ubridge`, `dynamips` and an images directory (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally — the IOU - scenario takes its L3 image from that directory's `IOU` subfolder; the - Docker scenarios additionally need the Docker daemon the server talks to. + scenario takes its L3 image from that directory's `IOU` subfolder, the + QEMU scenario its L3 IOSv qcow2 from the `QEMU` subfolder; the Docker + scenarios additionally need the Docker daemon the server talks to. Either way each test creates its own project and deletes it again. Set `GNS3_E2E_KEEP=1` to keep the project (and a started instance) on failure @@ -71,6 +72,13 @@ Current scenarios: console): kernel link through the [IOL fabric ↔ TAP] port bridge (same lifecycle as the iol-xe containers), a serial link staying on the relay (serial bays never anchor), and a relay negative control. +* `test_qemu_kernel_datapath.py` — two real IOSv routers (a real qcow2 + image from the images directory's `QEMU` subfolder, linked clone, the + real IOS CLI on the server's telnet console): the anchor TAP is QEMU's + own netdev (no port bridge in between) — the full kernel lifecycle plus + a relay negative control where the anchor carries as the relay's + AF_PACKET endpoint. IOSv boots a real IOS: this is the slowest scenario + pair (~6½ min kernel, ~5 min relay). * `test_docker_kernel_datapath.py` — two real Alpine containers (the plain veth datapath): the per-link kernel bridge enslaves exactly the two veth host ends, real ICMP crosses, the filter matrix runs one type at a time diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 6c3267e85..41488b8f3 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -202,6 +202,37 @@ class Compute: }, ) + def create_qemu_node(self, project_id, name, image, adapters=2, adapter_type="e1000", ram=512): + """A QEMU VM node: the real qemu-system binary on this host running + the image (a linked clone keeps the image pristine), the guest on the + server's telnet console. The kernel scenario picks the image through + ``qemu_images`` (an L3 IOSv image — the IOS console driver drives it) + and addresses its first adapter. The disk interface must be named: + a raw node defaults to ``none``, which attaches no frontend device to + the drive backend — the VM comes up diskless (the IOSv appliance + boots virtio).""" + return self.call( + "POST", + f"/projects/{project_id}/nodes", + { + "compute_id": "local", + "name": name, + "node_type": "qemu", + "properties": { + # a raw node carries no template, so the binary (and with + # it the platform) must be named explicitly + "qemu_path": "qemu-system-x86_64", + "hda_disk_image": image, + "hda_disk_interface": "virtio", + "adapter_type": adapter_type, + "adapters": adapters, + "ram": ram, + "linked_clone": True, + "console_type": "telnet", + }, + }, + ) + def create_link(self, project_id, a, b): return self.call( "POST", @@ -232,6 +263,9 @@ class Compute: def iou_images(self): return [image["filename"] for image in self.call("GET", "/computes/local/iou/images")] + def qemu_images(self): + return [image["filename"] for image in self.call("GET", "/computes/local/qemu/images")] + # --------------------------------------------------------------------------- # Server lifecycle @@ -1093,6 +1127,13 @@ def iou_anchor_name(node_id, adapter, port=0): return "gi" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" +def qemu_anchor_name(node_id, adapter, port=0): + """The QEMU node's anchor TAP name: one persistent TAP per adapter — + the netdev QEMU itself opens by ifname (see QemuVM._tap_name and + utils.kernel_anchor).""" + return "gq" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" + + def stage_iol_base_config(server): """ Copy the shipped iol-xe base config into an isolated instance's configs diff --git a/tests/e2e/test_qemu_kernel_datapath.py b/tests/e2e/test_qemu_kernel_datapath.py new file mode 100644 index 000000000..795cce5c6 --- /dev/null +++ b/tests/e2e/test_qemu_kernel_datapath.py @@ -0,0 +1,293 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# This program is free software and is redistributed under the same terms as gns3-server. +# See the LICENSE file for licensing information. +# + +""" +Live end-to-end test for the QEMU node's kernel datapath (pytest marker +``e2e``): two real IOSv routers (a real qcow2 image run by the local qemu +binary, the real IOS CLI behind the server's telnet console), driven through +the controller REST API. QEMU is the node type whose guest leg *is* the +anchor: uBridge creates one persistent TAP per adapter before launch and +hands it to this user, and QEMU opens it by ifname as its netdev +(``-netdev tap,ifname=...,script=no,downscript=no``) — no userspace hop, no +port bridge, the anchor is the NIC's host side. + +* two QEMU nodes on one compute report ``kernel_datapath`` when the + compute's uBridge can create persistent TAPs (``ubridge_tap``); the + anchors (``gq`` TAPs, one per adapter) exist from node start, before any + link — and the per-link bridge enslaves exactly the two of them; +* real ICMP crosses the [QEMU netdev ↔ TAP] anchor; +* a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; +* suspend admin-downs the anchor (the e1000 loses carrier) and kills the + traffic; resume restores; +* a capture on the kernel link writes a real pcap; +* link delete releases the anchor from the bridge and it survives (the + running VM holds it open) — re-creating the link re-enslaves it and the + traffic returns; +* node stop deletes every anchor of the node together with the orphan + bridges; a restart recreates the TAPs and rewires the link from the NIO; +* deleting the project leaves no anchors and no bridges behind. + +``test_qemu_relay_control`` is the negative control on an isolated +relay-configured instance: the anchors still exist (a QEMU node creates its +adapter TAPs at start whenever uBridge can), but nothing is enslaved — the +relay attaches the anchor itself as an AF_PACKET endpoint, so it stays up +and carrying while the link rides the uBridge relay. This differs from +IOU's relay, where the fabric socket carries the traffic and the anchors +sit idle: QEMU has no other leg, the TAP is the netdev either way. +""" + +import re +import time + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# One NIC per adapter, port number always 0: adapter 0 is GigabitEthernet0/0. +ETH = (0, 0) +# IOSv boots a real IOS from an emulated disk — slower than IOU (the boot +# wait is event-driven; this is only the failure cap). +BOOT_TIMEOUT = 480 + +# L3 images only (an IOSv-L2 image switchports its interfaces). The first +# is the image on this host the kernel datapath was validated with. +IMAGE_PREFERENCE = ("vios-adventerprisek9-m.spa.159-3.m12.qcow2",) +GUEST_ETH_IF = "GigabitEthernet0/0" + + +def _pick_image(server): + images = server.compute.qemu_images() + for name in IMAGE_PREFERENCE: + if name in images: + return name + l3 = next( + (name for name in images if "vios" in name.lower() and "l2" not in name.lower() and name.endswith(".qcow2")), + None, + ) + if not l3: + pytest.skip(f"no L3 IOSv qcow2 available on this server (images: {images[:5]})") + return l3 + + +def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): + console = harness.ios_console(server, project_id, node_id) + console.boot_wait(timeout=timeout) + return console + + +def test_qemu_kernel_datapath(): + server = harness.live_server(kernel=True) + compute = server.compute + caps = compute.capabilities() + if caps.get("ubridge_tap") is not True: + pytest.skip( + f"this compute's uBridge cannot create the QEMU adapter TAPs (ubridge_tap={caps.get('ubridge_tap')})" + ) + image = _pick_image(server) + + project = compute.create_project("qemu-e2e") + pid = project["project_id"] + a1 = a2 = None + try: + n1 = compute.create_qemu_node(pid, "E2E-R1", image) + n2 = compute.create_qemu_node(pid, "E2E-R2", image) + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.qemu_anchor_name(n1_id, *ETH) + a2 = harness.qemu_anchor_name(n2_id, *ETH) + + # The link is created while both nodes are stopped: the kernel NIO + # is bound with no anchors in existence yet and wired by node start + # (the deferred-wiring path). + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert not harness.tap_exists(a1), "anchors must not exist before start" + + print(f".. starting IOSv routers on {image} (a real IOS boot — the slow leg of this scenario)") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{n2_id}/start") + c1 = _boot(server, pid, n1_id) + c2 = _boot(server, pid, n2_id) + + # Kernel objects: one persistent TAP per adapter (both adapters of + # router 1, the addressed one of router 2), and the per-link bridge + # has exactly the two addressed ports. + for adapter in range(2): + name = harness.qemu_anchor_name(n1_id, adapter) + assert harness.tap_exists(name), name + assert harness.tap_exists(a2), a2 + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + + # L2-anchor spec §E.1: anchors and the per-link bridge carry no L3 + # identity (skipped on a uBridge without `link l2only`), and both + # ports are FORWARDING — the silent-failure guard. + if harness.l2only_supported(): + harness.assert_pure_l2(a1) + harness.assert_pure_l2(a2) + harness.assert_pure_l2(bridge) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(a1) + harness.assert_forwarding(a2) + + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if=GUEST_ETH_IF) + out = c1.run("show ip int brief") + assert re.search(r"GigabitEthernet0/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + print(".. both routers configured, pinging through the anchor TAPs") + baseline = harness.wait_ping(c1, "10.1.1.2") + assert baseline["success"] == 100, baseline["raw"] + + # delay 100: netem on both anchors (each impairs one direction), + # one-way ~100 ms => RTT grows by ~200 ms + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(a1) and "netem" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + delayed = harness.wait_ping(c1, "10.1.1.2") + assert delayed["success"] == 100, delayed["raw"] + assert delayed["avg"] >= 150, (baseline, delayed) + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) + fast = harness.wait_ping(c1, "10.1.1.2") + assert fast["success"] == 100 and fast["avg"] < 50, fast + + # suspend: anchor admin-down — the e1000 loses carrier, the bridge + # port is disabled and nothing crosses; resume restores + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) + assert not harness.tap_up(a1) + dead = harness.ping(c1, "10.1.1.2", repeat=3) + assert dead["success"] == 0, dead["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) + assert harness.tap_up(a1) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + # capture: on the anchor, a real pcap + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.ping(c1, "10.1.1.2") + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # delete / re-create the kernel link: the anchor leaves the bridge + # but survives (the running VM holds it open — this is QEMU's own + # netdev), and re-creating re-enslaves it without touching the VM. + compute.call("DELETE", f"/projects/{pid}/links/{lid}") + assert harness.bridge_members(bridge) is None + assert harness.tap_exists(a1) and harness.tap_exists(a2) + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + bridge = harness.link_bridge_name(lid) + assert link["kernel_datapath"] is True, link + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + # node stop: QEMU exits and uBridge deletes every anchor of the + # node together with the orphan bridges (a stopped node must not + # litter the host with gq devices); a restart recreates the TAPs + # and rewires the link from the NIO that outlived it. + print(".. stopping and restarting router 1") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/stop") + second = harness.qemu_anchor_name(n1_id, 1) + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(second), timeout=15), ( + a1, + second, + ) + # the per-link bridge lost this end's port; it may be gone entirely + # (both endpoints delete it, last one wins) + assert harness.bridge_members(bridge) in (None, [a2]), harness.bridge_members(bridge) + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + c1.close() + c1 = _boot(server, pid, n1_id) + assert harness.tap_exists(a1) + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closed the nodes: anchors and bridges go with them + assert harness.wait_until(lambda: not harness.tap_exists(a1) and not harness.tap_exists(a2), timeout=15), ( + a1, + a2, + ) + + +def test_qemu_relay_control(): + """ + Negative control: the same topology with ``enable_kernel_datapath = + false`` (an isolated instance) wires nothing into the kernel — no + per-link bridge, no enslavement — and still pings. + + The anchors DO exist and carry: a QEMU node creates its adapter TAPs at + start whenever uBridge can (the netdev needs them), and on the relay + the anchor itself becomes the uBridge relay's AF_PACKET endpoint — up + and forwarding into the relay bridge. The server configuration only + decides whether a *link* enslaves them. + """ + + server = harness.live_server(kernel=False) + compute = server.compute + caps = compute.capabilities() + if caps.get("ubridge_tap") is not True: + pytest.skip( + f"this compute's uBridge cannot create the QEMU adapter TAPs (ubridge_tap={caps.get('ubridge_tap')})" + ) + image = _pick_image(server) + + project = compute.create_project("qemu-relay-e2e") + pid = project["project_id"] + a1 = None + try: + n1 = compute.create_qemu_node(pid, "E2E-R1", image) + n2 = compute.create_qemu_node(pid, "E2E-R2", image) + n1_id, n2_id = n1["node_id"], n2["node_id"] + a1 = harness.qemu_anchor_name(n1_id, *ETH) + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + assert link["kernel_datapath"] is False, link + + print(".. starting IOSv routers and waiting for IOS boots (relay)") + compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{n2_id}/start") + c1 = _boot(server, pid, n1_id) + c2 = _boot(server, pid, n2_id) + + # no kernel wiring: nobody enslaved the anchors, the per-link + # bridge does not exist — but the anchor is up and carrying as the + # relay's AF_PACKET endpoint + assert harness.bridge_members(harness.link_bridge_name(link["link_id"])) is None + assert harness.tap_exists(a1), a1 + assert harness.tap_up(a1), a1 + if harness.l2only_supported(): + # the L2 hardening is a creation-time property, not a datapath + # one: relay anchors are pure L2 too + harness.assert_pure_l2(a1) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if=GUEST_ETH_IF) + relay_ping = harness.wait_ping(c1, "10.1.1.2") + assert relay_ping["success"] == 100, relay_ping["raw"] + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # deleting the project closed the nodes: the adapter TAPs go with them + assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 From 6839b279be73827a00b6702001a0f0a7d0586997 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 20:57:10 +0800 Subject: [PATCH 059/101] docs(qemu): record the in-repo e2e coverage --- docs/features/qemu-kernel-datapath.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 0c9441209..e55963a55 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -138,6 +138,18 @@ Relay run the same script with the flag off: same lifecycle, filter measured at 109.8 ms (applied by uBridge's bridge — no netem on the anchor, as expected), no kernel bridge holding the tap. +The repo suite `tests/e2e/test_qemu_kernel_datapath.py` covers the same +ground with a *real* guest: two IOSv routers (a real qcow2 image, linked +clone, the real IOS CLI on the telnet console) — the full kernel lifecycle +(anchors at start, per-link bridge membership, pure-L2 anchors, real ICMP, +netem delay, suspend, capture, link delete/re-create, node stop/start +rewiring, zero residue on project delete) plus the relay negative control, +where the anchor is up and carrying as the relay's AF_PACKET endpoint (QEMU +has no other leg — the TAP is the netdev on the relay too). It needs an L3 +IOSv image in the images directory's `QEMU` subfolder; a raw node must also +name `hda_disk_interface` (the default `none` attaches no frontend device +and the VM comes up diskless — the IOSv appliance boots virtio). + ## Configuration ```ini From 6e36b58a5d7bf7fd0e55a66e5d318b8bb6ca6f7c Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 21:16:01 +0800 Subject: [PATCH 060/101] fix(compute): refuse a duplicate port NIO and wire before bookkeeping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compute-side backstop of the controller's duplicate-port guard (keep-port-links + live-port bind made the controller guard reliable; this is the layer under it). IOU and Dynamips bound the adapter NIO before the uBridge/hypervisor attach: a failed attach left the port's binding replaced — the link that still owned the port lost its handle and its host state leaked (the IOU EBUSY incident's 8 orphan TAPs). Dynamips was worse: its wiring idempotence check silently skipped the attach on a second binding and rebound only the bookkeeping. A port that already carries an NIO is now refused, and both types bind the NIO only after the wiring succeeded. --- gns3server/compute/dynamips/nodes/router.py | 16 ++++++- gns3server/compute/iou/iou_vm.py | 44 ++++++++++++------- .../dynamips/test_dynamips_kernel_datapath.py | 42 ++++++++++++++++++ tests/compute/iou/test_iou_kernel_datapath.py | 40 +++++++++++++++++ 4 files changed, 125 insertions(+), 17 deletions(-) diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index 4cb6cf964..f9e4e8bae 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -1312,12 +1312,26 @@ class Router(KernelDatapathMixin, BaseNode): if not adapter.port_exists(port_number): raise DynamipsError(f"Port {port_number} does not exist on adapter {adapter}") + # The compute-side backstop of the controller's duplicate-port guard: + # a port carries at most one NIO, and overwriting the one a link + # still owns orphans that link's teardown (its host state leaks). + # Refusing also keeps the kernel wiring's own idempotence check from + # silently skipping the attach while rebinding the bookkeeping. + if adapter.get_nio(port_number) is not None: + raise DynamipsError( + "Port {port_number} on slot {slot_number} of router '{name}' already has a link".format( + name=self._name, slot_number=slot_number, port_number=port_number + ) + ) + if isinstance(nio, NIOBridge): # Kernel datapath: the port's anchor TAP is opened in the # hypervisor and enslaved into the NIO's per-link bridge (the # wiring is deferred to the node's start when no anchors exist). - adapter.add_nio(port_number, nio) + # Wire before bookkeeping: an attach that fails mid-way leaves + # the port unbound instead of half-wired. await self._attach_kernel_nio(slot_number, port_number, nio) + adapter.add_nio(port_number, nio) log.debug( 'Router "{name}" [{id}]: {nio} bound to port {slot_number}/{port_number}'.format( name=self._name, id=self._id, nio=nio, slot_number=slot_number, port_number=port_number diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index 676d46e3a..af3d82c1a 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -1345,27 +1345,39 @@ class IOUVM(KernelDatapathMixin, BaseNode): if not adapter.port_exists(port_number): raise IOUError(f"Port {port_number} does not exist on adapter {adapter}") - adapter.add_nio(port_number, nio) - log.debug(f'IOU "{self._name}" [{self._id}]: {nio} added to {adapter_number}/{port_number}') + # The compute-side backstop of the controller's duplicate-port guard: + # a port carries at most one NIO, and overwriting the one a link + # still owns orphans that link's teardown (its host state leaks). + if adapter.get_nio(port_number) is not None: + raise IOUError( + "Port {port_number} on adapter {adapter_number} of IOU '{name}' already has a link".format( + name=self._name, adapter_number=adapter_number, port_number=port_number + ) + ) + # Wire before bookkeeping: an attach that fails mid-way leaves the + # port unbound instead of half-wired. if self.ubridge: if isinstance(nio, NIOBridge): await self._attach_kernel_nio(adapter_number, port_number, nio) - return - await self._ubridge_send( - "iol_bridge add_nio_udp {name} {iol_id} {bay} {unit} {lport} {rhost} {rport}".format( - name=self._iol_bridge_name(), - iol_id=self.application_id, - bay=adapter_number, - unit=port_number, - lport=nio.lport, - rhost=nio.rhost, - rport=nio.rport, + else: + await self._ubridge_send( + "iol_bridge add_nio_udp {name} {iol_id} {bay} {unit} {lport} {rhost} {rport}".format( + name=self._iol_bridge_name(), + iol_id=self.application_id, + bay=adapter_number, + unit=port_number, + lport=nio.lport, + rhost=nio.rhost, + rport=nio.rport, + ) ) - ) - location = self._iol_location(adapter_number, port_number) - await self._ubridge_apply_filters(location, nio.filters) - await self._ubridge_apply_markers(location, nio) + location = self._iol_location(adapter_number, port_number) + await self._ubridge_apply_filters(location, nio.filters) + await self._ubridge_apply_markers(location, nio) + + adapter.add_nio(port_number, nio) + log.debug(f'IOU "{self._name}" [{self._id}]: {nio} added to {adapter_number}/{port_number}') async def adapter_update_nio_binding(self, adapter_number, port_number, nio): """ diff --git a/tests/compute/dynamips/test_dynamips_kernel_datapath.py b/tests/compute/dynamips/test_dynamips_kernel_datapath.py index b241be526..885780524 100644 --- a/tests/compute/dynamips/test_dynamips_kernel_datapath.py +++ b/tests/compute/dynamips/test_dynamips_kernel_datapath.py @@ -270,6 +270,48 @@ async def test_attach_kernel_nio_opens_the_anchor_in_the_hypervisor_and_enslaves router._ubridge_send.assert_any_call(f'link set "{TAP00}" up') +@pytest.mark.asyncio +async def test_slot_add_nio_binding_refuses_an_occupied_port(router): + """ + The compute-side backstop of the duplicate-port guard: a second NIO for + a port that already carries one is refused here instead of overwriting + (the wiring's own idempotence check would silently skip the attach and + rebind the bookkeeping) — the occupying link's teardown needs its own + NIO handle, and its host state with it. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + first = await _nio(router) + await router.slot_add_nio_binding(0, 0, first) + assert router._slots[0].get_nio(0) is first + + second = await _nio(router, bridge="gns3other") + with pytest.raises(DynamipsError, match="already has a link"): + await router.slot_add_nio_binding(0, 0, second) + + assert router._slots[0].get_nio(0) is first + + +@pytest.mark.asyncio +async def test_slot_add_nio_binding_attach_failure_leaves_the_port_unbound(router): + """ + The bookkeeping happens last: a kernel attach that fails mid-way (the + anchor is another link's bridge port — uBridge EBUSY) must leave the + port without a binding, so the failed link creates no half-wired state. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + router._ubridge_send.side_effect = UbridgeError("208-Device or resource busy") + + nio = await _nio(router) + with pytest.raises(UbridgeError): + await router.slot_add_nio_binding(0, 0, nio) + + assert router._slots[0].get_nio(0) is None + + @pytest.mark.asyncio async def test_attach_kernel_nio_without_anchor_on_a_running_node_raises(router): """ diff --git a/tests/compute/iou/test_iou_kernel_datapath.py b/tests/compute/iou/test_iou_kernel_datapath.py index 9a49853a7..40645b279 100644 --- a/tests/compute/iou/test_iou_kernel_datapath.py +++ b/tests/compute/iou/test_iou_kernel_datapath.py @@ -33,6 +33,7 @@ from gns3server.compute.iou import IOU from gns3server.compute.iou.iou_vm import IOUVM from gns3server.compute.iou.iou_error import IOUError from gns3server.compute.nios.nio_bridge import NIOBridge +from gns3server.compute.ubridge.ubridge_error import UbridgeError NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" @@ -168,6 +169,45 @@ async def test_attach_kernel_nio_without_anchor_raises(vm): await vm._attach_kernel_nio(2, 0, nio) +@pytest.mark.asyncio +async def test_add_nio_binding_refuses_an_occupied_port(vm): + """ + The compute-side backstop of the duplicate-port guard: a second NIO for + a port that already carries one is refused here instead of overwriting + — the occupying link's teardown needs its own NIO handle, and its host + state with it. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + first = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + await vm.adapter_add_nio_binding(0, 0, first) + assert vm._ethernet_adapters[0].get_nio(0) is first + + second = vm.manager.create_nio({"type": "nio_bridge", "bridge": "gns3other"}) + with pytest.raises(IOUError, match="already has a link"): + await vm.adapter_add_nio_binding(0, 0, second) + + assert vm._ethernet_adapters[0].get_nio(0) is first + + +@pytest.mark.asyncio +async def test_add_nio_binding_attach_failure_leaves_the_port_unbound(vm): + """ + The bookkeeping happens last: an attach that fails mid-way (uBridge + EBUSY on a held anchor) must leave the port without a binding, so the + failed link creates no half-wired state. + """ + + vm._kernel_taps[(0, 0)] = TAP00 + vm._ubridge_send.side_effect = UbridgeError("208-Device or resource busy") + + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + with pytest.raises(UbridgeError): + await vm.adapter_add_nio_binding(0, 0, nio) + + assert vm._ethernet_adapters[0].get_nio(0) is None + + @pytest.mark.asyncio async def test_add_nio_binding_relay_keeps_the_iol_udp_path(vm): """ From cb7f74c77b71cd6965f37ebb0a13588db4b36e21 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 21:49:14 +0800 Subject: [PATCH 061/101] fix(esw): harden the cascade end the create race leaves unhardened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker create_veth hardens only its FIRST end (the Docker shape: host anchor plus a container leg that keeps its L3 life). A cascade end is that second, unhardened end whenever the peer switch's create won the race, and the kernel gives it an IPv6 link-local — a silent L3 identity on the switch fabric (ubridge-l2-anchor-spec §E). Each switch now hardens its OWN end with 'link l2only on' once the pair exists, whatever branch brought it there (fresh create, lost race, reuse); idempotent, and best-effort with a warning on a uBridge without the command. --- .../compute/builtin/nodes/ethernet_switch.py | 16 ++++++++++ .../compute/test_ethernet_switch_nodes.py | 31 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/gns3server/compute/builtin/nodes/ethernet_switch.py b/gns3server/compute/builtin/nodes/ethernet_switch.py index d0fd0a478..2b9dab4d0 100644 --- a/gns3server/compute/builtin/nodes/ethernet_switch.py +++ b/gns3server/compute/builtin/nodes/ethernet_switch.py @@ -476,6 +476,22 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): else: with contextlib.suppress(UbridgeError): await self._ubridge_send(f'tc reset "{anchor}"') + # ``docker create_veth`` hardens only its FIRST end (the Docker + # shape: host anchor plus a container leg that keeps its L3 + # life). Here BOTH ends are anchors, and whichever switch loses + # the create race owns the second, unhardened end — its kernel + # IPv6 link-local would give the switch fabric a silent L3 + # identity (ubridge-l2-anchor-spec §E). Each switch hardens its + # OWN end once the pair exists; idempotent, and it covers a + # reused pre-existing end as well. + try: + await self._ubridge_send(f'link l2only "{anchor}" on') + except UbridgeError as e: + log.warning( + 'Ethernet switch "{name}" [{id}]: could not harden cascade end "{anchor}" L2-only: {error}'.format( + name=self._name, id=self._id, anchor=anchor, error=e + ) + ) # Register before _kernel_attach: the marker reconcile recognises # kernel anchors through _kernel_anchors(). self._kernel_ports[port_number] = anchor diff --git a/tests/api/routes/compute/test_ethernet_switch_nodes.py b/tests/api/routes/compute/test_ethernet_switch_nodes.py index 215c1d9ad..b685b6a0d 100644 --- a/tests/api/routes/compute/test_ethernet_switch_nodes.py +++ b/tests/api/routes/compute/test_ethernet_switch_nodes.py @@ -956,6 +956,37 @@ class TestEthernetSwitchNodesRoutes: node._ubridge_send.assert_any_call(f'brctl addif "{br}" "{anchor}"') assert node._kernel_ports[0] == anchor + async def test_ethernet_switch_create_cascade_nio_hardens_own_end_l2only( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + ``docker create_veth`` hardens only its FIRST end (the Docker shape: + host anchor plus a container leg that keeps its L3 life) — but a + cascade end the peer's winning create minted is that second, + unhardened end, and the kernel gives it an IPv6 link-local (a silent + L3 identity on the switch fabric, ubridge-l2-anchor-spec §E). Each + switch therefore hardens its OWN end once the pair exists, whatever + branch brought it there (fresh create, lost race, reuse). + """ + + anchor, peer = "gs1a2b3c4d5e0", "gs1a2b3c4d5e1" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + # pre-existing end: the branch whose end create_veth did NOT harden + with self._cascade_sysfs(anchor, {"value": True}): + response = await compute_client.post(url, json=self._cascade_params(anchor, peer)) + assert response.status_code == status.HTTP_201_CREATED + + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.assert_any_call(f'link l2only "{anchor}" on') + # only its own end — the peer's end is the peer switch's to harden + assert not any(f'"{peer}"' in str(c) and "l2only" in str(c) for c in node._ubridge_send.call_args_list) + async def test_ethernet_switch_delete_cascade_nio_destroys_the_pair( self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict ) -> None: From ad39b22ab4f8b1e41e530f5a4e0f9f2f01b66f06 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 21:49:14 +0800 Subject: [PATCH 062/101] test(e2e): live coverage for the switch cascade kernel datapath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two switches cascaded through the link-owned veth pair, two c7200 routers per switch, real ICMP crossing both bridges: the pair is born UP with exactly one end per switch bridge, a dot1q trunk cascade carries two VLANs at once, the symmetric access/trunk mode flip on the cascade ends isolates and revives a VLAN in place (asymmetric flips are semantically broken — the access side egresses untagged into the far trunk's native PVID), a delay filter lands as netem on both ends (one per direction, RTT ~= 2x delay), suspend admin-downs both ends, and a link delete/re-create retires the whole pair and mints a new one under the new link id — zero host residue after the project goes. --- .../ethernet-switch-kernel-datapath.md | 11 + tests/e2e/README.md | 7 + tests/e2e/harness.py | 8 + tests/e2e/test_ethernet_switch_cascade.py | 278 ++++++++++++++++++ 4 files changed, 304 insertions(+) create mode 100644 tests/e2e/test_ethernet_switch_cascade.py diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index 406088b2e..410f82758 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -148,6 +148,17 @@ plus scripted reproduction rounds over the REST API): relay TAPs likewise stay enslaved across VLAN changes, with a real ping proving isolation and recovery. +The cascade has its own in-repo e2e +(`tests/e2e/test_ethernet_switch_cascade.py`): two switches joined by the +link-owned veth pair, two c7200 routers per switch, real ICMP crossing both +bridges — the pair is born UP with exactly one end per bridge, a dot1q +trunk cascade carries two VLANs at once, the symmetric access/trunk mode +flip on the cascade ends isolates and revives a VLAN in place, a `delay` +filter lands as netem on both ends (one per direction, RTT ≈ 2×delay), +suspend admin-downs both ends, and a link delete/re-create retires the whole +pair and mints a new one under the new link id — with zero host residue +after the project goes. + The host-side checks throughout are the kernel's own view: bridge membership under `/sys/class/net//brif/`, the `master` link attribute, `bridge vlan show`, and live pings where the peer allows it. diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 35e21df9b..37fb173f7 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -62,6 +62,13 @@ Current scenarios: (kernel bridge) between two c7200 routers: deferred joins on boot, VLAN isolation, absorbed-anchor filters/suspend/capture, VPCS peers staying on the relay, and a relay negative control. +* `test_ethernet_switch_cascade.py` — two Ethernet switches cascaded through + the link-owned veth pair (`gs0/1`, each end enslaved into its own + switch's bridge): a dot1q trunk cascade carrying two VLANs at once across + two c7200 routers per switch, the symmetric access/trunk mode flip on the + cascade ends, two-sided netem (one end per direction), suspend, link + delete/re-create (the pair dies with the link, the replacement mints new + names), zero residue. * `test_iol_docker_kernel_datapath.py` — two real iol-xe containers (Cisco CML iol-runner images): kernel link through the port bridge's swappable TAP leg (anchors at start, netem delay, suspend, capture, link diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 41488b8f3..904b2e86d 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -1104,6 +1104,14 @@ def link_bridge_name(link_id): return "gns3" + link_id.replace("-", "")[:11] +def cascade_end_names(link_id): + """The two veth ends of a switch-to-switch cascade link, as + ``(end0, end1)`` — a pure function of the link id, like the per-link + bridge name (see utils.kernel_cascade_names).""" + stem = link_id.replace("-", "")[:10] + return f"gs{stem}0", f"gs{stem}1" + + def anchor_name(node_id, adapter, port): """The dynamips anchor TAP name for a slot/port (see Router._tap_name).""" return "gd" + node_id.replace("-", "")[:8] + f"e{adapter}p{port}" diff --git a/tests/e2e/test_ethernet_switch_cascade.py b/tests/e2e/test_ethernet_switch_cascade.py new file mode 100644 index 000000000..69d3653b4 --- /dev/null +++ b/tests/e2e/test_ethernet_switch_cascade.py @@ -0,0 +1,278 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Live end-to-end test for the switch-to-switch cascade kernel datapath +(pytest marker ``e2e``): two Ethernet switches (two Linux kernel bridges) +interconnected by a link-owned veth pair, with two c7200 routers hanging +off each switch. Real ICMP crossing both switches proves the datapath: + +* the cascade link reports ``kernel_datapath`` and the veth pair + (``gs0/1``, a pure function of the link id) exists at once + — switches are always-on, nothing defers — with each switch's bridge + carrying exactly its own end as a member, both ends UP; +* a dot1q trunk cascade carries two VLANs at once: hosts in VLAN 10 and + VLAN 20 ping across both switches simultaneously (tagged frames cross + the pair, which is what a trunk means here); +* flipping both cascade ports symmetrically to access VLAN 10 (an + in-place ``_reconfigure_port_vlan`` on each veth end) keeps VLAN 10 + flowing untagged and kills VLAN 20; restoring the trunk revives VLAN 20. + Asymmetric flips are deliberately not tested: an access end egresses + untagged, and the far trunk end would park those frames in its native + VLAN — symmetric modes are the only sane inter-switch configuration; +* a ``delay 100`` filter lands as netem on BOTH veth ends (each direction + impaired exactly once, like any two-sided kernel link) and the measured + RTT grows by ~2 x 100 ms; +* suspend admin-downs both ends and kills both VLANs; resume restores; +* deleting the cascade link destroys the whole veth pair from either side + (the host links and their anchors survive); re-creating the link mints a + new pair under the new link id and the traffic returns without touching + the routers; +* deleting the project removes both switch bridges, both veth pairs and + the router anchors — zero host residue. + +No relay control twin here: on the relay datapath a cascade is just two +per-port uBridge UDP relays, the machinery +``test_ethernet_switch_kernel_datapath.py::test_ethernet_switch_relay_control`` +already drives. +""" + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# c7200 slot 0 is the fixed IO slot; the 2-port Ethernet PA goes in slot 1, +# so f1/0 = (1, 0) and f1/1 = (1, 1). +SLOTS = {"slot1": "PA-2FE-TX"} +ETH10 = (1, 0) +ETH20 = (1, 1) + +IP10 = "10.1.10.%s" +IP20 = "10.1.20.%s" + + +def _pick_image(server): + images = server.compute.dynamips_images() + image = next((name for name in images if name.startswith("c7200")), None) + if not image: + pytest.skip(f"no c7200 image available on this server (images: {images[:5]})") + return image + + +def _switch_bridge_name(switch_id): + # see EthernetSwitch._ensure_bridge + return "gns3" + switch_id.replace("-", "")[:6] + + +def _ports(cascade_type): + """The switches' ports: two access ports (VLAN 10 / 20) and the cascade + port — dot1q trunk (native 1) or access 10 for the symmetric flip.""" + return [ + {"name": "Ethernet0", "port_number": 0, "type": "access", "vlan": 10}, + {"name": "Ethernet1", "port_number": 1, "type": "access", "vlan": 20}, + {"name": "Ethernet2", "port_number": 2, "type": cascade_type, "vlan": 1 if cascade_type == "dot1q" else 10}, + ] + + +def _create_switch(compute, pid, name): + return compute.call( + "POST", + f"/projects/{pid}/nodes", + { + "compute_id": "local", + "name": name, + "node_type": "ethernet_switch", + "properties": {"ports_mapping": _ports("dot1q")}, + }, + ) + + +def _set_ports(compute, pid, switch_id, cascade_type): + compute.call("PUT", f"/projects/{pid}/nodes/{switch_id}", {"properties": {"ports_mapping": _ports(cascade_type)}}) + + +def _add_eth(console, ip, ifname): + console.run("conf t") + console.run(f"interface {ifname}") + console.run(f"ip address {ip} 255.255.255.0") + console.run("no shutdown") + console.run("end") + + +def _boot(server, project_id, node_id, timeout=360): + console = harness.ios_console(server, project_id, node_id) + console.boot_wait(timeout=timeout) + return console + + +def test_ethernet_switch_cascade_kernel_datapath(): + server = harness.live_server(kernel=True) + compute = server.compute + if compute.capabilities().get("ubridge_tap") is not True: + pytest.skip("this compute's uBridge cannot create persistent TAPs (ubridge_tap != True)") + image = _pick_image(server) + idlepc = harness.dynamips_idlepc(server, image) + + project = compute.create_project("esw-cascade-e2e") + pid = project["project_id"] + bridges = [] + try: + sw1 = _create_switch(compute, pid, "E2E-SW1") + sw2 = _create_switch(compute, pid, "E2E-SW2") + sw1_id, sw2_id = sw1["node_id"], sw2["node_id"] + br1, br2 = _switch_bridge_name(sw1_id), _switch_bridge_name(sw2_id) + bridges = [br1, br2] + r1 = compute.create_dynamips_router(pid, "E2E-R1", image, SLOTS, idlepc=idlepc) + r2 = compute.create_dynamips_router(pid, "E2E-R2", image, SLOTS, idlepc=idlepc) + r1_id, r2_id = r1["node_id"], r2["node_id"] + a1_10, a1_20 = harness.anchor_name(r1_id, *ETH10), harness.anchor_name(r1_id, *ETH20) + a2_10, a2_20 = harness.anchor_name(r2_id, *ETH10), harness.anchor_name(r2_id, *ETH20) + + # Both switch bridges exist from node creation; the cascade wires at + # once (switches are always-on), the host links defer on the stopped + # routers exactly like the single-switch scenario. + assert harness.bridge_members(br1) == [] + assert harness.bridge_members(br2) == [] + host_links = [ + compute.create_link(pid, (r1_id, *ETH10), (sw1_id, 0, 0)), + compute.create_link(pid, (r1_id, *ETH20), (sw1_id, 0, 1)), + compute.create_link(pid, (r2_id, *ETH10), (sw2_id, 0, 0)), + compute.create_link(pid, (r2_id, *ETH20), (sw2_id, 0, 1)), + ] + cascade = compute.create_link(pid, (sw1_id, 0, 2), (sw2_id, 0, 2)) + assert all(link["kernel_datapath"] is True for link in [*host_links, cascade]), ( + [link.get("kernel_datapath") for link in host_links], + cascade.get("kernel_datapath"), + ) + e0, e1 = harness.cascade_end_names(cascade["link_id"]) + assert harness.tap_exists(e0) and harness.tap_exists(e1) + assert harness.tap_up(e0) and harness.tap_up(e1) + assert harness.bridge_members(br1) == [e0], harness.bridge_members(br1) + assert harness.bridge_members(br2) == [e1], harness.bridge_members(br2) + + print(".. starting routers and waiting for IOS boots") + compute.call("POST", f"/projects/{pid}/nodes/{r1_id}/start") + compute.call("POST", f"/projects/{pid}/nodes/{r2_id}/start") + c1 = _boot(server, pid, r1_id) + c2 = _boot(server, pid, r2_id) + + # The deferred host joins completed via the node-start re-push: each + # switch bridges its two router anchors plus its cascade end. + assert harness.wait_until(lambda: harness.bridge_members(br1) == sorted([a1_10, a1_20, e0]), timeout=15), ( + harness.bridge_members(br1) + ) + assert harness.wait_until(lambda: harness.bridge_members(br2) == sorted([a2_10, a2_20, e1]), timeout=15), ( + harness.bridge_members(br2) + ) + + harness.configure_ios(c1, "R1", eth_ip=IP10 % 1, eth_if="f1/0") + _add_eth(c1, IP20 % 1, "f1/1") + harness.configure_ios(c2, "R2", eth_ip=IP10 % 2, eth_if="f1/0") + _add_eth(c2, IP20 % 2, "f1/1") + print(".. both routers configured, pinging across the cascade") + + # A trunk cascade carries both VLANs at once — two simultaneous + # adjacencies over the same veth pair is the tagged-frames proof. + baseline10 = harness.wait_ping(c1, IP10 % 2) + assert baseline10["success"] == 100, baseline10["raw"] + baseline20 = harness.wait_ping(c1, IP20 % 2) + assert baseline20["success"] == 100, baseline20["raw"] + + # L2-anchor spec §E.1: the cascade ends and both bridges carry no L3 + # identity, and the bridged ends are FORWARDING. + if harness.l2only_supported(): + harness.assert_pure_l2(e0) + harness.assert_pure_l2(e1) + harness.assert_pure_l2(br1) + harness.assert_pure_l2(br2) + else: + print(".. uBridge without link l2only: skipping the §E.1 assertions") + harness.assert_forwarding(e0) + harness.assert_forwarding(e1) + + # Symmetric mode flip on the cascade ports (in place, no member ever + # leaves its bridge): access 10 passes VLAN 10 untagged and drops + # VLAN 20; the trunk restores the dual-VLAN crossing. + _set_ports(compute, pid, sw1_id, "access") + _set_ports(compute, pid, sw2_id, "access") + assert harness.bridge_members(br1) == sorted([a1_10, a1_20, e0]), harness.bridge_members(br1) + assert harness.bridge_members(br2) == sorted([a2_10, a2_20, e1]), harness.bridge_members(br2) + assert harness.wait_ping(c1, IP10 % 2)["success"] == 100 + isolated = harness.ping(c1, IP20 % 2, repeat=3) + assert isolated["success"] == 0, isolated["raw"] + _set_ports(compute, pid, sw1_id, "dot1q") + _set_ports(compute, pid, sw2_id, "dot1q") + rejoined = harness.wait_ping(c1, IP20 % 2, attempts=5) + assert rejoined["success"] == 100, rejoined["raw"] + + # Two-sided impairment: each veth end is its own interface, so the + # filter lands on both — one netem per direction, RTT ~= 2 x delay. + compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(e0), harness.qdiscs(e0) + assert "netem" in harness.qdiscs(e1), harness.qdiscs(e1) + delayed = harness.wait_ping(c1, IP10 % 2) + assert delayed["success"] == 100, delayed["raw"] + assert 150 <= delayed["avg"] <= 350, (baseline10, delayed) + compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"filters": {}}) + assert "netem" not in harness.qdiscs(e0) and "netem" not in harness.qdiscs(e1) + fast = harness.wait_ping(c1, IP10 % 2) + assert fast["success"] == 100 and fast["avg"] < 50, fast + + # suspend: both ends admin-down, both VLANs die; resume restores. + compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"suspend": True}) + assert not harness.tap_up(e0) and not harness.tap_up(e1) + assert harness.ping(c1, IP10 % 2, repeat=3)["success"] == 0 + assert harness.ping(c1, IP20 % 2, repeat=3)["success"] == 0 + compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"suspend": False}) + assert harness.tap_up(e0) and harness.tap_up(e1) + assert harness.wait_ping(c1, IP10 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP20 % 2, attempts=5)["success"] == 100 + + # Cascade link fault: the whole veth pair dies with the link (either + # side's teardown), the host links and their anchors survive; a + # re-create mints a new pair under the new link id and the traffic + # returns without touching the routers. + compute.call("DELETE", f"/projects/{pid}/links/{cascade['link_id']}") + assert not harness.tap_exists(e0) and not harness.tap_exists(e1) + assert harness.bridge_members(br1) == sorted([a1_10, a1_20]), harness.bridge_members(br1) + assert harness.bridge_members(br2) == sorted([a2_10, a2_20]), harness.bridge_members(br2) + assert harness.ping(c1, IP10 % 2, repeat=3)["success"] == 0 + cascade = compute.create_link(pid, (sw1_id, 0, 2), (sw2_id, 0, 2)) + assert cascade["kernel_datapath"] is True, cascade + n0, n1 = harness.cascade_end_names(cascade["link_id"]) + assert harness.wait_until(lambda: harness.bridge_members(br1) == sorted([a1_10, a1_20, n0]), timeout=10) + assert harness.wait_until(lambda: harness.bridge_members(br2) == sorted([a2_10, a2_20, n1]), timeout=10) + assert harness.wait_ping(c1, IP10 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP20 % 2, attempts=5)["success"] == 100 + + c1.close() + c2.close() + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid, failed=False) + + # Project delete closed everything: both switch bridges, both veth pairs + # (the faulted one and its replacement) and all four router anchors gone. + for bridge in bridges: + assert harness.wait_until(lambda b=bridge: harness.bridge_members(b) is None, timeout=15), bridge + for end in (e0, e1, n0, n1): + assert not harness.tap_exists(end), end + for anchor in (a1_10, a1_20, a2_10, a2_20): + assert not harness.tap_exists(anchor), anchor From 4fc0398423e4229f6f1820ea72253a31aa447cfe Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 22:00:00 +0800 Subject: [PATCH 063/101] test(e2e): live coverage for markers on the kernel datapath Two Alpine containers on a kernel-only link with traffic-insight markers attached to a veth anchor (marker add_kernel), observed on the dedicated marker WebSocket via a new harness piece (WebSocketCollector, a sync facade over websockets' sync client). Real ICMP proves: the marker.match identity is complete (filter/tag/link_id/capture node/ts/len and the tx/rx direction set of a ping), the main project channel carries no matches, the BPF and direction are enforced (a non-matching sibling and a tx-only marker each see exactly their traffic), the host pcap holds exactly the matched frames, pausing silences one marker while a sibling keeps counting (silence is never a dead channel), tag replay serves the paused tag's timeline with display-filter/link/window narrowing and refuses with 409 while capturing, and deleting a marker takes its pcap and signals with it. --- docs/features/marker-tag-replay.md | 6 + docs/features/marker-traffic-insight.md | 9 + tests/e2e/README.md | 18 +- tests/e2e/harness.py | 66 ++++++ tests/e2e/test_marker_kernel_datapath.py | 280 +++++++++++++++++++++++ 5 files changed, 375 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/test_marker_kernel_datapath.py diff --git a/docs/features/marker-tag-replay.md b/docs/features/marker-tag-replay.md index 51b35d3ac..d0dd346f5 100644 --- a/docs/features/marker-tag-replay.md +++ b/docs/features/marker-tag-replay.md @@ -314,6 +314,12 @@ All error bodies are `{"message": "…"}` (the app's unified format). ## Notes +- **In-repo e2e coverage (2026-10).** `tests/e2e/test_marker_kernel_datapath.py` exercises + the whole contract against a paused tag on real marker pcaps: the `range` aggregate + (frame_count, sources inventory with the node/link/marker identity, first-frame + src/dst/proto from sharkd), the display filter narrowing (`icmp.type == 8`), the + unknown-link empty stream, the `frames` window endpoint, and the 409 gate while a + marker under the tag is still capturing. - **Heterogeneous link types coexist.** Frames are never merged into a single pcap (mergecap is deliberately not used) — each frame carries its source and is decoded individually, so Ethernet and serial (cHDLC/PPP) markers can share one timeline. diff --git a/docs/features/marker-traffic-insight.md b/docs/features/marker-traffic-insight.md index 4beb725c2..a27ad5a61 100644 --- a/docs/features/marker-traffic-insight.md +++ b/docs/features/marker-traffic-insight.md @@ -344,6 +344,15 @@ direction relative to the capture node; see [Direction](#direction). ## Notes +- **Live e2e coverage.** `tests/e2e/test_marker_kernel_datapath.py` drives two real + Alpine containers over a kernel-only link with markers attached to a veth anchor + (`marker add_kernel`): it asserts the exact `marker.match` identity on the dedicated + marker WebSocket (filter/tag/link_id/node_id/ts/len and the tx/rx direction set of a + ping), that the main project channel carries no `marker.match`, that a non-matching + BPF and a `tx`-directioned sibling each see exactly their intended traffic, the host + pcap frame count, pause/resume silence counting against a still-firing sibling, + marker delete removing the pcap and silencing signals, and the empty marker + directory at the end. - **Marker name is immutable.** It is the identifier across the controller, the uBridge filter, the pcap filename, and `MARK` signal routing — so rename is a delete + recreate, not a field update. PUT ignores the body `name`; the `{name}` path parameter identifies diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 37fb173f7..d29b18394 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -47,10 +47,11 @@ fatal: the routers simply run without one). `harness.py` holds the shared pieces — REST client, server lifecycle, the IOS console driver, the Docker guest driver (a tiny Docker Engine API -client over the daemon socket, plus the digest-pinned image helper), -host-side kernel inspections (bridge membership, tap admin state, tc -qdiscs, the L2-anchor spec §E assertions) — so a new node type adds only -its own scenario module. +client over the daemon socket, plus the digest-pinned image helper), a +WebSocket notification collector (a sync facade over `websockets`' sync +client for asserting on live notification streams), host-side kernel +inspections (bridge membership, tap admin state, tc qdiscs, the L2-anchor +spec §E assertions) — so a new node type adds only its own scenario module. Current scenarios: @@ -93,6 +94,15 @@ Current scenarios: each measured through real traffic), plus a relay negative control and a relay→kernel reopen upgrade (the server's datapath choice is flipped across a restart). +* `test_marker_kernel_datapath.py` — two real Alpine containers on a + kernel-only link with traffic-insight markers attached to a veth anchor + (`marker add_kernel`): `marker.match` signals collected off the dedicated + marker WebSocket with their full identity (filter/tag/link_id/node_id/len + and the tx/rx direction set of a ping), the main project channel asserted + free of matches, BPF and direction enforcement, host pcap exactness, + pause/resume silence counting against a still-firing sibling, tag replay + (timeline, display filter, link narrowing, window, the 409 gate) and + marker delete taking pcap plus signals with it. The Docker scenarios run a digest-pinned `alpine:3`, so every developer tests the same bytes: a cached copy in the local Docker daemon costs no diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 904b2e86d..bff5397c9 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -62,12 +62,14 @@ import socket import subprocess import sys import tempfile +import threading import time import urllib.error import urllib.parse import urllib.request import pytest +from websockets.sync.client import connect as ws_connect REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -468,6 +470,70 @@ def release(server, project_id, failed=False): server.stop() +# --------------------------------------------------------------------------- +# WebSocket notification collector +# --------------------------------------------------------------------------- + + +class WebSocketCollector: + """Collects server notification messages from one WebSocket. + + The e2e tests are synchronous; this is a sync facade over ``websockets``' + sync client: a background thread drains the socket into a list and the + test polls :meth:`events` / :meth:`wait_events`. Messages are the + server's ``{"action": ..., "event": ...}`` JSON objects. + """ + + def __init__(self, server, path, open_timeout=15): + base = server.compute.url.replace("https", "wss", 1).replace("http", "ws", 1) + self._ws = ws_connect(f"{base}{path}?token={server.compute.token}", open_timeout=open_timeout) + self._messages = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._pump, daemon=True) + self._thread.start() + + def _pump(self): + while not self._stop.is_set(): + try: + raw = self._ws.recv(timeout=0.5) + except TimeoutError: + continue + except Exception: + break # socket closed under us: stop draining + try: + self._messages.append(json.loads(raw)) + except ValueError: + continue + + def messages(self, action=None): + """Every message received so far, optionally only for one action.""" + messages = list(self._messages) + if action is not None: + messages = [m for m in messages if m.get("action") == action] + return messages + + def events(self, action): + """The ``event`` payload of every message carrying the action.""" + return [m.get("event") for m in self.messages(action)] + + def wait_events(self, action, count=1, timeout=10): + """Poll until at least *count* events of the action arrived (or the + timeout passes); returns whatever was collected either way.""" + deadline = time.time() + timeout + while time.time() < deadline: + events = self.events(action) + if len(events) >= count: + return events + time.sleep(0.2) + return self.events(action) + + def close(self): + self._stop.set() + with contextlib.suppress(Exception): + self._ws.close() + self._thread.join(timeout=5) + + # --------------------------------------------------------------------------- # Dynamips idle-PC (CPU) # --------------------------------------------------------------------------- diff --git a/tests/e2e/test_marker_kernel_datapath.py b/tests/e2e/test_marker_kernel_datapath.py new file mode 100644 index 000000000..fba6b5d3f --- /dev/null +++ b/tests/e2e/test_marker_kernel_datapath.py @@ -0,0 +1,280 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +Live end-to-end test for traffic-insight markers on the kernel datapath +(pytest marker ``e2e``): two real Alpine containers on a kernel-only link, +markers attached to a veth anchor (``marker add_kernel``) and observed on +the dedicated marker notification WebSocket. Real ICMP proves: + +* a marker match arrives as a ``marker.match`` on + ``/projects/{id}/notifications/markers/ws`` with the full identity — + filter name, tag, link_id, capture node, timestamp, packet length, + travel direction — and **nothing** on the main project channel (the + whole point of the dedicated stream: matches must not head-of-line-block + topology events); +* the BPF is enforced on the anchor: a sibling marker whose expression + cannot match the traffic stays silent, a directioned marker only sees + its direction (``tx`` = the capture node sending), and the pcap on the + host holds exactly the matched frames; +* pausing a marker silences its signals while keeping its pcap; resuming + restores them — and other markers' matches keep flowing throughout, so + silence is never just "the channel died"; +* tag replay serves the paused tag's aggregate timeline off that pcap + (merged frames with src/dst/proto, per-source inventory, display-filter + and link narrowing, the window endpoint) and refuses with 409 while any + marker under the tag is still capturing (skipped when sharkd is absent); +* deleting a marker removes its pcap and silences its signals; deleting + the project leaves no marker state behind. +""" + +import json +import os +import shutil +import urllib.error +import urllib.parse +import urllib.request + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# every standard Docker adapter has a single port +ETH = (0, 0) +R1_IP = "10.1.1.1" +R2_IP = "10.1.1.2" +TAG = 4242 + +# a busybox `ping` default frame: 14 Ethernet + 20 IP + 8 ICMP + 56 data. +PING_FRAME_LEN = 98 + + +def _resolve_container(daemon, node): + """Fill in the node's container id from the daemon (the controller's + node payload does not carry the compute-side id), by the deterministic + name the server gives the container.""" + container_id = daemon.container_id(f"GNS3.{node['name']}.{node['project_id']}") + assert container_id, f"container for {node['name']} not found on the Docker daemon" + node["container_id"] = container_id + return node + + +def _create_topology(server, pid, image): + compute = server.compute + daemon = harness.DockerDaemon() + nodes = [] + for suffix, address in (("1", R1_IP), ("2", R2_IP)): + node = compute.create_docker_node(pid, f"MARKER-E2E-{suffix}", image) + harness.configure_docker_interfaces(node, address) + nodes.append(_resolve_container(daemon, node)) + return nodes[0], nodes[1], daemon + + +def _start_topology(compute, pid, daemon, nodes): + """Start the containers and wait for init.sh to have applied their + addresses (busybox ifup runs a moment after container start).""" + for node in nodes: + compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start") + for node, address in zip(nodes, (R1_IP, R2_IP), strict=False): + harness.docker_wait_address(daemon, node["container_id"], address) + + +def _raw(compute, path): + """GET without raising on an error status: returns ``(status, body)`` + (body parsed as JSON when possible, else the raw text). The replay + endpoints are asserted on their status codes (409 gate / 200 data).""" + # S310: the base URL comes from the operator (GNS3_E2E_URL or the + # isolated instance this test just started), not untrusted input + req = urllib.request.Request(compute.url + path, method="GET") # noqa: S310 + req.add_header("Authorization", f"Bearer {compute.token}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310 + payload = resp.read() + return resp.status, json.loads(payload) if payload else None + except urllib.error.HTTPError as e: + return e.code, e.read().decode(errors="replace") + + +def _matches(marker_ws, marker_name): + return [e for e in marker_ws.events("marker.match") if e["filter"] == marker_name] + + +def _close(*collectors): + for collector in collectors: + if collector is not None: + collector.close() + + +def test_marker_kernel_datapath(): + server = harness.live_server(kernel=True) + compute = server.compute + image = harness.ensure_docker_image(server) + + project = compute.create_project("marker-e2e") + pid = project["project_id"] + marker_ws = main_ws = None + try: + n1, n2, daemon = _create_topology(server, pid, image) + n1_id, n2_id = n1["node_id"], n2["node_id"] + c1 = n1["container_id"] + + link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) + lid = link["link_id"] + assert link["kernel_datapath"] is True, link + _start_topology(compute, pid, daemon, (n1, n2)) + baseline = harness.docker_wait_ping(daemon, c1, R2_IP) + assert baseline["loss"] == 0, baseline["raw"] + + a1 = harness.docker_anchor_name(n1_id, *ETH) + markers_dir = os.path.join(compute.call("GET", f"/projects/{pid}")["path"], "project-files", "markers") + + # Both channels open before the markers exist so no match is missed. + marker_ws = harness.WebSocketCollector(server, f"/projects/{pid}/notifications/markers/ws") + main_ws = harness.WebSocketCollector(server, f"/projects/{pid}/notifications/ws") + + print(".. attaching markers to the kernel link") + m = compute.call( + "POST", + f"/projects/{pid}/links/{lid}/markers", + {"name": "m-icmp", "bpf": "icmp", "tag": TAG, "capture_node_id": n1_id}, + ) + assert m["capture_node_id"] == n1_id and m["tag"] == TAG and m["enabled"] is True, m + # a sibling whose expression cannot match ICMP traffic — it must stay + # silent while the others fire (the BPF is the filter, not the marker) + compute.call( + "POST", f"/projects/{pid}/links/{lid}/markers", {"name": "m-tcp", "bpf": "tcp port 9999", "tag": 4343} + ) + # direction is relative to the capture node: node1 sending only + m_tx = compute.call( + "POST", + f"/projects/{pid}/links/{lid}/markers", + {"name": "m-tx", "bpf": "icmp", "direction": "tx", "tag": 4344, "capture_node_id": n1_id}, + ) + assert m_tx["direction"] == "tx", m_tx + + print(".. sending 5 ICMP echoes and collecting marker.match signals") + result = harness.docker_ping(daemon, c1, R2_IP, count=5) + assert result["loss"] == 0, result["raw"] + + harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 10, timeout=10) + icmp = _matches(marker_ws, "m-icmp") + assert len(icmp) == 10, len(icmp) # 5 requests (tx) + 5 replies (rx) + assert not _matches(marker_ws, "m-tcp"), "a non-matching BPF must stay silent" + assert len(_matches(marker_ws, "m-tx")) == 5, "the tx direction sees requests only" + + for event in icmp: + assert event["node_id"] == n1_id, event + assert event["link_id"] == lid, event + assert event["tag"] == TAG, event + assert event["ts"] > 0, event + assert event["len"] == PING_FRAME_LEN, event + assert {e["dir"] for e in icmp} == {"tx", "rx"}, [e["dir"] for e in icmp] + assert {e["dir"] for e in _matches(marker_ws, "m-tx")} == {"tx"} + + # The dedicated channel carries matches; the main project stream must + # not (and must still be alive — its link.updated proves the socket). + assert main_ws.messages("marker.match") == [], main_ws.messages("marker.match")[:3] + assert main_ws.messages("link.updated"), "the main channel is not receiving at all" + + # Host pcap: exactly the matched frames, off the anchor + pcap = os.path.join(markers_dir, f"{n1_id}_{lid}_m-icmp.pcap") + assert os.path.exists(pcap), os.listdir(markers_dir) + count, ethertypes = harness.pcap_records(pcap) + assert count == 10, count + assert set(ethertypes) == {"0800"}, ethertypes + + # pause: this marker's signals stop, its pcap survives, and the + # sibling marker's matches keep flowing (the channel is alive — + # silence is the pause, not a dead stream) + print(".. pausing m-icmp: its signals stop while the siblings keep firing") + compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp", {"enabled": False}) + harness.docker_ping(daemon, c1, R2_IP, count=3) + harness.wait_until(lambda: len(_matches(marker_ws, "m-tx")) >= 8, timeout=10) + assert len(_matches(marker_ws, "m-icmp")) == 10, "a paused marker must not signal" + assert os.path.exists(pcap) + + # resume: signals come back, pcap keeps appending + print(".. resuming m-icmp") + compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp", {"enabled": True}) + harness.docker_ping(daemon, c1, R2_IP, count=2) + harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 14, timeout=10) + assert len(_matches(marker_ws, "m-icmp")) == 14 + assert harness.pcap_records(pcap)[0] == 14 + + if shutil.which("sharkd") is None: + print(".. sharkd is not installed: skipping the tag replay section") + else: + # Tag replay reads the pcaps at rest: pause every marker of the + # tag. m-icmp is the only one carrying TAG. + print(f".. pausing m-icmp and replaying tag {TAG}") + compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp", {"enabled": False}) + status, timeline = _raw(compute, f"/projects/{pid}/markers/tags/{TAG}/replay/range") + assert status == 200, timeline + assert timeline["tag"] == TAG + assert timeline["frame_count"] == 14, timeline["frame_count"] + assert timeline["start"] and timeline["end"], timeline + assert timeline["sources"] == [ + { + "node_id": n1_id, + "link_id": lid, + "marker": "m-icmp", + "data_link_type": "DLT_EN10MB", + "count": 14, + } + ], timeline["sources"] + first = timeline["frames"][0] + assert first["src"] == R1_IP and first["dst"] == R2_IP, first + assert "ICMP" in (first["proto"] or ""), first + + # display filter: echo requests only (5 + 2 of the 14) + query = urllib.parse.urlencode({"filter": "icmp.type == 8"}) + status, filtered = _raw(compute, f"/projects/{pid}/markers/tags/{TAG}/replay/range?{query}") + assert status == 200, filtered + assert filtered["frame_count"] == 7, filtered["frame_count"] + # unknown link: an empty stream, same shape as a zero-match filter + query = urllib.parse.urlencode({"link": "00000000-0000-0000-0000-000000000000"}) + status, empty = _raw(compute, f"/projects/{pid}/markers/tags/{TAG}/replay/range?{query}") + assert status == 200 and empty["frame_count"] == 0 and empty["frames"] == [], empty + # window endpoint: the whole capture inside one window + query = urllib.parse.urlencode({"ts": timeline["start"], "window_ms": 120000, "limit": 1000}) + status, window = _raw(compute, f"/projects/{pid}/markers/tags/{TAG}/replay/frames?{query}") + assert status == 200 and len(window["frames"]) == 14, window + + # the gate: replay refuses while the tag is still capturing + compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp", {"enabled": True}) + status, gate = _raw(compute, f"/projects/{pid}/markers/tags/{TAG}/replay/range") + assert status == 409, (status, gate) + + # delete: the pcap goes with the marker and the signals stop + print(".. deleting m-icmp: pcap and signals go with it") + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-icmp") + assert harness.wait_until(lambda: not os.path.exists(pcap), timeout=10), pcap + harness.docker_ping(daemon, c1, R2_IP, count=2) + harness.wait_until(lambda: len(_matches(marker_ws, "m-tx")) >= 12, timeout=10) + assert len(_matches(marker_ws, "m-icmp")) == 14, "a deleted marker must not signal" + + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-tcp") + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-tx") + assert not os.listdir(markers_dir), os.listdir(markers_dir) + except BaseException: + harness.release(server, pid, failed=True) + _close(marker_ws, main_ws) + raise + _close(marker_ws, main_ws) + harness.release(server, pid, failed=False) From 9d937802d511ab8c98406d7b109a6428228c6105 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 22:13:30 +0800 Subject: [PATCH 064/101] test(e2e): relay filters shape real traffic on both relay engines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the docker and IOU relay negative controls with the filters the relay itself supports (delay, frequency_drop, bpf), asserted through real ICMP — the first live coverage of the relay filter path: * bridge engine (docker): delay 100 grows the RTT by ~200 ms and comes back on clear; frequency_drop 2 kills the round trips (the filter counts every packet crossing the bridge, both directions sharing one counter, so on an alternating ping stream one whole direction dies; verified against uBridge directly), frequency_drop -1 blackholes, and bpf drops by libpcap expression ('icmp', then 'greater 150' discriminating by frame size). * iol_bridge engine (IOU): the same delay and frequency_drop effects through the port's iol_bridge filter list, plus the available_filters spot check. Docker 33 s, IOU 168 s, both green. --- docs/features/docker-kernel-datapath.md | 7 ++-- docs/features/iou-kernel-datapath.md | 3 +- tests/e2e/README.md | 11 ++++-- tests/e2e/test_docker_kernel_datapath.py | 48 +++++++++++++++++++++++- tests/e2e/test_iou_kernel_datapath.py | 36 +++++++++++++++++- 5 files changed, 94 insertions(+), 11 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 2d400adea..ed1212226 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -465,9 +465,10 @@ L2-anchor spec §E assertions (no L3 identity, forwarding ports, idle silence with the guests' links down), the filter matrix one type at a time (netem core and extensions, cls_bpf match-drop, the eBPF classifier, each asserted through real traffic), suspend, capture, link delete/re-create, -container stop/start, a relay negative control, and the relay→kernel -upgrade on project reopen (the datapath choice is flipped across a server -restart). The container image is digest-pinned and pulled through the +container stop/start, a relay negative control — whose own filters +(delay/frequency_drop/bpf) are asserted to really shape the relay wire as +uBridge userspace bridge filters — and the relay→kernel upgrade on project +reopen (the datapath choice is flipped across a server restart). The container image is digest-pinned and pulled through the server's own pull route on a cache miss, so collaborators test the same bytes. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index a0fc2df66..e6858d152 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -148,7 +148,8 @@ IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker the server's telnet console, driving anchors-born-with-the-node, real ICMP through the port bridge, netem delay, suspend, capture, link delete/re-create, node stop/start rewiring, the serial link staying on the -relay, and a relay negative control. +relay, and a relay negative control whose real `delay`/`frequency_drop` +filters ride the port's `iol_bridge add_packet_filter` list. The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the fork), including the DOWN-anchor resilience and kernel-bridge interop this diff --git a/tests/e2e/README.md b/tests/e2e/README.md index d29b18394..bd538c2c8 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -79,7 +79,9 @@ Current scenarios: from the images directory, the real IOS CLI on the server's telnet console): kernel link through the [IOL fabric ↔ TAP] port bridge (same lifecycle as the iol-xe containers), a serial link staying on the relay - (serial bays never anchor), and a relay negative control. + (serial bays never anchor), and a relay negative control whose filters + ride the IOU-specific `iol_bridge add_packet_filter` engine (delay and + frequency_drop measured through real traffic). * `test_qemu_kernel_datapath.py` — two real IOSv routers (a real qcow2 image from the images directory's `QEMU` subfolder, linked clone, the real IOS CLI on the server's telnet console): the anchor TAP is QEMU's @@ -91,9 +93,10 @@ Current scenarios: veth datapath): the per-link kernel bridge enslaves exactly the two veth host ends, real ICMP crosses, the filter matrix runs one type at a time (netem core and extensions, cls_bpf match-drop, the eBPF classifier — - each measured through real traffic), plus a relay negative control and a - relay→kernel reopen upgrade (the server's datapath choice is flipped - across a restart). + each measured through real traffic), plus a relay negative control whose + own filters (delay, frequency_drop, bpf) really shape the wire as uBridge + userspace bridge filters, and a relay→kernel reopen upgrade (the server's + datapath choice is flipped across a restart). * `test_marker_kernel_datapath.py` — two real Alpine containers on a kernel-only link with traffic-insight markers attached to a veth anchor (`marker add_kernel`): `marker.match` signals collected off the dedicated diff --git a/tests/e2e/test_docker_kernel_datapath.py b/tests/e2e/test_docker_kernel_datapath.py index a56101a91..ec5276460 100644 --- a/tests/e2e/test_docker_kernel_datapath.py +++ b/tests/e2e/test_docker_kernel_datapath.py @@ -36,7 +36,10 @@ server itself uses. The scenarios cover: offered by ``available_filters`` and everything restores cleanly. * ``test_docker_relay_control`` — the negative control on an isolated relay-configured instance: no per-link kernel bridge, the wire is the - uBridge relay, and the kernel-only filter types are hidden. + uBridge relay, the kernel-only filter types are hidden — and the relay's + own filter set (delay, frequency_drop, bpf) really shapes the wire, as + uBridge userspace filters on the filter node's bridge + (``bridge add_packet_filter``). * ``test_docker_relay_to_kernel_reopen_upgrade`` — the server's datapath choice is flipped (relay → kernel) across a restart and the project is reopened: the link is rebuilt on the kernel datapath and forwards. @@ -481,6 +484,49 @@ def test_docker_relay_control(): assert not (offered & {"rate", "reorder", "gemodel", "duplicate", "seed", "limit", "quota", "window_drop"}), ( offered ) + + # The relay's filters are real uBridge userspace filters walked by + # the bridge's two listener threads: both directions of the wire + # cross them. delay 100 creates one delay line per direction, so the + # RTT grows by ~200 ms. + print(".. relay filters: delay 100 ms (one delay line per direction, RTT +~200 ms)") + _put_filters(compute, pid, link["link_id"], {"delay": [100]}) + harness.docker_wait_ping(daemon, n1["container_id"], R2_IP) + delayed = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5) + assert delayed["loss"] == 0, delayed["raw"] + assert delayed["avg"] >= 150, (relay_ping, delayed) + _put_filters(compute, pid, link["link_id"], {}) + fast = harness.docker_wait_ping(daemon, n1["container_id"], R2_IP) + assert fast["loss"] == 0 and fast["avg"] < 50, (relay_ping, fast) + + # frequency_drop counts every packet crossing the bridge, both + # directions sharing one counter, so on an alternating ping stream + # the same direction dies on every crossing (probed against uBridge + # directly: replies drop, round trips die wholesale). -1 is the + # drop-everything encoding the suspend emulation uses. + print(".. relay filters: frequency_drop 2 (shared counter, one direction dies)") + _put_filters(compute, pid, link["link_id"], {"frequency_drop": [2]}) + dropped = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=10) + print(f".. {dropped['loss']} % round-trip loss") + assert dropped["loss"] >= 50, dropped["raw"] + _put_filters(compute, pid, link["link_id"], {"frequency_drop": [-1]}) + blackholed = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) + assert blackholed["loss"] == 100, blackholed["raw"] + + # bpf on the relay is uBridge's libpcap userspace match-drop — a + # different engine from the kernel datapath's cls_bpf + print(".. relay filters: bpf 'icmp' drops every ICMP frame") + _put_filters(compute, pid, link["link_id"], {"bpf": ["icmp"]}) + icmp_dropped = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) + assert icmp_dropped["loss"] == 100, icmp_dropped["raw"] + print(".. relay filters: bpf 'greater 150' discriminates by frame size") + _put_filters(compute, pid, link["link_id"], {"bpf": ["greater 150"]}) + small = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) + assert small["loss"] == 0, small["raw"] + big = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4, size=300) + assert big["loss"] == 100, big["raw"] + _put_filters(compute, pid, link["link_id"], {}) + assert harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5)["loss"] == 0 except BaseException: harness.release(server, pid, failed=True) raise diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index 55b80f82c..a706b2dc9 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -34,7 +34,9 @@ Linux bridge, kernel filters, suspend and capture. ``test_iou_relay_control`` is the negative control on an isolated relay-configured instance: the same topology rides the fabric socket ↔ UDP -relay (no per-link bridge, nothing enslaved) and still pings. +relay (no per-link bridge, nothing enslaved) and still pings — with its +filters applied by the IOU-specific engine +(``iol_bridge add_packet_filter`` on the IOL port) really shaping the wire. """ import re @@ -238,7 +240,9 @@ def test_iou_relay_control(): Negative control: the same topology with ``enable_kernel_datapath = false`` (an isolated instance) wires nothing into the kernel — no per-link bridge, no enslavement — and still pings over the fabric - socket ↔ UDP relay. + socket ↔ UDP relay. The relay's filters ride the IOU-specific engine: + the IOL port's userspace filter list (``iol_bridge add_packet_filter``), + walked by both the fabric → wire and wire → fabric paths. Anchors DO exist here (an IOU node creates its port TAPs at start whenever the uBridge can terminate an IOL port on one); the server @@ -281,6 +285,34 @@ def test_iou_relay_control(): relay_ping = harness.wait_ping(c1, "10.1.1.2") assert relay_ping["success"] == 100, relay_ping["raw"] + # The relay link's filters really shape the wire on the IOU engine + # too. delay 100: one delay line per direction (the port's IOL and + # NIO sides are separate lines), so the RTT grows by ~200 ms. + lid = link["link_id"] + offered = {entry["type"] for entry in compute.call("GET", f"/projects/{pid}/links/{lid}/available_filters")} + assert "delay" in offered and "frequency_drop" in offered, offered + + print(".. iol_bridge relay filter: delay 100 ms (per-direction delay lines, RTT +~200 ms)") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) + delayed = harness.ping(c1, "10.1.1.2", repeat=5) + assert delayed["success"] == 100, delayed["raw"] + assert delayed["avg"] >= 150, (relay_ping, delayed) + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + fast = harness.ping(c1, "10.1.1.2", repeat=3) + assert fast["success"] == 100 and fast["avg"] < 50, (relay_ping, fast) + + # frequency_drop shares one counter between the port's two + # directions: on an alternating ping stream one whole direction dies + # (same semantics as the plain relay's bridge filters). + print(".. iol_bridge relay filter: frequency_drop 2 (shared counter, one direction dies)") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"frequency_drop": [2]}}) + dropped = harness.ping(c1, "10.1.1.2", repeat=6) + print(f".. success {dropped['success']} %") + assert dropped["success"] <= 50, dropped["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + recovered = harness.wait_ping(c1, "10.1.1.2") + assert recovered["success"] == 100, recovered["raw"] + c1.close() c2.close() except BaseException: From 1f544c96a532d6ce662f73977795e211581addb4 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 22:32:42 +0800 Subject: [PATCH 065/101] test(e2e): classifier spot check and filter persistence on TAP anchors The filter matrix (cls_bpf, the eBPF classifier) only ran on Docker veth host ends; the IOU and IOL-container scenarios now spot-check it on their TAP anchors and prove a filter survives a node restart: * bpf "icmp" drops everything with clsact on the anchor, both ends, and clears back to an empty qdisc; * frequency_drop 3 (the eBPF every-nth mode) measured 56 % round-trip loss (1 - (2/3)^2 = 55.6 % expected) in both scenarios; * the IOU test applies delay 100 before a router stop/start: the fresh anchor gets netem back (tc level) and the restarted router's pings measure RTT >= 150 ms until the filter is cleared (a re-attach that dropped the filter would show ~100 ms). IOU file 492 s, IOL file 182 s, both green. --- docs/features/iol-docker-kernel-datapath.md | 4 ++ docs/features/iou-kernel-datapath.md | 11 ++-- tests/e2e/README.md | 16 +++--- tests/e2e/test_iol_docker_kernel_datapath.py | 37 ++++++++++++++ tests/e2e/test_iou_kernel_datapath.py | 53 ++++++++++++++++++-- 5 files changed, 106 insertions(+), 15 deletions(-) diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index 91d3a4b08..f80f1423c 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -140,6 +140,10 @@ isolated instance, real `iol-xe/iol-xe:17-18-02` images, real uBridge with * real ICMP crosses the [unix ↔ tap] port-bridge relay (100 % ping); * `delay 100` lands as netem on both anchors, measured RTT grows ≥ 150 ms and returns < 50 ms when cleared; +* the classifier spot check on a TAP anchor (the Docker suite runs the full + matrix on veth host ends): `bpf "icmp"` drops everything (clsact on the + anchor), `frequency_drop 3` as the eBPF every-nth mode measured 56 % + round-trip loss (theory 55.6 %), both restoring on clear; * suspend admin-downs the anchor and kills the traffic; resume restores; * AF_PACKET capture writes a real pcap of the ICMP exchange; * link delete/re-create swaps the port bridge's TAP leg out and back in diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index e6858d152..774a0b39d 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -146,10 +146,13 @@ The repository's live e2e suite carries the same scenario with **real IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker `e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on the server's telnet console, driving anchors-born-with-the-node, real ICMP -through the port bridge, netem delay, suspend, capture, link -delete/re-create, node stop/start rewiring, the serial link staying on the -relay, and a relay negative control whose real `delay`/`frequency_drop` -filters ride the port's `iol_bridge add_packet_filter` list. +through the port bridge, netem delay, the TAP-anchor classifier spot check +(bpf match-drop; the eBPF every-nth mode at 56 % measured round-trip loss, +55.6 % expected), suspend, capture, link delete/re-create, node +stop/start rewiring with a delay filter restored on the fresh anchor +(tc and traffic both), the serial link staying on the relay, and a relay +negative control whose real `delay`/`frequency_drop` filters ride the +port's `iol_bridge add_packet_filter` list. The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the fork), including the DOWN-anchor resilience and kernel-bridge interop this diff --git a/tests/e2e/README.md b/tests/e2e/README.md index bd538c2c8..30dcf0e77 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -72,16 +72,18 @@ Current scenarios: names), zero residue. * `test_iol_docker_kernel_datapath.py` — two real iol-xe containers (Cisco CML iol-runner images): kernel link through the port bridge's swappable - TAP leg (anchors at start, netem delay, suspend, capture, link - delete/re-create swap, node stop/start rewiring) and a relay negative - control. + TAP leg (anchors at start, netem delay, the TAP-anchor classifier spot + check for bpf match-drop and the eBPF every-nth mode, suspend, capture, + link delete/re-create swap, node stop/start rewiring) and a relay + negative control. * `test_iou_kernel_datapath.py` — two real IOU routers (a real IOU image from the images directory, the real IOS CLI on the server's telnet console): kernel link through the [IOL fabric ↔ TAP] port bridge (same - lifecycle as the iol-xe containers), a serial link staying on the relay - (serial bays never anchor), and a relay negative control whose filters - ride the IOU-specific `iol_bridge add_packet_filter` engine (delay and - frequency_drop measured through real traffic). + lifecycle as the iol-xe containers) including the TAP-anchor classifier + spot check and a filter surviving a node restart, a serial link staying + on the relay (serial bays never anchor), and a relay negative control + whose filters ride the IOU-specific `iol_bridge add_packet_filter` engine + (delay and frequency_drop measured through real traffic). * `test_qemu_kernel_datapath.py` — two real IOSv routers (a real qcow2 image from the images directory's `QEMU` subfolder, linked clone, the real IOS CLI on the server's telnet console): the anchor TAP is QEMU's diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py index b39484e6e..735c2f00c 100644 --- a/tests/e2e/test_iol_docker_kernel_datapath.py +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -19,6 +19,9 @@ bridge, kernel filters, suspend and capture. before any link — and the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [unix ↔ TAP] port bridge relay; * a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; +* the classifier spot check on a TAP anchor: ``bpf`` match-drop and the + eBPF ``frequency_drop`` every-nth mode (the Docker suite runs the full + matrix on veth host ends); * suspend admin-downs the anchor and kills the traffic; resume restores; * link delete releases the port bridge's TAP leg (stop → delete_nio_tap) and the anchor survives — re-creating the link swaps the leg back in (the @@ -145,6 +148,40 @@ def test_iol_docker_kernel_datapath(): fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < 50, fast + # Classifier spot check on the container's TAP anchor (the Docker + # suite runs the full matrix on veth host ends): cls_bpf match-drop + # and the eBPF stateful classifier attach to a tun/tap anchor the + # same way. + tc_caps = (caps or {}).get("ubridge_tc") or {} + if tc_caps.get("cbpf"): + print(".. bpf 'icmp' drops everything on the TAP anchor") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"bpf": ["icmp"]}}) + assert "clsact" in harness.qdiscs(a1) and "clsact" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + blocked = harness.ping(c1, "10.1.1.2", repeat=4) + assert blocked["success"] == 0, blocked["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert "clsact" not in harness.qdiscs(a1), harness.qdiscs(a1) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + else: + print(".. uBridge reports no cbpf: skipping the bpf check") + + modes = tc_caps.get("ebpf_modes") or [] + if "nth" in modes: + print(".. frequency_drop 3 (eBPF nth) on the TAP anchor") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"frequency_drop": [3]}}) + nth = harness.ping(c1, "10.1.1.2", repeat=9) + loss = 100 - nth["success"] + print(f".. {loss} % round-trip loss") + # the kernel counts per direction: 1 - (2/3)^2 = 55.6 % round trip + assert 20 <= loss <= 90, nth["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + else: + print(".. uBridge reports no eBPF nth: skipping the frequency_drop check") + # suspend: anchor admin-down (the port bridge's TAP writes fail EIO, # nothing comes back); resume restores compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index a706b2dc9..aff55b417 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -21,6 +21,9 @@ Linux bridge, kernel filters, suspend and capture. bridge enslaves exactly the two of them; * real ICMP crosses the [IOL fabric ↔ TAP] port bridge; * a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; +* the classifier spot check on a TAP anchor: ``bpf`` match-drop and the + eBPF ``frequency_drop`` every-nth mode (the Docker suite runs the full + matrix on veth host ends); * suspend admin-downs the anchor and kills the traffic; resume restores; * a serial link between the same nodes never anchors (per-port exclusion): it stays on the relay, ``kernel_datapath`` is False, and real traffic @@ -29,7 +32,7 @@ Linux bridge, kernel filters, suspend and capture. re-creating the link swaps the leg back in and the traffic returns; * node stop tears the anchors down (uBridge holds their fds — the IOL bridge goes first, then ``tap delete``); a restart recreates the whole - wiring from the NIO; + wiring from the NIO, a delay filter applied before the stop included; * deleting the project leaves no anchors and no bridges behind. ``test_iou_relay_control`` is the negative control on an isolated @@ -159,6 +162,39 @@ def test_iou_kernel_datapath(): fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < 50, fast + # Classifier spot check on a TAP anchor (the Docker suite runs the + # full matrix on veth host ends): cls_bpf match-drop and the eBPF + # stateful classifier attach to a tun/tap anchor the same way. + tc_caps = (caps or {}).get("ubridge_tc") or {} + if tc_caps.get("cbpf"): + print(".. bpf 'icmp' drops everything on the TAP anchor") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"bpf": ["icmp"]}}) + assert "clsact" in harness.qdiscs(a1) and "clsact" in harness.qdiscs(a2), ( + harness.qdiscs(a1), + harness.qdiscs(a2), + ) + blocked = harness.ping(c1, "10.1.1.2", repeat=4) + assert blocked["success"] == 0, blocked["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert "clsact" not in harness.qdiscs(a1), harness.qdiscs(a1) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + else: + print(".. uBridge reports no cbpf: skipping the bpf check") + + modes = tc_caps.get("ebpf_modes") or [] + if "nth" in modes: + print(".. frequency_drop 3 (eBPF nth) on the TAP anchor") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"frequency_drop": [3]}}) + nth = harness.ping(c1, "10.1.1.2", repeat=9) + loss = 100 - nth["success"] + print(f".. {loss} % round-trip loss") + # the kernel counts per direction: 1 - (2/3)^2 = 55.6 % round trip + assert 20 <= loss <= 90, nth["raw"] + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + else: + print(".. uBridge reports no eBPF nth: skipping the frequency_drop check") + # suspend: anchor admin-down (the port bridge's TAP writes fail EIO, # nothing comes back); resume restores compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) @@ -206,8 +242,11 @@ def test_iou_kernel_datapath(): # node stop: uBridge holds the anchor fds, so the IOL bridge goes # first and the taps with it (a stopped node must not litter the # host with gi devices); a restart recreates the whole wiring from - # the NIO that outlived it. - print(".. stopping and restarting router 1") + # the NIO that outlived it — *including the NIO's filters*, which + # land on the freshly created anchor. The delay filter goes on + # before the stop so the restart has something to restore. + print(".. stopping and restarting router 1 with a delay filter applied") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/stop") assert harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15), a1 # the per-link bridge lost this end's port; it may be gone entirely @@ -218,8 +257,14 @@ def test_iou_kernel_datapath(): c1 = _boot(server, pid, n1_id) assert harness.tap_exists(a1) assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + assert harness.wait_until(lambda: "netem" in harness.qdiscs(a1), timeout=15), harness.qdiscs(a1) harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") - assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + survivor = harness.wait_ping(c1, "10.1.1.2", attempts=5) + assert survivor["success"] == 100, survivor["raw"] + assert survivor["avg"] >= 150, (baseline, survivor) + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) + cleared = harness.ping(c1, "10.1.1.2", repeat=3) + assert cleared["success"] == 100 and cleared["avg"] < 50, (baseline, cleared) c1.close() c2.close() From ff41a54364bd9824cd057bd6adc515d9817d4cee Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 23:07:56 +0800 Subject: [PATCH 066/101] =?UTF-8?q?test(e2e):=20assert=20the=20=C2=A7E.2?= =?UTF-8?q?=20idle-silence=20window=20in=20every=20kernel=20scenario?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The L2-anchor spec's behavioral guarantee — an idle link stays silent for 5 s once the enslavement burst settles — only ran in the Dynamips and Docker suites. It now also runs in QEMU, IOU, the IOL container, the switch fast path and the switch cascade, which covers every host-device creation path: * tap create (Dynamips/QEMU/IOU anchors), docker create_veth (Docker host ends; the cascade's race-loser end is hardened after create by the server), bridge add_nio_tap (the IOL container anchor, the fifth creator), and the brctl switch bridges, whose multicast snooping is off precisely so the bridge role can reach silence; * each scenario silences its guests first (IOS interfaces shut through the console; Docker guests' links down) and re-verifies the ping after; * the cascade measures 8 devices at once (four absorbed router anchors, both cascade ends, both switch bridges). IOL 158 s, cascade 293 s, ESW fast path 378 s, QEMU 428 s, IOU 308 s — all green. --- docs/design/ubridge-l2-anchor-spec.md | 12 +++++-- .../ethernet-switch-kernel-datapath.md | 14 ++++---- docs/features/iol-docker-kernel-datapath.md | 3 ++ docs/features/iou-kernel-datapath.md | 5 +-- docs/features/qemu-kernel-datapath.md | 7 ++-- tests/e2e/README.md | 8 +++-- tests/e2e/test_ethernet_switch_cascade.py | 32 +++++++++++++++++++ .../test_ethernet_switch_kernel_datapath.py | 25 +++++++++++++++ tests/e2e/test_iol_docker_kernel_datapath.py | 28 ++++++++++++++++ tests/e2e/test_iou_kernel_datapath.py | 24 ++++++++++++++ tests/e2e/test_qemu_kernel_datapath.py | 24 ++++++++++++++ 11 files changed, 166 insertions(+), 16 deletions(-) diff --git a/docs/design/ubridge-l2-anchor-spec.md b/docs/design/ubridge-l2-anchor-spec.md index 279e5ebba..168386d4c 100644 --- a/docs/design/ubridge-l2-anchor-spec.md +++ b/docs/design/ubridge-l2-anchor-spec.md @@ -224,8 +224,16 @@ is hardened like the rest — best-effort, before the NIO is handed back. assertion is what exercises the fifth creator end-to-end (the same uBridge creation path cloud's transient TAPs take); - an idle link stays silent for 5 s after the settle window - (`assert_idle_silence`, §E.2) — measured with the guest interfaces shut - (silencing IOS's own CDP/keepalives) over the anchors and the bridge; + (`assert_idle_silence`, §E.2) — measured with the guests silenced (the + IOS consoles' interfaces shut, the Docker guests' links down, all four + cascade routers shut) over the anchors, the per-link bridge and the + switch bridges, in every kernel scenario: Dynamips, Docker, QEMU, IOU, + the IOL container, the switch fast path and the switch cascade. That + covers each creation + path — `tap create`, `docker create_veth` (plus the server-side + harden-after-create on the cascade's race-loser end), `bridge + add_nio_tap` and the `brctl` switch bridges (whose multicast snooping + is off precisely so the bridge role can reach silence); - a silent-failure guard found while validating this spec: after attach, every bridge port reports `brport/state == 3` (forwarding) (`assert_forwarding`). A bridge device left DOWN keeps its ports `DISABLED` (`state == 0`) and diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index 410f82758..c57e406c2 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -152,12 +152,14 @@ The cascade has its own in-repo e2e (`tests/e2e/test_ethernet_switch_cascade.py`): two switches joined by the link-owned veth pair, two c7200 routers per switch, real ICMP crossing both bridges — the pair is born UP with exactly one end per bridge, a dot1q -trunk cascade carries two VLANs at once, the symmetric access/trunk mode -flip on the cascade ends isolates and revives a VLAN in place, a `delay` -filter lands as netem on both ends (one per direction, RTT ≈ 2×delay), -suspend admin-downs both ends, and a link delete/re-create retires the whole -pair and mints a new one under the new link id — with zero host residue -after the project goes. +trunk cascade carries two VLANs at once, the §E.2 idle-silence window with +all four router interfaces shut (the absorbed anchors, both cascade ends +and both switch bridges), the symmetric access/trunk mode flip on the +cascade ends isolates and revives a VLAN in place, a `delay` filter lands +as netem on both ends (one per direction, RTT ≈ 2×delay), suspend +admin-downs both ends, and a link delete/re-create retires the whole pair +and mints a new one under the new link id — with zero host residue after +the project goes. The host-side checks throughout are the kernel's own view: bridge membership under `/sys/class/net//brif/`, the `master` link diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index f80f1423c..cbbbcf415 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -138,6 +138,9 @@ isolated instance, real `iol-xe/iol-xe:17-18-02` images, real uBridge with * kernel link reports `kernel_datapath`, anchors exist from node start, the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [unix ↔ tap] port-bridge relay (100 % ping); +* the L2-anchor spec §E.2 silence window with the guests shut — the + behavioral half of the hardening on anchors created by uBridge's bridge + TAP module (`bridge add_nio_tap`); * `delay 100` lands as netem on both anchors, measured RTT grows ≥ 150 ms and returns < 50 ms when cleared; * the classifier spot check on a TAP anchor (the Docker suite runs the full diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 774a0b39d..8f1ad117a 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -145,8 +145,9 @@ net for the port-coordinate capture restore this work touched. The repository's live e2e suite carries the same scenario with **real IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker `e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on -the server's telnet console, driving anchors-born-with-the-node, real ICMP -through the port bridge, netem delay, the TAP-anchor classifier spot check +the server's telnet console, driving anchors-born-with-the-node, the §E.2 +idle-silence window with the guests shut, real ICMP through the port +bridge, netem delay, the TAP-anchor classifier spot check (bpf match-drop; the eBPF every-nth mode at 56 % measured round-trip loss, 55.6 % expected), suspend, capture, link delete/re-create, node stop/start rewiring with a delay filter restored on the fresh anchor diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index e55963a55..18a35843e 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -141,9 +141,10 @@ expected), no kernel bridge holding the tap. The repo suite `tests/e2e/test_qemu_kernel_datapath.py` covers the same ground with a *real* guest: two IOSv routers (a real qcow2 image, linked clone, the real IOS CLI on the telnet console) — the full kernel lifecycle -(anchors at start, per-link bridge membership, pure-L2 anchors, real ICMP, -netem delay, suspend, capture, link delete/re-create, node stop/start -rewiring, zero residue on project delete) plus the relay negative control, +(anchors at start, per-link bridge membership, pure-L2 anchors, the §E.2 +idle-silence window with the guests shut, real ICMP, netem delay, suspend, +capture, link delete/re-create, node stop/start rewiring, zero residue on +project delete) plus the relay negative control, where the anchor is up and carrying as the relay's AF_PACKET endpoint (QEMU has no other leg — the TAP is the netdev on the relay too). It needs an L3 IOSv image in the images directory's `QEMU` subfolder; a raw node must also diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 30dcf0e77..3f87fc827 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -121,9 +121,11 @@ daemon socket the server itself uses. Every kernel scenario also asserts the L2-anchor spec's server-side guarantees (`docs/design/ubridge-l2-anchor-spec.md` §E): anchors and bridges carry no L3 identity (`assert_pure_l2`), joined ports are FORWARDING -(`assert_forwarding`), and — in the Dynamips suite — an idle link with the -guests silenced stays silent for 5 s (`assert_idle_silence`). These skip on -a uBridge without `link l2only` (probed once by `harness.l2only_supported()`). +(`assert_forwarding`), and an idle link with the guests silenced stays +silent for 5 s (`assert_idle_silence`) — the guests are the IOS consoles' +interfaces shut, the Docker guests' links down, or all four cascade routers +silenced, whichever the scenario runs. These skip on a uBridge without +`link l2only` (probed once by `harness.l2only_supported()`). Set `GNS3_E2E_DEBUG=1` to run the isolated instance with debug logging (the uBridge command stream lands in its `server.log`). diff --git a/tests/e2e/test_ethernet_switch_cascade.py b/tests/e2e/test_ethernet_switch_cascade.py index 69d3653b4..6e082d175 100644 --- a/tests/e2e/test_ethernet_switch_cascade.py +++ b/tests/e2e/test_ethernet_switch_cascade.py @@ -34,6 +34,9 @@ off each switch. Real ICMP crossing both switches proves the datapath: Asymmetric flips are deliberately not tested: an access end egresses untagged, and the far trunk end would park those frames in its native VLAN — symmetric modes are the only sane inter-switch configuration; +* the L2-anchor spec §E.2 silence window with all four router interfaces + shut: the absorbed anchors, both cascade ends and both switch bridges + stay silent for 5 s; * a ``delay 100`` filter lands as netem on BOTH veth ends (each direction impaired exactly once, like any two-sided kernel link) and the measured RTT grows by ~2 x 100 ms; @@ -121,6 +124,19 @@ def _boot(server, project_id, node_id, timeout=360): return console +def _eth_shutdown(console, shut): + """Shut/unshut both router interfaces on their switch links — the only + legitimate speakers on these segments (IOS's own CDP/keepalives) — for + §E.2's silence window; the addresses stay configured and answer once the + interfaces are back.""" + console.run("conf t") + console.run("interface f1/0") + console.run("shutdown" if shut else "no shutdown") + console.run("interface f1/1") + console.run("shutdown" if shut else "no shutdown") + console.run("end") + + def test_ethernet_switch_cascade_kernel_datapath(): server = harness.live_server(kernel=True) compute = server.compute @@ -194,6 +210,22 @@ def test_ethernet_switch_cascade_kernel_datapath(): baseline20 = harness.wait_ping(c1, IP20 % 2) assert baseline20["success"] == 100, baseline20["raw"] + # L2-anchor spec §E.2: with all four router interfaces shut, every + # host-side device of this fabric stays silent for 5 s — the four + # absorbed router anchors, both cascade ends and both switch bridges + # (the bridges are created with multicast snooping off precisely so + # the bridge role can reach silence, see uBridge's brctl create). + if harness.l2only_supported(): + for console in (c1, c2): + _eth_shutdown(console, shut=True) + harness.assert_idle_silence(a1_10, a1_20, a2_10, a2_20, e0, e1, br1, br2) + for console in (c1, c2): + _eth_shutdown(console, shut=False) + assert harness.wait_ping(c1, IP10 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP20 % 2, attempts=5)["success"] == 100 + else: + print(".. uBridge without link l2only: skipping the §E.2 assertion") + # L2-anchor spec §E.1: the cascade ends and both bridges carry no L3 # identity, and the bridged ends are FORWARDING. if harness.l2only_supported(): diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index a20a92fbf..bbc0e00fa 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -24,6 +24,8 @@ Real ICMP crossing the switch proves the datapath: and defer the join; once the routers boot, the controller's re-push makes the switch join their anchors — the switch bridge's members become exactly the two router anchors (no relay TAPs anywhere); +* the L2-anchor spec §E.2 silence window with both routers shut: the + absorbed anchors and the switch bridge stay silent for 5 s; * VLAN programming applies to absorbed anchors: different access VLANs isolate the routers (ping dies), the same VLAN bridges them again; * a ``delay 100`` filter lands as netem on the absorbed anchor (single @@ -94,6 +96,16 @@ def _boot(server, project_id, node_id, timeout=360): return console +def _eth_shutdown(console, shut): + """Shut/unshut the router's f1/0 — the only legitimate speaker on the + segment (IOS's own CDP/keepalives) — for §E.2's silence window; the + address stays configured and answers once it is back.""" + console.run("conf t") + console.run("interface f1/0") + console.run("shutdown" if shut else "no shutdown") + console.run("end") + + def test_ethernet_switch_kernel_fast_path(): server = harness.live_server(kernel=True) compute = server.compute @@ -140,6 +152,19 @@ def test_ethernet_switch_kernel_fast_path(): baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + # L2-anchor spec §E.2: with both routers shut, the absorbed anchors + # and the switch bridge stay silent for 5 s (the bridge is created + # with multicast snooping off precisely so the bridge role can). + if harness.l2only_supported(): + for console in (c1, c2): + _eth_shutdown(console, shut=True) + harness.assert_idle_silence(a1, a2, sw_bridge) + for console in (c1, c2): + _eth_shutdown(console, shut=False) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + else: + print(".. uBridge without link l2only: skipping the §E.2 assertion") + # L2-anchor spec §E.1: the absorbed anchors and the switch bridge # carry no L3 identity (skipped on a uBridge without `link l2only`), # and the joined ports are FORWARDING — the silent-failure guard. diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py index 735c2f00c..addc125e6 100644 --- a/tests/e2e/test_iol_docker_kernel_datapath.py +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -18,6 +18,9 @@ bridge, kernel filters, suspend and capture. (``ubridge_bridge_tap``); the anchors (``gx`` TAPs) exist from node start, before any link — and the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [unix ↔ TAP] port bridge relay; +* the L2-anchor spec §E.2 silence window with the guests shut — the + behavioral half of the hardening on an anchor created by uBridge's + bridge TAP module; * a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; * the classifier spot check on a TAP anchor: ``bpf`` match-drop and the eBPF ``frequency_drop`` every-nth mode (the Docker suite runs the full @@ -66,6 +69,16 @@ def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): return console +def _eth_shutdown(console, shut): + """Shut/unshut the guest's Ethernet0/0 — the only legitimate speaker on + the link segment (the IOL instance's own chatter) — for §E.2's silence + window; the address stays configured and answers once it is back.""" + console.run("conf t") + console.run("interface Ethernet0/0") + console.run("shutdown" if shut else "no shutdown") + console.run("end") + + def _topology(compute, pid, image, name): n1 = compute.create_iol_router(pid, f"{name}-1", image) n2 = compute.create_iol_router(pid, f"{name}-2", image) @@ -133,6 +146,21 @@ def test_iol_docker_kernel_datapath(): baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + # L2-anchor spec §E.2: with the guests shut the anchors and the + # per-link bridge stay silent for 5 s. These anchors come from + # uBridge's bridge TAP module (`bridge add_nio_tap`) — the creator + # whose missing hardening the spec caught live; this is the + # behavioral half of that guarantee. + if harness.l2only_supported(): + for console in (c1, c2): + _eth_shutdown(console, shut=True) + harness.assert_idle_silence(a1, a2, bridge) + for console in (c1, c2): + _eth_shutdown(console, shut=False) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + else: + print(".. uBridge without link l2only: skipping the §E.2 assertion") + # delay 100: netem on both anchors (each impairs one direction), # one-way ~100 ms => RTT grows by ~200 ms compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index aff55b417..b6d3c7d88 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -20,6 +20,7 @@ Linux bridge, kernel filters, suspend and capture. bay/unit) exist from node start, before any link — and the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [IOL fabric ↔ TAP] port bridge; +* the L2-anchor spec §E.2 silence window with the guests shut; * a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; * the classifier spot check on a TAP anchor: ``bpf`` match-drop and the eBPF ``frequency_drop`` every-nth mode (the Docker suite runs the full @@ -81,6 +82,16 @@ def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): return console +def _eth_shutdown(console, shut): + """Shut/unshut the guest's Ethernet0/0 — the only legitimate speaker on + the link segment (the IOL fabric's own chatter) — for §E.2's silence + window; the address stays configured and answers once it is back.""" + console.run("conf t") + console.run("interface Ethernet0/0") + console.run("shutdown" if shut else "no shutdown") + console.run("end") + + def test_iou_kernel_datapath(): server = harness.live_server(kernel=True) compute = server.compute @@ -147,6 +158,19 @@ def test_iou_kernel_datapath(): baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + # L2-anchor spec §E.2: with the guests shut the anchors and the + # per-link bridge stay silent for 5 s (these anchors are the `tap + # create` persistent TAPs, one per Ethernet bay/unit). + if harness.l2only_supported(): + for console in (c1, c2): + _eth_shutdown(console, shut=True) + harness.assert_idle_silence(a1, a2, bridge) + for console in (c1, c2): + _eth_shutdown(console, shut=False) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + else: + print(".. uBridge without link l2only: skipping the §E.2 assertion") + # delay 100: netem on both anchors (each impairs one direction), # one-way ~100 ms => RTT grows by ~200 ms compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) diff --git a/tests/e2e/test_qemu_kernel_datapath.py b/tests/e2e/test_qemu_kernel_datapath.py index 795cce5c6..721895c53 100644 --- a/tests/e2e/test_qemu_kernel_datapath.py +++ b/tests/e2e/test_qemu_kernel_datapath.py @@ -19,6 +19,8 @@ port bridge, the anchor is the NIC's host side. anchors (``gq`` TAPs, one per adapter) exist from node start, before any link — and the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [QEMU netdev ↔ TAP] anchor; +* the L2-anchor spec §E.2 silence window with the guests shut (the anchors + here are QEMU's own netdevs); * a ``delay 100`` filter lands as netem on both anchors, RTT grows ~200 ms; * suspend admin-downs the anchor (the e1000 loses carrier) and kills the traffic; resume restores; @@ -80,6 +82,16 @@ def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): return console +def _eth_shutdown(console, shut): + """Shut/unshut the guest's GigabitEthernet0/0 — the only legitimate + speaker on the link segment (IOSv's own chatter) — for §E.2's silence + window; the address stays configured and answers once it is back.""" + console.run("conf t") + console.run(f"interface {GUEST_ETH_IF}") + console.run("shutdown" if shut else "no shutdown") + console.run("end") + + def test_qemu_kernel_datapath(): server = harness.live_server(kernel=True) compute = server.compute @@ -144,6 +156,18 @@ def test_qemu_kernel_datapath(): baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + # L2-anchor spec §E.2: with the guests shut the anchors (QEMU's own + # netdevs) and the per-link bridge stay silent for 5 s. + if harness.l2only_supported(): + for console in (c1, c2): + _eth_shutdown(console, shut=True) + harness.assert_idle_silence(a1, a2, bridge) + for console in (c1, c2): + _eth_shutdown(console, shut=False) + assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + else: + print(".. uBridge without link l2only: skipping the §E.2 assertion") + # delay 100: netem on both anchors (each impairs one direction), # one-way ~100 ms => RTT grows by ~200 ms compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) From 86651604c4ad9c347012a7f7e00a2dba497feb0e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Fri, 2 Oct 2026 23:17:00 +0800 Subject: [PATCH 067/101] test(e2e): relay capture and markers on both relay engines The relay engines' observable paths had no live coverage: every capture/start call in the suite sat on a kernel anchor (AF_PACKET capture start_kernel) and markers were only asserted on marker add_kernel. The two relay engines' own command shapes were unit-tested only: * bridge engine - bridge start_capture / bridge add_packet_filter ... mark * iol_bridge engine - iol_bridge start_capture / iol_bridge add_packet_filter ... mark Docker relay control (bridge engine, 1 passed 33s): the capture start REST call writes the ICMP exchange to a pcap off the relay bridge; a mark filter attached with capture_node_id n1 signals exactly 10 marker.match events over the dedicated marker WS for 5 pings (5 requests + 5 replies, both directions), carrying node_id/link_id/tag and the busybox frame length, writes exactly those 10 frames to the marker pcap, and deleting the marker takes the pcap with it. IOU relay control (iol_bridge engine, 1 passed 173s): the same two checks on iol_bridge start_capture and the port's mark filter, driven by the real IOS ping through the fabric socket relay. Not duplicated in the ESW/VPCS/cloud relay controls: they issue the same uBridge bridge commands through their own call sites, which stay covered by unit tests; the IOL container relay goes through the same iol_bridge engine already exercised here. --- docs/features/docker-kernel-datapath.md | 7 +-- docs/features/iou-kernel-datapath.md | 3 +- tests/e2e/README.md | 9 ++-- tests/e2e/test_docker_kernel_datapath.py | 63 +++++++++++++++++++++++- tests/e2e/test_iou_kernel_datapath.py | 53 ++++++++++++++++++++ 5 files changed, 126 insertions(+), 9 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index ed1212226..5ac1a68a6 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -466,9 +466,10 @@ silence with the guests' links down), the filter matrix one type at a time (netem core and extensions, cls_bpf match-drop, the eBPF classifier, each asserted through real traffic), suspend, capture, link delete/re-create, container stop/start, a relay negative control — whose own filters -(delay/frequency_drop/bpf) are asserted to really shape the relay wire as -uBridge userspace bridge filters — and the relay→kernel upgrade on project -reopen (the datapath choice is flipped across a server restart). The container image is digest-pinned and pulled through the +(delay/frequency_drop/bpf) really shape the relay wire, whose capture and +markers ride the same relay engine (`bridge start_capture`, the bridge +`mark` filter) — and the relay→kernel upgrade on project reopen (the +datapath choice is flipped across a server restart). The container image is digest-pinned and pulled through the server's own pull route on a cache miss, so collaborators test the same bytes. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 8f1ad117a..65fe4c6ff 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -153,7 +153,8 @@ bridge, netem delay, the TAP-anchor classifier spot check stop/start rewiring with a delay filter restored on the fresh anchor (tc and traffic both), the serial link staying on the relay, and a relay negative control whose real `delay`/`frequency_drop` filters ride the -port's `iol_bridge add_packet_filter` list. +port's `iol_bridge add_packet_filter` list — capture and markers on the +same engine (`iol_bridge start_capture` / the port's `mark` filter). The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the fork), including the DOWN-anchor resilience and kernel-bridge interop this diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 3f87fc827..0ec9a509a 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -83,7 +83,8 @@ Current scenarios: spot check and a filter surviving a node restart, a serial link staying on the relay (serial bays never anchor), and a relay negative control whose filters ride the IOU-specific `iol_bridge add_packet_filter` engine - (delay and frequency_drop measured through real traffic). + (delay and frequency_drop measured through real traffic, capture and + markers on `iol_bridge start_capture` and the port's `mark` filter). * `test_qemu_kernel_datapath.py` — two real IOSv routers (a real qcow2 image from the images directory's `QEMU` subfolder, linked clone, the real IOS CLI on the server's telnet console): the anchor TAP is QEMU's @@ -97,8 +98,10 @@ Current scenarios: (netem core and extensions, cls_bpf match-drop, the eBPF classifier — each measured through real traffic), plus a relay negative control whose own filters (delay, frequency_drop, bpf) really shape the wire as uBridge - userspace bridge filters, and a relay→kernel reopen upgrade (the server's - datapath choice is flipped across a restart). + userspace bridge filters and whose capture and markers ride the relay + engine (`bridge start_capture` / the bridge `mark` filter), and a + relay→kernel reopen upgrade (the server's datapath choice is flipped + across a restart). * `test_marker_kernel_datapath.py` — two real Alpine containers on a kernel-only link with traffic-insight markers attached to a veth anchor (`marker add_kernel`): `marker.match` signals collected off the dedicated diff --git a/tests/e2e/test_docker_kernel_datapath.py b/tests/e2e/test_docker_kernel_datapath.py index ec5276460..04c7d5e50 100644 --- a/tests/e2e/test_docker_kernel_datapath.py +++ b/tests/e2e/test_docker_kernel_datapath.py @@ -39,7 +39,9 @@ server itself uses. The scenarios cover: uBridge relay, the kernel-only filter types are hidden — and the relay's own filter set (delay, frequency_drop, bpf) really shapes the wire, as uBridge userspace filters on the filter node's bridge - (``bridge add_packet_filter``). + (``bridge add_packet_filter``); its capture (``bridge start_capture``) + and its markers (``bridge add_packet_filter … mark``) ride the same + relay engine, not the kernel datapath's AF_PACKET modules. * ``test_docker_relay_to_kernel_reopen_upgrade`` — the server's datapath choice is flipped (relay → kernel) across a restart and the project is reopened: the link is rebuilt on the kernel datapath and forwards. @@ -114,6 +116,10 @@ def _filter_types(compute, pid, lid): return {entry["type"] for entry in compute.call("GET", f"/projects/{pid}/links/{lid}/available_filters")} +def _matches(marker_ws, marker_name): + return [e for e in marker_ws.events("marker.match") if e["filter"] == marker_name] + + def _silence_guest(daemon, container_id): """Silence the guest for the idle-silence measurement — the docker analogue of shutting the IOS interfaces in the dynamips scenario. The @@ -448,7 +454,8 @@ def test_docker_kernel_filter_matrix(): def test_docker_relay_control(): """Negative control: the same topology on an isolated relay-configured instance wires nothing into the kernel — no per-link bridge, no - enslavement — and still pings over the uBridge UDP relay.""" + enslavement — and still pings over the uBridge UDP relay, its filters, + capture and markers all riding the relay engine.""" server = harness.live_server(kernel=False) compute = server.compute image = harness.ensure_docker_image(server) @@ -527,6 +534,58 @@ def test_docker_relay_control(): assert big["loss"] == 100, big["raw"] _put_filters(compute, pid, link["link_id"], {}) assert harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5)["loss"] == 0 + + # capture on the relay: the same REST call as the kernel datapath's, + # hosted by the relay node's uBridge bridge (bridge start_capture) + # instead of AF_PACKET on an anchor + print(".. relay capture: bridge start_capture writes the ICMP exchange") + capture = compute.call( + "POST", f"/projects/{pid}/links/{link['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} + ) + harness.docker_ping(daemon, n1["container_id"], R2_IP) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{link['link_id']}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # markers on the relay are uBridge's `mark` packet filter on the + # relay bridge (bridge add_packet_filter ... mark), signalled over + # the same dedicated marker WS — the relay engine's observability + # path (the kernel datapath's marker add_kernel is the other one) + print(".. relay marker: 5 ICMP echoes through a mark filter on the relay bridge") + marker_ws = harness.WebSocketCollector(server, f"/projects/{pid}/notifications/markers/ws") + try: + m = compute.call( + "POST", + f"/projects/{pid}/links/{link['link_id']}/markers", + {"name": "m-icmp", "bpf": "icmp", "tag": 4242, "capture_node_id": n1_id}, + ) + assert m["capture_node_id"] == n1_id and m["enabled"] is True, m + result = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5) + assert result["loss"] == 0, result["raw"] + harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 10, timeout=10) + icmp = _matches(marker_ws, "m-icmp") + assert len(icmp) == 10, len(icmp) # 5 requests + 5 replies + assert {e["dir"] for e in icmp} == {"tx", "rx"}, [e["dir"] for e in icmp] + for event in icmp: + assert event["node_id"] == n1_id, event + assert event["link_id"] == link["link_id"], event + assert event["tag"] == 4242, event + assert event["len"] == 98, event # busybox ping frame + # the marker's pcap on the host holds exactly the matched frames + markers_dir = os.path.join(compute.call("GET", f"/projects/{pid}")["path"], "project-files", "markers") + pcap = os.path.join(markers_dir, f"{n1_id}_{link['link_id']}_m-icmp.pcap") + assert os.path.exists(pcap), os.listdir(markers_dir) + count, ethertypes = harness.pcap_records(pcap) + assert count == 10, count + assert set(ethertypes) == {"0800"}, ethertypes + # deleting the marker removes its pcap with it + compute.call("DELETE", f"/projects/{pid}/links/{link['link_id']}/markers/m-icmp") + assert harness.wait_until(lambda: not os.path.exists(pcap), timeout=10), pcap + finally: + marker_ws.close() except BaseException: harness.release(server, pid, failed=True) raise diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index b6d3c7d88..23f2f1c18 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -41,8 +41,11 @@ relay-configured instance: the same topology rides the fabric socket ↔ UDP relay (no per-link bridge, nothing enslaved) and still pings — with its filters applied by the IOU-specific engine (``iol_bridge add_packet_filter`` on the IOL port) really shaping the wire. +Its capture and markers ride the same engine +(``iol_bridge start_capture`` / ``iol_bridge add_packet_filter … mark``). """ +import os import re import time @@ -82,6 +85,10 @@ def _boot(server, project_id, node_id, timeout=BOOT_TIMEOUT): return console +def _matches(marker_ws, marker_name): + return [e for e in marker_ws.events("marker.match") if e["filter"] == marker_name] + + def _eth_shutdown(console, shut): """Shut/unshut the guest's Ethernet0/0 — the only legitimate speaker on the link segment (the IOL fabric's own chatter) — for §E.2's silence @@ -382,6 +389,52 @@ def test_iou_relay_control(): recovered = harness.wait_ping(c1, "10.1.1.2") assert recovered["success"] == 100, recovered["raw"] + # capture on the relay: iol_bridge start_capture on the port's IOL + # location (the IOU engine's own capture path, distinct from the + # kernel datapath's capture start_kernel on the anchor) + print(".. iol_bridge relay capture: start_capture writes the ICMP exchange") + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.ping(c1, "10.1.1.2", repeat=5) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + path = capture["capture_file_path"] + count, ethertypes = harness.pcap_records(path) + assert count >= 4, f"{path}: {count} records" + assert "0800" in ethertypes, (path, ethertypes) + + # markers on the relay: iol_bridge add_packet_filter ... mark on the + # port's IOL location, signalled over the dedicated marker WS + print(".. iol_bridge relay marker: 5 ICMP echoes through a mark filter") + marker_ws = harness.WebSocketCollector(server, f"/projects/{pid}/notifications/markers/ws") + try: + m = compute.call( + "POST", + f"/projects/{pid}/links/{lid}/markers", + {"name": "m-icmp", "bpf": "icmp", "tag": 4242, "capture_node_id": n1_id}, + ) + assert m["capture_node_id"] == n1_id and m["enabled"] is True, m + result = harness.ping(c1, "10.1.1.2", repeat=5) + assert result["success"] == 100, result["raw"] + harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 10, timeout=10) + icmp = _matches(marker_ws, "m-icmp") + assert len(icmp) == 10, len(icmp) # 5 requests + 5 replies + assert {e["dir"] for e in icmp} == {"tx", "rx"}, [e["dir"] for e in icmp] + for event in icmp: + assert event["node_id"] == n1_id, event + assert event["link_id"] == lid, event + assert event["tag"] == 4242, event + markers_dir = os.path.join(compute.call("GET", f"/projects/{pid}")["path"], "project-files", "markers") + pcap = os.path.join(markers_dir, f"{n1_id}_{lid}_m-icmp.pcap") + assert os.path.exists(pcap), os.listdir(markers_dir) + count, ethertypes = harness.pcap_records(pcap) + assert count == 10, count + assert set(ethertypes) == {"0800"}, ethertypes + # deleting the marker removes its pcap with it + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-icmp") + assert harness.wait_until(lambda: not os.path.exists(pcap), timeout=10), pcap + finally: + marker_ws.close() + c1.close() c2.close() except BaseException: From f03c94db6da7835ef93c78839dc6c29080748695 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 00:55:35 +0800 Subject: [PATCH 068/101] =?UTF-8?q?docs:=20link-local=20frame=20transparen?= =?UTF-8?q?cy=20on=20kernel=20links=20=E2=80=94=20spec=20and=20per-datapat?= =?UTF-8?q?h=20sections?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Linux bridge standing in for a cable does not forward the IEEE 802.1D reserved range (01:80:c2:00:00:00-0f) by default, so LACP/LLDP/802.1X never crossed a kernel-datapath link while the UDP relay carried them fine. uBridge now opens every enslave'd port's IFLA_BRPORT_GROUP_FWD_MASK to 0xfffd at brctl addif time (branch feature/bridge-tap-l2only, commits 0284500/03a18f1/ cf4e494), which the six kernel-datapath feature docs now document as a section: STP, LACP, 802.1X and LLDP/DCBX cross; 802.3x PAUSE / PFC stays dropped — br_handle_frame()'s case 0x01 discards it unconditionally, no mask can enable it, and PFC labs must use a relay link. The full contract, kernel references and requirements live in the new design spec, marked landed. --- ...bridge-link-local-frame-forwarding-spec.md | 266 ++++++++++++++++++ docs/features/docker-kernel-datapath.md | 21 ++ docs/features/dynamips-kernel-datapath.md | 21 ++ .../ethernet-switch-kernel-datapath.md | 26 ++ docs/features/iol-docker-kernel-datapath.md | 21 ++ docs/features/iou-kernel-datapath.md | 21 ++ docs/features/qemu-kernel-datapath.md | 21 ++ 7 files changed, 397 insertions(+) create mode 100644 docs/design/ubridge-link-local-frame-forwarding-spec.md diff --git a/docs/design/ubridge-link-local-frame-forwarding-spec.md b/docs/design/ubridge-link-local-frame-forwarding-spec.md new file mode 100644 index 000000000..427863e4c --- /dev/null +++ b/docs/design/ubridge-link-local-frame-forwarding-spec.md @@ -0,0 +1,266 @@ +# uBridge bridge ports must forward link-local frames (per-port `group_fwd_mask`) + +> **Status: landed and verified.** uBridge branch `feature/bridge-tap-l2only`, +> commits `0284500` (feat), `03a18f1` (test suite), `cf4e494` (docs) — the +> implicit `addif` hook of §A.1 plus the `setportgroupfwd` escape hatch of +> §A.3. gns3-server carries **zero product-code changes**: the behavior +> reaches every `brctl addif` call site as specified. Verified by the uBridge +> suite (`tests/brctl/test_linklocal.py`, 28/28) and live through a real +> server by `tests/e2e/test_docker_link_local_frames.py`. The three +> `test_vlan` kernel-state failures seen alongside are pre-existing on the +> parent commit (verified by an fb72758 control run). The version bump (R8) +> is still open, pending review. + +## 0. Background & scope + +### The datapath this serves + +GNS3's kernel datapath wires a link by enslaving each endpoint's host-side +anchor (veth host end, TAP) into a per-link Linux bridge. The same shape is +used by the Ethernet switch node (its bridge absorbs peers' anchors), by +switch-to-switch cascades (a veth pair joining two bridges) and by the Cloud +node (pnet bridges). Functionally these bridges stand in for **a cable** — a +transparent L2 segment — and that is the contract users expect from them. + +### The problem: a Linux bridge is not a transparent cable + +`br_handle_frame()` in `net/bridge/br_input.c` treats the IEEE 802.1D +reserved link-local range `01:80:c2:00:00:00`–`01:80:c2:00:00:0f` specially +and, by default, does **not** forward it. Measured on a stock mainline kernel +(7.2.6, no patches) with two ports in one bridge: + +| Destination MAC | Protocol | Default bridge | Notes | +|---|---|---|---| +| `ff:ff:ff:ff:ff:ff`, `01:00:5e:*`, `33:33:*` | broadcast, IPv4/IPv6 multicast | forwarded | unaffected | +| `01:80:c2:00:00:00` | STP / RSTP BPDU | **forwarded** | special-cased: forwarded while the bridge runs no STP | +| `01:80:c2:00:00:01` | 802.3x PAUSE / 802.1Qbb PFC | not forwarded | kernel drops unconditionally — see §0.3 | +| `01:80:c2:00:00:02` | LACP / slow protocols | not forwarded | | +| `01:80:c2:00:00:03` | 802.1X EAPOL | not forwarded | | +| `01:80:c2:00:00:0e` | LLDP (DCBX) | not forwarded | | +| `01:80:c2:00:00:10` and up | — | forwarded | outside the reserved nibble range | + +Consequence: on a kernel-datapath link, two emulated devices that run LACP, +LLDP/DCBX or 802.1X never see each other, while the same lab over the uBridge +UDP relay (a dumb store-and-forward, fully transparent) works. The two +datapaths disagree, and the failure is silent. + +### Why the bridge-level knob is not enough + +`IFLA_BR_GROUP_FWD_MASK` (the bridge-level attribute, already reachable via +uBridge's `brctl setgroupfwd`) is restricted by `BR_GROUPFWD_RESTRICTED` +(`BR_GROUPFWD_STP | BR_GROUPFWD_MACPAUSE | BR_GROUPFWD_LACP`): writes +containing bits 0/1/2 fail with `EINVAL`. Measured: bridge-level accepts at +most `0xfff8`, which forwards LLDP/EAPOL but **not LACP**. + +The per-port attribute `IFLA_BRPORT_GROUP_FWD_MASK` (commit +`5af48b59f35cf712793badabe1a574a0d0ce3bd3`, "net: bridge: add per-port +group_fwd_mask with less restrictions", net-next 2017, Linux 4.15) exists +precisely for this: its commit message names "transparent forwarding of +most link-local frames (e.g. STP, LACP) through tunnels (vxlan, qinq)" and +restricts only MAC PAUSE. Measured: port-level accepts `0xfffd` and then +forwards STP, LACP, EAPOL, LLDP and every other reserved address. + +The mask is evaluated on the **ingress** port (setting it on the egress port +only has no effect), so every port of a bridge needs it — which is exactly +what a per-addif hook gives us. + +### What this change is — and what it cannot be + +PAUSE / PFC is **not** in scope and cannot be delivered by any mask: the +`case 0x01:` branch in `br_handle_frame()` drops unconditionally +(`kfree_skb_reason(..., SKB_DROP_REASON_MAC_IEEE_MAC_CONTROL)`) without +consulting `fwd_mask`, and the port-level setter rejects bit 1 with `EINVAL`. +Forwarding PAUSE would require patching that branch. Emulation of PFC +semantics is out of reach anyway (no hardware pause generation/response, no +per-priority queues, no headroom/watchdog), so the GNS3 documentation will +state PFC as unsupported on kernel links rather than pretending otherwise. + +### Goal + +A kernel-datapath link (and every other uBridge bridge) carries link-local +frames exactly like the cable it emulates: STP, LACP, EAPOL, LLDP and the +remaining reserved addresses flow; PAUSE/PFC is documented as impossible. + +--- + +## A. The requested change + +### A.1 One step in `br_enslave_if()` (`hypervisor_brctl.c:251`) + +After the port is enslaved (step 1) — and using the existing u16 port-attribute +helper — set the link-local forwarding mask: + +```c +/* Step 3 – link-local transparency. A Linux bridge stands in for a cable + * here: LACP/LLDP/EAPOL/STP must cross it. The kernel restricts the + * *bridge-level* mask to bits 3..15 (BR_GROUPFWD_RESTRICTED), so LACP is + * only reachable per port. 0xfffd = every reserved address except MAC + * PAUSE (bit 1), which the kernel refuses and hard-drops anyway. + * Best-effort: kernels before 4.15 have no such port attribute, and the + * cable simply keeps today's behaviour there. */ +if (br_set_port_attr_u16(bridge, port, IFLA_BRPORT_GROUP_FWD_MASK, + LINK_LOCAL_FWD_MASK) < 0) + fprintf(stderr, "ubridge: %s: link-local forwarding mask not set\n", port); +``` + +with + +```c +/* bit 0 STP (harmless: already forwarded while STP is off), bit 2 LACP, + * bit 3 802.1X EAPOL, bit 14 LLDP; bit 1 MAC PAUSE must stay clear or the + * kernel rejects the whole value with EINVAL. */ +#define LINK_LOCAL_FWD_MASK 0xfffdu +``` + +`br_set_port_attr_u16()` (`hypervisor_brctl.c:811`) already does everything +needed: it checks membership via `br_check_master()`, builds the +`IFLA_PROTINFO` nested attribute on `AF_BRIDGE`/`RTM_SETLINK`, and — critically +— encodes the value as **2 bytes**, which is what the kernel's port policy +requires (`NLA_U16`); a 1-byte payload is rejected with `-ERANGE` (the same +trap documented in that helper's comment for `IFLA_BRPORT_PRIORITY`). + +### A.2 Why in `br_enslave_if()` rather than in any single caller + +* **Every** enslavement in uBridge funnels through this helper — per-link + bridges, the Ethernet switch's own ports, peers absorbed into a switch + bridge, cascades, pnet/Cloud bridges. One hook covers all of them, and + covers future callers too. +* gns3-server **cannot** do it itself: the packaged service runs unprivileged + (`init/gns3.service.systemd`: `User=gns3`) and `CAP_NET_ADMIN` is carried by + the uBridge binary (`base_manager.py:_has_privileged_access()`). A + server-side `ip link set … type bridge_slave …` is not possible in the + supported deployment. +* Doing it at addif time also means ports that are re-attached after a node + restart (the server re-pushes NIOs) are re-covered without extra logic. + +### A.3 Alternative interface, if the project prefers explicit policy + +If uBridge would rather keep this policy out of `addif`, the equivalent +explicit command is: + +``` +brctl setportgroupfwd (3 args) +``` + +implemented as a thin wrapper over `br_set_port_attr_u16(..., (unsigned +short)mask)` and registered next to `setportprio` in the command table +(`hypervisor_brctl.c:2049`). In that case gns3-server will call it at its +five `brctl addif` sites and gate it on a probe of the running build. The +implicit form is preferred by gns3-server because it needs no server change +and cannot be forgotten at a new call site. + +--- + +## B. Requirements (normative) + +* **R1** After a successful `brctl addif `, the port carries + `group_fwd_mask = 0xfffd` (`LINK_LOCAL_FWD_MASK`). +* **R2** The value must **exclude bit 1 (MAC PAUSE)**. `0xfffe`/`0xffff` are + rejected by the kernel with `EINVAL` and would silently lose the whole + mask — do not use them. +* **R3** The attribute payload must be encoded as u16 (`br_set_port_attr_u16`). + A u8 payload fails `validate_nla` with `-ERANGE` on kernels that enforce the + policy strictly. +* **R4** **Best-effort**: a failed mask write must never fail the `addif` + (kernels < 4.15 have no per-port attribute; unusual deployments may lack + `CAP_NET_ADMIN`). Reply to `addif` is unchanged; the failure is logged to + stderr. +* **R5** Ordering: set the mask after the port is enslaved (both + `br_check_master()` and the kernel's `IFLA_PROTINFO` handling require + membership). The step-2 "bring the port up" may run before or after; both + work. +* **R6** Idempotent: re-`addif` of an already-enslaved port must succeed and + keep the mask set (gns3-server re-attaches links on node start). +* **R7** Applies uniformly to every bridge/port this helper touches — no + per-bridge or per-caller switch (a real switch terminates link-local frames, + but GNS3's Ethernet switch node is used as a transparent segment, so + forwarding is the wanted behaviour there too). +* **R8** Version: bump the reported version so gns3-server can raise its + minimum (the server already gates on `ubridge version X.Y.Z`, + `compute/ubridge/hypervisor.py:_check_ubridge_version`, currently ≥ 1.2.3). +* **R9** Document the PAUSE/PFC exclusion in uBridge's help/docs, so the gap + is not rediscovered as a bug. + +--- + +## C. Implementation notes + +* `IFLA_BRPORT_GROUP_FWD_MASK` is defined in `include/uapi/linux/if_link.h` + (Linux ≥ 4.15); guard the build on its presence if uBridge must compile + against older headers — behavior on such kernels is "no mask", which R4 + already tolerates. +* The kernel's `br_set_group_fwd_mask()` (port variant) validates against + `BR_GROUPFWD_MACPAUSE`; there is also a follow-up fix + (`16c42db48522`, "bridge: check for attempt to forward STP PDU's with STP + enabled") that moves STP validation into the setter — bit 0 is harmless for + our case because we never enable STP on these bridges. +* No interaction with existing commands: `brctl setgroupfwd` (bridge-level) + stays as is and is unused by gns3-server; `hairpin`/`isolated`/`setportstate` + continue to use their own attributes through the same helper family. + +--- + +## D. Reply contract & error behavior + +`addif` replies (`HSC_INFO_OK` / `HSC_ERR_*`) are unchanged. The mask write +does not produce its own reply and does not alter the addif verdict; on +failure it writes one line to stderr (§A.1). No new error codes. + +--- + +## E. Test requirements + +The uBridge test suite already drives bridge/veth setups; the following cases +pin the contract (all are cheap AF_PACKET probes — send a frame with a given +destination MAC into port A, observe whether it arrives on port B, using +`PACKET_OUTGOING` filtering so a port's own transmissions do not count): + +* **T1 transparency**: with two veths in one bridge created via `brctl`: + `01:00:5e:00:00:01` forwarded, `01:80:c2:00:00:0e` (LLDP) forwarded, + `01:80:c2:00:00:02` (LACP) forwarded, `01:80:c2:00:00:00` (STP) forwarded, + `01:80:c2:00:00:01` (PAUSE) **not** forwarded. +* **T2 ingress semantics**: mask on the egress port only → LACP not forwarded + (documents why every port is set; regression guard against "set it once"). +* **T3 best-effort**: force the mask write to fail (fault-injection hook or an + invalid attribute id) and assert `addif` still replies `HSC_INFO_OK`. +* **T4 idempotency**: `addif` twice → both succeed, mask still `0xfffd`. +* **T5 value boundary** (optional, documents R2): a direct write of `0xfffe` + through the port setter fails with `EINVAL`. + +--- + +## gns3-server alignment (informational — not uBridge scope) + +* **No server change is required** for the behavior: all five server-side + enslavements (`compute/kernel_datapath.py:165`, + `compute/builtin/nodes/ethernet_switch.py:399,518`, + `compute/builtin/nodes/cloud.py:377`, and anything added later) go through + `brctl addif`. +* If R8's version bump lands, gns3-server can raise + `_check_ubridge_version`'s minimum so that a compute that cannot deliver + the behavior is reported instead of silently degrading. +* GNS3 documentation will state the resulting matrix for kernel links: + LLDP/DCBX ✅, LACP ✅, 802.1X ✅, STP ✅, **PAUSE/PFC ❌** (kernel-level, + not fixable by mask). +* The planned VXLAN datapath inherits the same story: the vxlan device is a + bridge port, so the same per-port mask applies; PAUSE/PFC stays out of + reach there too. That must be decided before VXLAN ships, because today's + cross-compute relay links *do* carry PFC and would silently regress. + +## Roadmap note + +If PFC emulation ever becomes a hard requirement, the only paths are a +patched kernel (`case 0x01`), a tc/eBPF ingress redirect on the anchor, or a +userspace AF_PACKET pass-through in uBridge (packet sockets see these frames +before the bridge drops them). None of these is part of this spec. + +## References + +* Kernel: `net/bridge/br_input.c` (`br_handle_frame`, the link-local switch), + `net/bridge/br_private.h` (`BR_GROUPFWD_*`, `BR_GROUPFWD_RESTRICTED`). +* Per-port mask: commit + [`5af48b59f35c`](https://github.com/Freescale/linux-fslc/commit/5af48b59f35cf712793badabe1a574a0d0ce3bd3) + ("net: bridge: add per-port group_fwd_mask with less restrictions"); + [patchwork entry](https://patchwork.ozlabs.org/project/netdev/patch/1506517964-17479-1-git-send-email-nikolay@cumulusnetworks.com/). +* Measurements: stock mainline 7.2.6-1-default, two veth ports in one Linux + bridge, AF_PACKET probes (2026-10). diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 5ac1a68a6..4134a4c39 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -333,6 +333,27 @@ Semantics: nothing — older servers, failed probe — keep the full list; apply-time validation (409) stays the guard. +## Link-local frames (LACP, LLDP, 802.1X, STP) + +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does +not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by +default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while +the UDP relay carries them fine. uBridge therefore opens every port's +per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at +`brctl addif` time (Linux 4.15+, best-effort on older kernels). The +bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, +so LACP is reachable per port only. + +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and +LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in +`br_handle_frame()` drops it unconditionally and no mask can enable it. +That is a documented limit, not a regression (PFC's hardware semantics are +out of emulation's reach anyway); a lab that needs PAUSE frames on the wire +must use a relay link. Verified live by +`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest +matrix); the contract and kernel references live in +`docs/design/ubridge-link-local-frame-forwarding-spec.md`. + ## uBridge command surface No uBridge changes were needed beyond the marker module's `*_kernel` commands diff --git a/docs/features/dynamips-kernel-datapath.md b/docs/features/dynamips-kernel-datapath.md index 4d731c7b9..baed487c7 100644 --- a/docs/features/dynamips-kernel-datapath.md +++ b/docs/features/dynamips-kernel-datapath.md @@ -93,6 +93,27 @@ filters are one tc netem qdisc per anchor plus cls_bpf/eBPF drops. A relay port keeps its Dynamips NIO and the node's uBridge relay bridge (`DYNAMIPS-*`) with userspace filters, exactly as before. +## Link-local frames (LACP, LLDP, 802.1X, STP) + +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does +not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by +default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while +the UDP relay carries them fine. uBridge therefore opens every port's +per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at +`brctl addif` time (Linux 4.15+, best-effort on older kernels). The +bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, +so LACP is reachable per port only. + +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and +LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in +`br_handle_frame()` drops it unconditionally and no mask can enable it. +That is a documented limit, not a regression (PFC's hardware semantics are +out of emulation's reach anyway); a lab that needs PAUSE frames on the wire +must use a relay link. Verified live by +`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest +matrix); the contract and kernel references live in +`docs/design/ubridge-link-local-frame-forwarding-spec.md`. + ## Relay fallback A uBridge without the tap module keeps the node relay-only: `_prepare_tap_datapath` diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index c57e406c2..322e06d43 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -118,6 +118,32 @@ sequence by hand against uBridge (held TAPs, no server involved) did not reproduce it, and the in-place reconcile removes the operation entirely — the failure mode no longer exists by construction. +## Link-local frames (LACP, LLDP, 802.1X, STP) + +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does +not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by +default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while +the UDP relay carries them fine. uBridge therefore opens every port's +per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at +`brctl addif` time (Linux 4.15+, best-effort on older kernels). The +bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, +so LACP is reachable per port only. + +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and +LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in +`br_handle_frame()` drops it unconditionally and no mask can enable it. +That is a documented limit, not a regression (PFC's hardware semantics are +out of emulation's reach anyway); a lab that needs PAUSE frames on the wire +must use a relay link. Verified live by +`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest +matrix); the contract and kernel references live in +`docs/design/ubridge-link-local-frame-forwarding-spec.md`. + +The switch node's own bridge ports carry the same mask: the builtin +Ethernet switch is a transparent L2 segment in GNS3, not a protocol +endpoint, so LACP/LLDP between two peers crosses it (a physical switch +would terminate them itself). + ## Verified Unit level (all under `tests/`): the switch's kernel-port lifecycle (join diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index cbbbcf415..5b7b29aab 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -122,6 +122,27 @@ traffic stops both ways. The runner itself is unaware of the link state (netiomux has no carrier signal), exactly as on the relay datapath where suspend rode the synthetic frequency_drop filter instead. +## Link-local frames (LACP, LLDP, 802.1X, STP) + +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does +not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by +default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while +the UDP relay carries them fine. uBridge therefore opens every port's +per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at +`brctl addif` time (Linux 4.15+, best-effort on older kernels). The +bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, +so LACP is reachable per port only. + +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and +LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in +`br_handle_frame()` drops it unconditionally and no mask can enable it. +That is a documented limit, not a regression (PFC's hardware semantics are +out of emulation's reach anyway); a lab that needs PAUSE frames on the wire +must use a relay link. Verified live by +`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest +matrix); the contract and kernel references live in +`docs/design/ubridge-link-local-frame-forwarding-spec.md`. + ## Relay fallback Missing either capability, the node runs relay-only: links ride diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 65fe4c6ff..eea88f783 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -101,6 +101,27 @@ one tc netem qdisc per anchor plus cls_bpf/eBPF drops — the port's userspace `iol_bridge add_packet_filter` filters carry relay links only and are never combined with tc on the same port. +## Link-local frames (LACP, LLDP, 802.1X, STP) + +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does +not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by +default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while +the UDP relay carries them fine. uBridge therefore opens every port's +per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at +`brctl addif` time (Linux 4.15+, best-effort on older kernels). The +bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, +so LACP is reachable per port only. + +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and +LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in +`br_handle_frame()` drops it unconditionally and no mask can enable it. +That is a documented limit, not a regression (PFC's hardware semantics are +out of emulation's reach anyway); a lab that needs PAUSE frames on the wire +must use a relay link. Verified live by +`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest +matrix); the contract and kernel references live in +`docs/design/ubridge-link-local-frame-forwarding-spec.md`. + ## Relay fallback A uBridge without `iol_bridge add_nio_tap` (old build) keeps the node on the diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 18a35843e..29f340a66 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -110,6 +110,27 @@ Identical to Docker's kernel links, on the TAP: `capture start_kernel ` direction is impaired once. `frequency_drop`/`quota`/`window_drop` run as the eBPF classifier on the anchor's clsact egress. +## Link-local frames (LACP, LLDP, 802.1X, STP) + +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does +not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by +default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while +the UDP relay carries them fine. uBridge therefore opens every port's +per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at +`brctl addif` time (Linux 4.15+, best-effort on older kernels). The +bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, +so LACP is reachable per port only. + +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and +LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in +`br_handle_frame()` drops it unconditionally and no mask can enable it. +That is a documented limit, not a regression (PFC's hardware semantics are +out of emulation's reach anyway); a lab that needs PAUSE frames on the wire +must use a relay link. Verified live by +`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest +matrix); the contract and kernel references live in +`docs/design/ubridge-link-local-frame-forwarding-spec.md`. + ## Legacy fallback A uBridge build without the `tap` module (or without `CAP_NET_ADMIN`) keeps From df79773bc0b0c05147fa9012841796cc0bebb12e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 00:55:35 +0800 Subject: [PATCH 069/101] test(e2e): live link-local matrix on the kernel datapath Two Alpine containers on a kernel link, a static raw-Ethernet probe built at run time and docker-cp'd in (the image has no raw-frame tool; DockerDaemon gains the cp), injecting per-destination-MAC frames from guest 1 while guest 2 counts arrivals: ordinary multicast, STP, LACP, 802.1X and LLDP must cross, PAUSE must not (the documented kernel limit). Asserts the anchors' sysfs brport group_fwd_mask reads 0xfffd through the server's own wiring (int-parsed: newer kernels print it hex), that the bridge level stays 0, and that a container stop/start replays the mask with LACP surviving. Negative control is the uBridge suite's setportgroupfwd 0 case. --- tests/e2e/harness.py | 19 ++ tests/e2e/test_docker_link_local_frames.py | 366 +++++++++++++++++++++ 2 files changed, 385 insertions(+) create mode 100644 tests/e2e/test_docker_link_local_frames.py diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index bff5397c9..a7f821565 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -890,6 +890,25 @@ class DockerDaemon: except (AssertionError, ValueError): return "unknown" + def cp(self, container_id, tar_bytes, path): + """Upload a tar archive into the container at *path* (``docker cp`` + in); the daemon extracts it, honoring the entries' modes.""" + conn = _UnixHTTPConnection(self.socket_path, self.timeout) + try: + conn.request( + "PUT", + f"/containers/{container_id}/archive?path={urllib.parse.quote(path)}", + body=tar_bytes, + headers={"Content-Type": "application/x-tar"}, + ) + response = conn.getresponse() + payload = response.read() + assert response.status == 200, f"docker cp into {container_id[:12]} -> HTTP {response.status}: {payload[:300]!r}" + except (OSError, http.client.HTTPException) as e: + raise AssertionError(f"docker cp into {container_id[:12]} failed: {e}") from None + finally: + conn.close() + def ensure_docker_image(server): """The Docker image the scenarios run, ensured present locally. diff --git a/tests/e2e/test_docker_link_local_frames.py b/tests/e2e/test_docker_link_local_frames.py new file mode 100644 index 000000000..bd4b2a394 --- /dev/null +++ b/tests/e2e/test_docker_link_local_frames.py @@ -0,0 +1,366 @@ +# +# Copyright (C) 2026 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Live end-to-end test for link-local frame transparency on kernel-datapath +links (pytest marker ``e2e``): a per-link Linux bridge stands in for a cable, +so the IEEE 802.1D reserved range must cross it exactly like the uBridge UDP +relay always carried it. uBridge opens every enslave'd port's +``IFLA_BRPORT_GROUP_FWD_MASK`` to ``0xfffd`` (ubridge commit 0284500; the +contract lives in ``docs/design/ubridge-link-local-frame-forwarding-spec.md``), +which makes STP, LACP, 802.1X and LLDP/DCBX flow between the guests while the +kernel keeps hard-dropping MAC PAUSE. + +``test_docker_link_local_frames`` — two real Alpine containers on a kernel +link; a static raw-Ethernet probe (built at run time, docker-cp'd in — the +image has no raw-frame tool) injects frames from guest 1's eth0 while guest +2's eth0 counts what actually arrived, per destination MAC. The scenario +asserts, through the server's own wiring path: + +* the per-link bridge carries both anchors and each anchor's sysfs + ``brport/group_fwd_mask`` reads 65533 (0xfffd) — the uBridge default + applied by ``brctl addif``, which is the only place gns3-server wires ports; +* the bridge-level ``group_fwd_mask`` stays 0 (the fix is per-port only); +* the live matrix from the guest: ordinary multicast crosses (wire sanity), + STP / LACP / EAPOL / LLDP cross, and PAUSE does not — ``br_handle_frame``'s + ``case 0x01`` drops it unconditionally, so PFC stays unsupported by design; +* a container stop/start replays the NIO wiring and the mask is re-applied + (the reconnect story), with a fresh LACP round surviving it. + +The negative control for the old behaviour is the uBridge suite's +``setportgroupfwd 0`` case (stock kernel semantics), not a second server run. +""" + +import functools +import io +import os +import subprocess +import tarfile +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from tests.e2e import harness + +pytestmark = pytest.mark.e2e + +# every standard Docker adapter has a single port +ETH = (0, 0) +R1_IP = "10.2.1.1" +R2_IP = "10.2.1.2" + +# (destination MAC, ethertype) — what the guest protocol would emit +CONTROL = ("01005e000001", "0800") # ordinary multicast: must always cross +STP = ("0180c2000000", "0026") # STP BPDU +LACP = ("0180c2000002", "8809") # LACP / slow protocols +EAPOL = ("0180c2000003", "888e") # 802.1X EAPOL +LLDP = ("0180c200000e", "88cc") # LLDP / DCBX +PAUSE = ("0180c2000001", "8808") # 802.3x PAUSE — kernel hard-drop + +MUST_CROSS = [CONTROL, STP, LACP, EAPOL, LLDP] +MUST_NOT_CROSS = [PAUSE] + +RECV_WINDOW = 12 + +_LLPROBE_C = r""" +/* + * llprobe — raw Ethernet send/recv with explicit destination MACs, for the + * link-local forwarding e2e. Built static by the test, docker-cp'd into the + * containers (the Alpine image has no raw-frame tool of its own). + * + * llprobe send + * llprobe recv ... + * + * recv prints one "=" line per requested MAC, counting only + * inbound frames (PACKET_OUTGOING excluded — a veth host end sees its own + * transmissions too). + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_MACS 8 + +static int parse_mac(const char *hex, unsigned char *out) +{ + char buf[13]; + int v[6]; + size_t j = 0; + for (size_t i = 0; hex[i] && j < 12; i++) + if (hex[i] != ':' && hex[i] != '-') + buf[j++] = hex[i]; + buf[j] = '\0'; + if (j != 12 || sscanf(buf, "%2x%2x%2x%2x%2x%2x", &v[0], &v[1], &v[2], &v[3], &v[4], &v[5]) != 6) + return -1; + for (int i = 0; i < 6; i++) + out[i] = (unsigned char)v[i]; + return 0; +} + +static int raw_socket(const char *ifname, struct ifreq *hwaddr) +{ + int sck = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); + if (sck < 0) { + perror("llprobe: socket"); + return -1; + } + if (hwaddr) { + memset(hwaddr, 0, sizeof(*hwaddr)); + strncpy(hwaddr->ifr_name, ifname, IFNAMSIZ - 1); + if (ioctl(sck, SIOCGIFHWADDR, hwaddr) < 0) { + perror("llprobe: SIOCGIFHWADDR"); + close(sck); + return -1; + } + } + struct sockaddr_ll sa; + memset(&sa, 0, sizeof(sa)); + sa.sll_family = AF_PACKET; + sa.sll_protocol = htons(ETH_P_ALL); + sa.sll_ifindex = if_nametoindex(ifname); + if (sa.sll_ifindex == 0) { + fprintf(stderr, "llprobe: no interface %s\n", ifname); + close(sck); + return -1; + } + if (bind(sck, (struct sockaddr *)&sa, sizeof(sa)) < 0) { + perror("llprobe: bind"); + close(sck); + return -1; + } + return sck; +} + +static int do_send(const char *ifname, const char *dst_hex, const char *et_hex, int count) +{ + unsigned char dst[6], frame[64]; + unsigned int et = 0; + struct ifreq ifr; + int sck = raw_socket(ifname, &ifr); + if (sck < 0) + return 1; + if (parse_mac(dst_hex, dst) < 0 || sscanf(et_hex, "%x", &et) != 1) { + fprintf(stderr, "llprobe: bad mac %s or ethertype %s\n", dst_hex, et_hex); + close(sck); + return 1; + } + for (int n = 0; n < count; n++) { + memcpy(frame, dst, 6); + memcpy(frame + 6, ifr.ifr_hwaddr.sa_data, 6); + frame[12] = (et >> 8) & 0xff; + frame[13] = et & 0xff; + frame[14] = 'L'; + frame[15] = 'L'; + frame[16] = (unsigned char)n; + memset(frame + 17, 0, sizeof(frame) - 17); + if (send(sck, frame, sizeof(frame), 0) < 0) { + perror("llprobe: send"); + close(sck); + return 1; + } + usleep(20000); + } + close(sck); + printf("sent %d frames to %s on %s\n", count, dst_hex, ifname); + return 0; +} + +static int do_recv(const char *ifname, int window, char **macs, int nmacs) +{ + unsigned char want[MAX_MACS][6]; + long counts[MAX_MACS] = {0}; + unsigned char buf[2048]; + int sck = raw_socket(ifname, NULL); + if (sck < 0) + return 1; + for (int i = 0; i < nmacs; i++) + if (parse_mac(macs[i], want[i]) < 0) { + fprintf(stderr, "llprobe: bad mac %s\n", macs[i]); + close(sck); + return 1; + } + struct timeval tv = {0, 500000}; + setsockopt(sck, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + long deadline = time(NULL) + window; + for (;;) { + if (time(NULL) >= deadline) + break; + struct sockaddr_ll from; + socklen_t flen = sizeof(from); + ssize_t n = recvfrom(sck, buf, sizeof(buf), 0, (struct sockaddr *)&from, &flen); + if (n < 14) + continue; /* timeout, error or runt */ + if (from.sll_pkttype == PACKET_OUTGOING) + continue; + for (int i = 0; i < nmacs; i++) + if (memcmp(buf, want[i], 6) == 0) + counts[i]++; + } + close(sck); + for (int i = 0; i < nmacs; i++) + printf("%s=%ld\n", macs[i], counts[i]); + return 0; +} + +int main(int argc, char **argv) +{ + if (argc >= 5 && strcmp(argv[1], "send") == 0) + return do_send(argv[2], argv[3], argv[4], atoi(argv[5])); + if (argc >= 5 && strcmp(argv[1], "recv") == 0) + return do_recv(argv[2], atoi(argv[3]), argv + 4, argc - 4); + fprintf(stderr, "usage: llprobe send \n" + " llprobe recv ...\n"); + return 2; +} +""" + + +@functools.cache +def _llprobe(): + """Build the static raw-Ethernet probe (cached for the session).""" + directory = tempfile.mkdtemp(prefix="gns3-e2e-llprobe-") + source = os.path.join(directory, "llprobe.c") + binary = os.path.join(directory, "llprobe") + with open(source, "w") as f: + f.write(_LLPROBE_C) + try: + subprocess.run(["cc", "-static", "-O2", "-o", binary, source], check=True, capture_output=True) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + pytest.skip(f"cannot build the raw-frame probe (cc -static): {e}") + return binary + + +def _upload_probe(daemon, *container_ids): + """docker-cp the probe into every container's /tmp.""" + with open(_llprobe(), "rb") as f: + payload = f.read() + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as tar: + info = tarfile.TarInfo("llprobe") + info.size = len(payload) + info.mode = 0o755 + tar.addfile(info, io.BytesIO(payload)) + archive = buffer.getvalue() + for container_id in container_ids: + daemon.cp(container_id, archive, "/tmp") + + +def _recv_counts(daemon, container_id, window, macs): + """Run the receiver to completion and parse its "=" lines.""" + code, output = daemon.exec( + container_id, ["/tmp/llprobe", "recv", "eth0", str(window), *macs], timeout=window + 30 + ) + assert code == 0, output + return {line.split("=")[0]: int(line.split("=")[1]) for line in output.strip().splitlines() if "=" in line} + + +def _send_all(daemon, container_id, matrix, count=5): + for mac, ethertype in matrix: + code, output = daemon.exec(container_id, ["/tmp/llprobe", "send", "eth0", mac, ethertype, str(count)]) + assert code == 0, output + + +def _port_mask(anchor): + """The anchor's link-local forwarding mask, as an int — newer kernels + print the sysfs attribute in hex (``0xfffd``), older ones in decimal.""" + return int(harness.sysfs(f"/sys/class/net/{anchor}/brport/group_fwd_mask"), 0) + + +def test_docker_link_local_frames(): + server = harness.live_server(kernel=True) + compute = server.compute + image = harness.ensure_docker_image(server) + + project = compute.create_project("docker-ll") + pid = project["project_id"] + try: + nodes = [] + for suffix, address in (("1", R1_IP), ("2", R2_IP)): + node = compute.create_docker_node(pid, f"LL-{suffix}", image) + harness.configure_docker_interfaces(node, address) + nodes.append(node) + n1, n2 = nodes + daemon = harness.DockerDaemon() + for node in nodes: + node["container_id"] = daemon.container_id(f"GNS3.{node['name']}.{pid}") + assert node["container_id"], f"container for {node['name']} not found" + c1, c2 = n1["container_id"], n2["container_id"] + + link = compute.create_link(pid, (n1["node_id"], *ETH), (n2["node_id"], *ETH)) + lid = link["link_id"] + assert link["kernel_datapath"] is True, link + + print(".. starting containers") + for node in nodes: + compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start") + for node, address in zip(nodes, (R1_IP, R2_IP)): + harness.docker_wait_address(daemon, node["container_id"], address) + + a1 = harness.docker_anchor_name(n1["node_id"], *ETH) + a2 = harness.docker_anchor_name(n2["node_id"], *ETH) + bridge = harness.link_bridge_name(lid) + assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) + + # The uBridge default as reached through the server's own wiring: + # both anchors open to 0xfffd, the bridge level untouched. + for anchor in (a1, a2): + mask = _port_mask(anchor) + assert mask == 0xFFFD, f"{anchor}: group_fwd_mask {mask:#x} != 0xfffd" + assert int(harness.sysfs(f"/sys/class/net/{bridge}/bridge/group_fwd_mask"), 0) == 0 + + print(".. injecting the link-local matrix from guest 1") + _upload_probe(daemon, c1, c2) + matrix = MUST_CROSS + MUST_NOT_CROSS + with ThreadPoolExecutor(max_workers=1) as pool: + receiver = pool.submit(_recv_counts, daemon, c2, RECV_WINDOW, [mac for mac, _ in matrix]) + time.sleep(1.5) # let the receiver bind before the first frame + _send_all(daemon, c1, matrix) + counts = receiver.result() + print(f".. guest 2 received: {counts}") + for mac, _ in MUST_CROSS: + assert counts.get(mac, 0) > 0, f"{mac} did not cross the kernel link: {counts}" + for mac, _ in MUST_NOT_CROSS: + assert counts.get(mac, 0) == 0, f"{mac} unexpectedly crossed: {counts}" + + print(".. restarting container 1: the NIO replay must re-apply the mask") + compute.call("POST", f"/projects/{pid}/nodes/{n1['node_id']}/stop") + harness.wait_until(lambda: not harness.tap_exists(a1), timeout=15) + compute.call("POST", f"/projects/{pid}/nodes/{n1['node_id']}/start") + harness.docker_wait_address(daemon, c1, R1_IP) + assert harness.wait_until(lambda: harness.bridge_members(bridge) == sorted([a1, a2]), timeout=15) + mask = _port_mask(a1) + assert mask == 0xFFFD, f"{a1} after restart: group_fwd_mask {mask:#x} != 0xfffd" + + with ThreadPoolExecutor(max_workers=1) as pool: + receiver = pool.submit(_recv_counts, daemon, c2, 8, [LACP[0]]) + time.sleep(1.5) + _send_all(daemon, c1, [LACP]) + counts = receiver.result() + assert counts.get(LACP[0], 0) > 0, f"LACP dead after the container restart: {counts}" + except BaseException: + harness.release(server, pid, failed=True) + raise + harness.release(server, pid) From 2e29d053cf76d4e05ee47ee2b88fa64e0bd60ecd Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 14:04:25 +0800 Subject: [PATCH 070/101] =?UTF-8?q?docs:=20docker=20kernel=20datapath=20?= =?UTF-8?q?=E2=80=94=20diagram=20suite,=20glossary=20and=20refreshed=20con?= =?UTF-8?q?figuration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the ASCII art with a mermaid suite (steady-state topology with namespace boxes and sideband edge semantics, one-way frame path with per-object boxes, link-creation sequence, full relay path with the four kernel/user copies), add a term glossary ahead of the diagrams, reflow the prose to soft-wrap, and refresh the stale parts: the configuration section now describes enable_kernel_datapath as the global switch it already is, with every anchorable node type landed and linked to its feature doc. --- docs/features/docker-kernel-datapath.md | 589 +++++++++--------------- 1 file changed, 219 insertions(+), 370 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 4134a4c39..423189a2c 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -3,20 +3,13 @@ SPDX-License-Identifier: CC-BY-SA-4.0 See LICENSE file for licensing information. --> -> This documentation is AI-generated with reference to actual code and verified -> against real-kernel testing. AI can make mistakes — please verify against -> the source code when in doubt. +> This documentation is AI-generated with reference to actual code and verified against real-kernel testing. AI can make mistakes — please verify against the source code when in doubt. # Docker Kernel Datapath (veth + per-link Linux bridge) ## Overview -Docker node links historically flowed through a uBridge userspace relay: each -adapter was a TAP whose file descriptor uBridge pumped between the container -and UDP tunnels. This implementation replaces that model with **kernel-native -forwarding**: every adapter is born as a **veth pair**, and each link is a -**dedicated Linux kernel bridge** enslaving the two endpoints' veth host ends. -Frames never leave the kernel. +Docker node links historically flowed through a uBridge userspace relay: each adapter was a TAP whose file descriptor uBridge pumped between the container and UDP tunnels. This implementation replaces that model with **kernel-native forwarding**: every adapter is born as a **veth pair**, and each link is a **dedicated Linux kernel bridge** enslaving the two endpoints' veth host ends. Frames never leave the kernel. | | Old (relay) | New (kernel datapath) | |---|---|---| @@ -27,59 +20,21 @@ Frames never leave the kernel. | Packet filters | uBridge userspace filters | tc netem on the veth host end (delay / packet_loss / corrupt / duplicate + the netem extensions rate, reorder, gemodel, seed, limit, jitter distributions, loss correlation); bpf as cls_bpf match-drop classifiers; frequency_drop, quota and window_drop as the eBPF stateful classifier — every filter type has a kernel equivalent | | Capture / markers | relay bridge | uBridge AF_PACKET modules on the veth host end | -The work landed in nine stages on branch stack `feat/docker-kernel-datapath` → -`feat/docker-kernel-capture` → `feat/docker-kernel-markers` → -`feat/docker-veth-everywhere` → `feat/docker-kernel-filters` → -`feat/docker-kernel-bpf-drop` → `feat/docker-kernel-netem-ext` → -`feat/docker-kernel-ebpf-drops` → `feat/docker-kernel-window-drop`: +The work landed in nine stages on branch stack `feat/docker-kernel-datapath` → `feat/docker-kernel-capture` → `feat/docker-kernel-markers` → `feat/docker-veth-everywhere` → `feat/docker-kernel-filters` → `feat/docker-kernel-bpf-drop` → `feat/docker-kernel-netem-ext` → `feat/docker-kernel-ebpf-drops` → `feat/docker-kernel-window-drop`: -1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based - suspend, crash-safe reconciliation. +1. **Kernel datapath** — NIOBridge NIO type, per-link bridges, carrier-based suspend, crash-safe reconciliation. 2. **Capture** — `capture start_kernel` (AF_PACKET) on the veth host end. -3. **Markers** — `marker add_kernel` (AF_PACKET + BPF) with the same signals, - pcaps and fine-grained REST operations as the relay `mark` filter. -4. **veth-everywhere** — the TAP path deleted; every adapter is a veth and the - datapath is a *runtime* decision (kernel or relay) that never touches a - running container's interfaces. -5. **Filters** — impairment filters (delay, packet_loss, corrupt) become one - tc netem qdisc per veth host end, pushed through uBridge's netlink `tc` - module. This also removes the relay delay filter's nanosleep bottleneck - (upstream #2827) from the kernel path. -6. **bpf match-drop** — the `bpf` filter (any line matching drops) runs as - cls_bpf classifiers on the veth host end's clsact egress (`tc bpf_drop`: - uBridge pcap-compiles the line to classic BPF, the kernel migrates it to - eBPF internally — no CAP_BPF needed). Requires a uBridge reporting - `cbpf=1` via `tc capabilities`; older builds get a clear upgrade error. -7. **netem extensions** — the netem-extension surface delivered in uBridge - `feature/tc-netem-ext` is exposed as GNS3 filter types: `rate`, - `reorder`, `gemodel`, `duplicate`, `seed`, `limit`, plus the delay - `distribution` and loss/dup `correlation` parameters. Kernel-datapath - links only (the relay has no equivalent); capability-gated per uBridge - process. Reconcile became reset-before-set: the kernel's netem replace - merges optional attributes, so removing a parameter needs the explicit - reset. -8. **eBPF stateful drops** — `frequency_drop` becomes the eBPF classifier's - exact every-Nth mode (`tc nth_drop`, uBridge `feature/tc-precision`; - -1 = drop everything → every 1st) and the new kernel-only `quota` type - the byte-cap mode (`tc quota_drop`). Needs a uBridge reporting `ebpf=1` - (setcap cap_bpf,cap_net_admin,cap_net_raw=ep — verified working for a - non-root server process, the production credential shape). With this, - no filter type forces the relay anymore: kernel/relay is a purely - topological choice. -9. **window_drop flaps** — the eBPF classifier's time-window mode - (`tc window_drop`, uBridge `feature/tc-window`): a single outage that - passes traffic before and after, a recurring flap with `period ≥ outage`, - and an optional per-cycle jitter that randomizes each beat's outage and - period. The originally delivered back-to-back window implementation - (which made "outside the window packets pass" unreachable) was fixed - uBridge-side; this stage exposes the corrected semantics as the - kernel-only `window_drop` filter type. +3. **Markers** — `marker add_kernel` (AF_PACKET + BPF) with the same signals, pcaps and fine-grained REST operations as the relay `mark` filter. +4. **veth-everywhere** — the TAP path deleted; every adapter is a veth and the datapath is a *runtime* decision (kernel or relay) that never touches a running container's interfaces. +5. **Filters** — impairment filters (delay, packet_loss, corrupt) become one tc netem qdisc per veth host end, pushed through uBridge's netlink `tc` module. This also removes the relay delay filter's nanosleep bottleneck (upstream #2827) from the kernel path. +6. **bpf match-drop** — the `bpf` filter (any line matching drops) runs as cls_bpf classifiers on the veth host end's clsact egress (`tc bpf_drop`: uBridge pcap-compiles the line to classic BPF, the kernel migrates it to eBPF internally — no CAP_BPF needed). Requires a uBridge reporting `cbpf=1` via `tc capabilities`; older builds get a clear upgrade error. +7. **netem extensions** — the netem-extension surface delivered in uBridge `feature/tc-netem-ext` is exposed as GNS3 filter types: `rate`, `reorder`, `gemodel`, `duplicate`, `seed`, `limit`, plus the delay `distribution` and loss/dup `correlation` parameters. Kernel-datapath links only (the relay has no equivalent); capability-gated per uBridge process. Reconcile became reset-before-set: the kernel's netem replace merges optional attributes, so removing a parameter needs the explicit reset. +8. **eBPF stateful drops** — `frequency_drop` becomes the eBPF classifier's exact every-Nth mode (`tc nth_drop`, uBridge `feature/tc-precision`; -1 = drop everything → every 1st) and the new kernel-only `quota` type the byte-cap mode (`tc quota_drop`). Needs a uBridge reporting `ebpf=1` (setcap cap_bpf,cap_net_admin,cap_net_raw=ep — verified working for a non-root server process, the production credential shape). With this, no filter type forces the relay anymore: kernel/relay is a purely topological choice. +9. **window_drop flaps** — the eBPF classifier's time-window mode (`tc window_drop`, uBridge `feature/tc-window`): a single outage that passes traffic before and after, a recurring flap with `period ≥ outage`, and an optional per-cycle jitter that randomizes each beat's outage and period. The originally delivered back-to-back window implementation (which made "outside the window packets pass" unreachable) was fixed uBridge-side; this stage exposes the corrected semantics as the kernel-only `window_drop` filter type. ## Adapter interface types -Which interface a Docker node's adapters use is decided by the node class -(`Docker._select_node_class`, from `console_type` + `GNS3_*` environment -markers): +Which interface a Docker node's adapters use is decided by the node class (`Docker._select_node_class`, from `console_type` + `GNS3_*` environment markers): | Node class | Selected by | Adapter interface | Kernel links | |---|---|---|---| @@ -88,90 +43,192 @@ markers): | `IOLDockerVM` | `GNS3_IOL_RUNNER=1` | AF_UNIX datagram socket pairs (`sNN`/`cNN`) — the container netns is unused; kernel links ride persistent `gx` TAP anchors through the port bridge's swappable TAP leg (see `iol-docker-kernel-datapath.md`) | yes (gated on `ubridge_bridge_tap` + `ubridge_tap`) | | `VendorDockerVM` (unix-socket) | `GNS3_UNIX_SOCKET_NIO=1` | same socket contract, generic capability for vendor NOS images | rejected | -Unix-socket containers are detected both in the controller -(`_is_unix_socket_docker`, link eligibility) and on the compute side (NIO -rejection) — a missed detection surfaces as a clearer-late error. +Unix-socket containers are detected both in the controller (`_is_unix_socket_docker`, link eligibility) and on the compute side (NIO rejection) — a missed detection surfaces as a clearer-late error. ## Architecture -Kernel link between two containers on the same compute: +Kernel link between two containers on the same compute. Interface names follow two deterministic rules — `gv|gc{node_id[:8]}e{adapter}p{port}` for the veth pair (`gv` = GNS3 veth host end, `gc` = its guest end, moved into the container and renamed `eth{N}`) and `gns3{link_id[:11]}` for the bridge; the concrete names in the diagrams use node ids `3f2a1b9c` / `7c9d4e2f` and link id prefix `5d3c2b1a4e6`. Terms the diagrams use: -``` - container A netns root namespace container B netns - ┌────────────┐ veth pair ┌───────────────────┐ veth pair ┌────────────┐ - │ eth0 ◄─────┼───────────────┼─► gv{id}e0p0 ──────┼──► gns3{lid} │──────────► │ eth0 - └────────────┘ gc{id}e0p0 │ (bridge port) │ (bridge port)│ └────────────┘ - │ per-link kernel │ gv{id}e0p0 - │ bridge = "cable" │ - └───────────────────┘ +| Term | Meaning | +|---|---| +| uBridge | the per-node helper process gns3-server spawns with elevated capabilities; its hypervisor command surface executes all host-side wiring (veths, bridges, carrier, tc, taps) — on a kernel link it builds and observes the path but never carries frames | +| anchor | the host-side interface a link attaches to — for Docker, the `gv` veth host end | +| per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | +| veth pair | a kernel virtual Ethernet pair — two interfaces created together and wired so a frame into one end comes out the other; here one end is the container's `eth0`, the other the `gv` host end | +| bridge port | an interface enslaved to a Linux bridge (`brctl addif`) — it no longer sends or receives on its own: frames arriving on it go to the bridge's forwarding logic, and the bridge transmits through it | +| veth crossing | a frame passing through the veth pair between a container netns and the root namespace — pure kernel, no userspace | +| FDB | the bridge's forwarding database (learned MAC → port); unknown, broadcast and multicast destinations are flooded | +| `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | +| tc / qdisc | traffic-control objects the kernel inserts into the host end's egress path — stages frames pass through in flight, not external packet consumers; uBridge only installs them over netlink | +| `clsact` | the tc classifier carrier on the host end; its egress chain is where the drop classifiers run | +| eBPF classifier | the stateful drop modes (every-Nth, quota, time window) at clsact egress prio 1 | +| `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | +| `netem` | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | +| `AF_PACKET tap` | uBridge's packet socket bound to the host end; the kernel clones every frame the interface sends or receives to it (the tcpdump mechanism) — read-only copies, the original stays on the kernel path | +| carrier | the host end's admin state (`link set up/down`); suspend is carrier down on both ends = 100 % loss | +| `nio_bridge` | the NIO record the controller posts to each endpoint node: bridge name, filters, markers, suspend flag | + +```mermaid +flowchart LR + subgraph NetA["container A netns"] + ethA["eth0"] + end + subgraph RootNS["root namespace"] + gvA["gv3f2a1b9ce0p0 — anchor"] + BR["gns35d3c2b1a4e6 — per-link bridge"] + gvB["gv7c9d4e2fe0p0 — anchor"] + UB1["uBridge A"] + UB2["uBridge B"] + end + subgraph NetB["container B netns"] + ethB["eth0"] + end + ethA <-->|"veth pair"| gvA + gvA <-->|"bridge port"| BR + BR <-->|"bridge port"| gvB + gvB <-->|"veth pair"| ethB + gvA -. "frame copies" .-> UB1 + UB1 -. "qdisc install" .-> gvA + gvB -. "frame copies" .-> UB2 + UB2 -. "qdisc install" .-> gvB ``` -Relay link on the same unified veth (fallback for mixed node types, cross- -compute wiring or when `enable_kernel_datapath` is off): +Everything in the root namespace is built the same way: gns3-server creates the veth pairs at container start, the bridge and its memberships at link creation, and every later change (carrier, capture, markers, filters) by sending hypervisor commands to its per-node uBridge process — the privileged agent that turns them into netlink/ioctl operations (the production credential shape is a non-root server with a capabilities-equipped ubridge). That build flow is the Link creation sequence below; this diagram is the steady state that remains, where the only uBridge relationships left are the two dashed kinds. -``` - ┌────────────┐ ┌────────────────────────────┐ ┌────────────┐ - │ eth0 │ veth │ uBridge relay bridge │ UDP │ peer node │ - │ (gc end) │◄────────────►│ add_nio_ethernet (AF_ │◄────────────►│ (any type, │ - └────────────┘ gv host end │ PACKET) ↔ add_nio_udp │ tunnel │ any host) │ - └────────────────────────────┘ └────────────┘ +uBridge sits off the forwarding path — the dashed edges are control and observation, never data flow. The `qdisc install` edge is uBridge configuring the kernel over netlink; the kernel then runs those qdiscs in-flight on the host end's egress (they are stages of the transmit path, not packet consumers). The `frame copies` edge is the AF_PACKET tap: bound to the host end, the kernel clones every frame the interface sends or receives to uBridge's packet socket (the tcpdump mechanism) — read-only copies; the originals never leave the kernel. + +Relay link on the same unified veth and the same single compute as above — what the link becomes when `enable_kernel_datapath` is off (mixed node types and cross-compute wiring fall back too; there the UDP hop rides the real network instead of loopback, the copy count unchanged): + +```mermaid +flowchart LR + subgraph KA["container A netns"] + EA["eth0"] + end + subgraph KERN["kernel — root ns"] + GVA["gv host end"] + UDP["UDP sockets (loopback)"] + GVB["gv host end"] + end + subgraph UBA["uBridge A — userspace"] + NEA["AF_PACKET nio"] + ENA["relay + userspace filters"] + NUA["UDP nio"] + end + subgraph UBB["uBridge B — userspace"] + NUB["UDP nio"] + ENB["relay + userspace filters"] + NEB["AF_PACKET nio"] + end + subgraph KB["container B netns"] + EB["eth0"] + end + EA -->|"veth pair"| GVA + GVA -->|"copy 1: kernel→user"| NEA + NEA --> ENA --> NUA + NUA -->|"copy 2: user→kernel"| UDP + UDP -->|"copy 3: kernel→user"| NUB + NUB --> ENB --> NEB + NEB -->|"copy 4: user→kernel"| GVB + GVB -->|"veth pair"| EB ``` -Both datapaths anchor on the *same* veth host end; switching between them -(link delete + recreate) never touches the container-side interface. +Drawn one-way; the reverse direction is the mirror image. A relayed frame crosses the kernel/user boundary **four times each way** (eight per round trip — the measured ≈0.3 ms RTT against ≈0.05 ms on the kernel datapath, whose crossings are zero), and the relay's filters run in userspace on the frames between the copies. + +Both datapaths anchor on the *same* veth host end; switching between them (link delete + recreate) never touches the container-side interface. + +Frame path for one A → B frame. The impairment chain fires on the egress of the *destination* end's veth, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gvA — hence `delay 100` ≈200 ms RTT, `packet_loss 30` ≈51 % round trip: + +```mermaid +flowchart LR + subgraph ANS["container A netns"] + S1["eth0 A sends"] + end + subgraph ROOT["root namespace"] + subgraph GVA["gvA — ingress (its egress chain serves B→A)"] + S3["ingress"] + end + subgraph BRP["per-link bridge"] + G1{"reserved 01:80:c2?"} + D["dropped"] + S4["FDB lookup / flood"] + end + subgraph GVB["gvB — egress chain (clsact, then netem)"] + S5["eBPF prio 1"] + S6["bpf_drop prio 10-99"] + S7["netem"] + end + end + subgraph BNS["container B netns"] + S9["eth0 B receives"] + end + S1 -->|"veth pair"| S3 + S3 --> G1 + G1 -->|"PAUSE 0x01"| D + G1 -->|"0xfffd passes"| S4 + S4 --> S5 + S5 --> S6 --> S7 + S7 -->|"veth pair"| S9 +``` + +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Packet filters below). The AF_PACKET taps observe both host ends but never a clsact-dropped frame. + +All dozen-plus GNS3 filter types ride these three kernel objects — nine of them (delay, packet_loss, corrupt, duplicate, rate, reorder, gemodel, seed, limit) are parameters of the single netem qdisc, `bpf` expressions are the cls_bpf filters, and frequency_drop / quota / window_drop are the three modes inside the one eBPF program (evaluated nth → quota → window). The complete type-to-object mapping is the Packet filters table below. ## Datapath selection -`UDPLink._kernel_datapath_eligible` decides in the controller at NIO prepare -time; the compute side only reacts to the NIO type: +`UDPLink._kernel_datapath_eligible` decides in the controller at NIO prepare time; the compute side only reacts to the NIO type: -* both endpoints Docker (any class except unix-socket containers), same - compute +* both endpoints on the same compute, both able to anchor — Docker anchors via its veth (any class except unix-socket containers), and QEMU / IOU / Dynamips / Ethernet-switch peers anchor via their own mechanisms, so mixed links take the kernel path too * `Server.enable_kernel_datapath` enabled (default) -Filters never disqualify the kernel path anymore — every type has a kernel -equivalent (netem / cls_bpf / eBPF classifier). +Filters never disqualify the kernel path anymore — every type has a kernel equivalent (netem / cls_bpf / eBPF classifier). -There is **no stopped-node requirement**. Since every adapter is a veth, links -attach to running containers: `brctl addif` (kernel) or -`bridge add_nio_ethernet` (relay) are runtime-safe operations on the host end. -On project (re)open every link re-runs `_prepare`, so relay links whose -endpoints became eligible are upgraded to the kernel datapath automatically. +There is **no stopped-node requirement**. Since every adapter is a veth, links attach to running containers: `brctl addif` (kernel) or `bridge add_nio_ethernet` (relay) are runtime-safe operations on the host end. On project (re)open every link re-runs `_prepare`, so relay links whose endpoints became eligible are upgraded to the kernel datapath automatically. -Which datapath a link ended up on is visible in its REST payload (and the -`link_list` / `link_get` MCP tools) as the read-only boolean -`kernel_datapath`, recomputed from the NIO wiring — a relay link upgraded on -reopen flips the field. It is not persisted and never sent on create/update. +Which datapath a link ended up on is visible in its REST payload (and the `link_list` / `link_get` MCP tools) as the read-only boolean `kernel_datapath`, recomputed from the NIO wiring — a relay link upgraded on reopen flips the field. It is not persisted and never sent on create/update. + +## Link creation sequence + +Both Docker nodes sit on the same compute (a kernel-link precondition) and each drives its own uBridge process. They attach concurrently and compute the same bridge name independently (a pure function of the link id), so either may win the `brctl create` race — the loser verifies the bridge exists (`brctl show`) and continues: + +```mermaid +sequenceDiagram + participant API as REST API + participant C as Controller + participant CA as Docker node A + participant CB as Docker node B + participant UA as uBridge A + participant UB as uBridge B + + API->>C: POST /links + C->>C: eligibility check + C->>C: bridge = gns3 + link_id[:11] + C--)CA: nio_bridge + C--)CB: nio_bridge (same bridge) + par Docker node A + CA->>UA: brctl create + CA->>UA: link set bridge up + CA->>UA: brctl addif gvA + CA->>UA: port fwd_mask 0xfffd + CA->>UA: markers / capture if any + CA->>UA: tc netem + bpf + eBPF + CA->>UA: link set gvA up + and Docker node B + CB->>UB: same on gvB + end + Note over UA,UB: steady state — kernel forwards, uBridge taps +``` + +Every arrow to uBridge is one `_ubridge_send` command line on the node's local hypervisor console; uBridge executes it (netlink/ioctl) and replies OK or an error synchronously before the next command is sent — those replies are omitted for brevity. uBridge never initiates anything: it is a pure executor, and the only data ever flowing back (capture/marker frame copies) rides separate AF_PACKET sockets, not this console. + +The veth pairs already exist at attach time — every adapter is born as a veth at container start (Adapter lifecycle below), which is what makes the attach a runtime-safe operation on a running container. ## Adapter lifecycle -* **Birth (container start)** — `_create_veth`: stale sweep of any leftover - pair (crash residue), `docker create_veth`, host end admin-down, MAC from - the adapter base, `docker move_to_ns` of the guest end (renamed `eth{N}`, or - the `GNS3_INTERFACE_NAMES` mapping). Names are deterministic: - `gv|gc{node_id[:8]}e{adapter}p{port}` (≤ 15 chars). Unconnected adapters are - born too (carrier off) so the interface is visible inside the container. -* **Life (running)** — the veth is never created/deleted/moved again. Link - create/delete/switch, suspend, capture and markers only change *what is - attached to the host end*. -* **Death (container stop)** — `_remove_kernel_veths` deletes the host ends - explicitly: unlike a TAP (which died with the container netns), a veth host - end outlives the container. Deleting either end destroys the pair. The - node's per-link kernel bridges go with them — deleting a node or closing a - project never runs the link-teardown path (no per-link delete is issued), - so a bridge whose ports just disappeared would stay behind as an empty - orphan. Both endpoints do this and the peer's still-enslaved port makes - the delete fail with EBUSY (suppressed, last one wins), the same contract - as link deletion; a node restart rebuilds the bridge from the NIO. -* **Host L3 noise (known, hardening specified).** An anchor is a host-side - netdev like any other, so the kernel gives it an IPv6 link-local address and - emits its own MLD/DAD from it (measured ≈6 frames / 2 s on an idle TAP; the - production anchors and per-link bridges carry `fe80::…` today). On a kernel - link the bridge floods those frames into the emulated segment — and the host - answers neighbor solicitations for its own address, which looks like a phantom - IPv6 neighbor to the emulated nodes. The fix is to make anchors pure L2 at - creation time on the uBridge side: - `docs/design/ubridge-l2-anchor-spec.md`. +* **Birth (container start)** — `_create_veth`: stale sweep of any leftover pair (crash residue), `docker create_veth`, host end admin-down, MAC from the adapter base, `docker move_to_ns` of the guest end (renamed `eth{N}`, or the `GNS3_INTERFACE_NAMES` mapping). Names are deterministic: `gv|gc{node_id[:8]}e{adapter}p{port}` (≤ 15 chars). Unconnected adapters are born too (carrier off) so the interface is visible inside the container. +* **Life (running)** — the veth is never created/deleted/moved again. Link create/delete/switch, suspend, capture and markers only change *what is attached to the host end*. +* **Death (container stop)** — `_remove_kernel_veths` deletes the host ends explicitly: unlike a TAP (which died with the container netns), a veth host end outlives the container. Deleting either end destroys the pair. The node's per-link kernel bridges go with them — deleting a node or closing a project never runs the link-teardown path (no per-link delete is issued), so a bridge whose ports just disappeared would stay behind as an empty orphan. Both endpoints do this and the peer's still-enslaved port makes the delete fail with EBUSY (suppressed, last one wins), the same contract as link deletion; a node restart rebuilds the bridge from the NIO. +* **Host L3 noise (known, hardening specified).** An anchor is a host-side netdev like any other, so the kernel gives it an IPv6 link-local address and emits its own MLD/DAD from it (measured ≈6 frames / 2 s on an idle TAP; the production anchors and per-link bridges carry `fe80::…` today). On a kernel link the bridge floods those frames into the emulated segment — and the host answers neighbor solicitations for its own address, which looks like a phantom IPv6 neighbor to the emulated nodes. The fix is to make anchors pure L2 at creation time on the uBridge side: `docs/design/ubridge-l2-anchor-spec.md`. ## Link operations @@ -184,18 +241,11 @@ reopen flips the field. It is not persisted and never sent on create/update. ## Capture -Kernel links capture via uBridge's AF_PACKET module bound to the veth host -end: `capture start_kernel "" [dlt]` / `capture stop_kernel`. -Single capture per uBridge process (second concurrent → EALREADY). Start/stop -key on the **NIO type**, not on veth presence — a relay NIO riding a veth -captures at its relay bridge. +Kernel links capture via uBridge's AF_PACKET module bound to the veth host end: `capture start_kernel "" [dlt]` / `capture stop_kernel`. Single capture per uBridge process (second concurrent → EALREADY). Start/stop key on the **NIO type**, not on veth presence — a relay NIO riding a veth captures at its relay bridge. ## Markers -Markers ride the NIO like on the relay datapath, routed by `capture_node_id`; -the anchor is the veth host end instead of a relay bridge -(`_ubridge_apply_markers(anchor, nio)` is datapath-agnostic; DockerVM -overrides the add/delete/enable primitives to translate the commands): +Markers ride the NIO like on the relay datapath, routed by `capture_node_id`; the anchor is the veth host end instead of a relay bridge (`_ubridge_apply_markers(anchor, nio)` is datapath-agnostic; DockerVM overrides the add/delete/enable primitives to translate the commands): ``` marker add_kernel "" [tag ] [link ] [dir ] [linktype ] [pcap ""] @@ -203,17 +253,13 @@ marker delete_kernel # idempotent marker enable_kernel # off = installed but silent ``` -* Per-deployment convention: **only the capture node's end** runs - `add_kernel` (symmetric with capture) — signals form clean send/return - pairs (`PACKET_OUTGOING` on the host end = container receiving = `rx`). -* Same BPF compilation, MARK UDP signals, pcap files and fine-grained REST - toggle/rebuild/delete endpoints as the relay `mark` filter. +* Per-deployment convention: **only the capture node's end** runs `add_kernel` (symmetric with capture) — signals form clean send/return pairs (`PACKET_OUTGOING` on the host end = container receiving = `rx`). +* Same BPF compilation, MARK UDP signals, pcap files and fine-grained REST toggle/rebuild/delete endpoints as the relay `mark` filter. * Node restart reinstalls markers from the NIO; link deletion tears them down. ## Packet filters -Impairment filters run **in the kernel** as tc netem qdiscs on the veth host -ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): +Impairment filters run **in the kernel** as tc netem qdiscs on the veth host ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): | GNS3 filter | kernel implementation | Notes | |---|---|---| @@ -233,131 +279,30 @@ ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): Semantics: -* **Both endpoints** receive the filter dict and attach one qdisc to *their* - veth host end. The qdisc's egress covers traffic entering that container, - so every direction of the link is impaired exactly once — the same net - effect as the relay, where both directions cross the single filtered - bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` - measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the - per-packet serialization time to the RTT. -* **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES - optional attributes (rate, correlation, reorder, corrupt, gemodel, - distribution: an absent attr keeps its previous value), so re-applying a - filter set with a parameter *removed* would silently keep the old value. - Every apply therefore starts with `tc reset` (also drops clsact and its - bpf_drop filters — re-added right after in the same flow) followed by one - `netem set` built from the current filters; no per-parameter diffing. An - empty filter set is the reset alone (ENOENT-tolerated). -* **Extension gating.** The netem-extension keywords (rate, reorder, - gemodel, dist, seed, limit, and the correl suffixes) probe - `tc capabilities` once per uBridge process and require the tokens; - a plain delay/loss/corrupt/dup filter never probes — an old uBridge - serves the original surface untouched. -* **kernel-only vs relay-available.** The extension types and `quota` have - no uBridge relay equivalent: `available_filters` hides them on relay - links, setting one on a created relay link returns 409, and a project - loaded with such filters on a link that cannot be kernel-wired drops them - with a warning (invalid filters are dropped the same way at load). - `frequency_drop` runs on both datapaths (relay userspace filter / eBPF - every-Nth). -* **frequency_drop counting semantics differ per datapath.** The relay's - single filtered bridge counts packets of BOTH directions through one - counter; the kernel attaches one classifier per veth end, so each - direction drops every Nth independently. For a round-trip measurement - with every-Nth N: relay ≈ 1/N of frames (phase-locked), kernel = 1 − - ((N−1)/N)² (e.g. N=3 → 55.6 % round-trip loss, measured 57 %). The - kernel count is exact (atomic counter), unlike netem's stochastic loss. -* **Classifier order** on clsact egress: the eBPF stateful filter runs at - prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the - stateful modes never reaches the expression drops, and dropped packets - never reach netem. Within the stateful program the modes evaluate in the - fixed order nth → quota → window. -* **window_drop semantics.** `start` is relative to the moment the filter - is applied — and because reconcile re-sets the classifier on every apply - (any filter change, node restart, link reset reloads the program after - the netem reset), every such event **restarts the schedule**. 3 - parameters = one single outage (`[start, start+outage)`), traffic passes - before and after — the classic "link dies at T, comes back at T+outage". - 4 parameters = recurring flap: `period ≥ outage` (validated), so the - outage occupies `outage/period` of each cycle. 5 parameters = randomized - flap: each cycle's outage and period are re-drawn uniformly in nominal - ± `jitter` (integer-ms grid; jitter 0 draws nothing and equals the fixed - schedule). Inside a window each packet drops with the given chance (100 - = full outage, lower = degraded service during the window). Both - endpoints run their own schedule, started ~simultaneously by the - controller — a round trip survives only when **both** directions are - outside their windows. Note `start = 0` (outage begins immediately) is a - valid, active configuration — unlike other filters, a zero first - parameter does not mean "disabled". A PUT re-sending an unchanged filters - dict is normally a controller no-op, but one still carrying `window_drop` - is reconciled anyway, so re-applying the same window re-arms a one-shot - outage that has already passed (arm the filter immediately before - generating traffic, or use a period for a repeatable event). -* **Validation** mirrors the tc grammar: `reorder` requires `delay`, - `gemodel` and `packet_loss` are mutually exclusive (both map to the netem - loss keyword), `distribution` requires jitter > 0, rate must be integer + - unit ≤ 100gbit. -* **Node restart** restores the qdisc from the NIO (like capture and markers); - **link deletion** detaches it explicitly — the veth survives the link and an - orphaned qdisc would keep impairing the next one. -* **Suspend** keeps the qdisc (carrier-driven loss); the first packet after - resume takes one extra delay interval (known netem idle-baseline behaviour) - and steady state is exact. -* **bpf reconcile = flush + re-add** (`tc bpf_drop flush`, then one add per - line at priorities 10, 11, …). `tc reset` (link delete / filter clear) is - the full restore in uBridge: filters → clsact → root qdisc, idempotent. -* **Observability caveat:** clsact drops happen before the AF_PACKET tap - points — markers/captures on the same veth do not observe cls_bpf-dropped - frames (on the relay, filter-vs-mark ordering determined visibility - instead). Also note `tc qdisc show` cannot print a distribution's name - (the kernel stores only the sampled table). -* **eBPF mode tokens.** A stateful mode is usable iff `tc capabilities` - reports `ebpf=1` **and** lists the mode's token in `ebpf_modes` - (uBridge `feature/tc-window`+). Builds predating the field keep the modes - that shipped with it — everything but `window`, whose semantics were - corrected exactly when the field was added: gating on `ebpf=1` alone - would silently install the broken back-to-back-window behaviour on a - pre-correction build. Requesting an unusable mode fails the apply with an - upgrade error; the cleanup ("off") path only touches modes the build - declares. -* **Capability reporting.** The compute exposes the probe through - `GET /v3/compute/capabilities` as `ubridge_tc` (netem keyword list, - `ebpf`/`cbpf` flags, and the usable `ebpf_modes` — legacy fallback - applied), answered by a throwaway uBridge spawned as the server's own - user and cached by binary identity — the runtime `ebpf` flag flips - between users and kernels for the same binary, so a root-run probe would - lie for a non-root server. The controller forwards it through - `GET /v3/computes` and `available_filters` hides kernel-datapath filter - types an endpoint compute reports it cannot run (both ends apply their - own filters, so one incapable compute is enough). Computes that report - nothing — older servers, failed probe — keep the full list; - apply-time validation (409) stays the guard. +* **Both endpoints** receive the filter dict and attach one qdisc to *their* veth host end. The qdisc's egress covers traffic entering that container, so every direction of the link is impaired exactly once — the same net effect as the relay, where both directions cross the single filtered bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the per-packet serialization time to the RTT. +* **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES optional attributes (rate, correlation, reorder, corrupt, gemodel, distribution: an absent attr keeps its previous value), so re-applying a filter set with a parameter *removed* would silently keep the old value. Every apply therefore starts with `tc reset` (also drops clsact and its bpf_drop filters — re-added right after in the same flow) followed by one `netem set` built from the current filters; no per-parameter diffing. An empty filter set is the reset alone (ENOENT-tolerated). +* **Extension gating.** The netem-extension keywords (rate, reorder, gemodel, dist, seed, limit, and the correl suffixes) probe `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. +* **kernel-only vs relay-available.** The extension types and `quota` have no uBridge relay equivalent: `available_filters` hides them on relay links, setting one on a created relay link returns 409, and a project loaded with such filters on a link that cannot be kernel-wired drops them with a warning (invalid filters are dropped the same way at load). `frequency_drop` runs on both datapaths (relay userspace filter / eBPF every-Nth). +* **frequency_drop counting semantics differ per datapath.** The relay's single filtered bridge counts packets of BOTH directions through one counter; the kernel attaches one classifier per veth end, so each direction drops every Nth independently. For a round-trip measurement with every-Nth N: relay ≈ 1/N of frames (phase-locked), kernel = 1 − ((N−1)/N)² (e.g. N=3 → 55.6 % round-trip loss, measured 57 %). The kernel count is exact (atomic counter), unlike netem's stochastic loss. +* **Classifier order** on clsact egress: the eBPF stateful filter runs at prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the stateful modes never reaches the expression drops, and dropped packets never reach netem. Within the stateful program the modes evaluate in the fixed order nth → quota → window. +* **window_drop semantics.** `start` is relative to the moment the filter is applied — and because reconcile re-sets the classifier on every apply (any filter change, node restart, link reset reloads the program after the netem reset), every such event **restarts the schedule**. 3 parameters = one single outage (`[start, start+outage)`), traffic passes before and after — the classic "link dies at T, comes back at T+outage". 4 parameters = recurring flap: `period ≥ outage` (validated), so the outage occupies `outage/period` of each cycle. 5 parameters = randomized flap: each cycle's outage and period are re-drawn uniformly in nominal ± `jitter` (integer-ms grid; jitter 0 draws nothing and equals the fixed schedule). Inside a window each packet drops with the given chance (100 = full outage, lower = degraded service during the window). Both endpoints run their own schedule, started ~simultaneously by the controller — a round trip survives only when **both** directions are outside their windows. Note `start = 0` (outage begins immediately) is a valid, active configuration — unlike other filters, a zero first parameter does not mean "disabled". A PUT re-sending an unchanged filters dict is normally a controller no-op, but one still carrying `window_drop` is reconciled anyway, so re-applying the same window re-arms a one-shot outage that has already passed (arm the filter immediately before generating traffic, or use a period for a repeatable event). +* **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + unit ≤ 100gbit. +* **Node restart** restores the qdisc from the NIO (like capture and markers); **link deletion** detaches it explicitly — the veth survives the link and an orphaned qdisc would keep impairing the next one. +* **Suspend** keeps the qdisc (carrier-driven loss); the first packet after resume takes one extra delay interval (known netem idle-baseline behaviour) and steady state is exact. +* **bpf reconcile = flush + re-add** (`tc bpf_drop flush`, then one add per line at priorities 10, 11, …). `tc reset` (link delete / filter clear) is the full restore in uBridge: filters → clsact → root qdisc, idempotent. +* **Observability caveat:** clsact drops happen before the AF_PACKET tap points — markers/captures on the same veth do not observe cls_bpf-dropped frames (on the relay, filter-vs-mark ordering determined visibility instead). Also note `tc qdisc show` cannot print a distribution's name (the kernel stores only the sampled table). +* **eBPF mode tokens.** A stateful mode is usable iff `tc capabilities` reports `ebpf=1` **and** lists the mode's token in `ebpf_modes` (uBridge `feature/tc-window`+). Builds predating the field keep the modes that shipped with it — everything but `window`, whose semantics were corrected exactly when the field was added: gating on `ebpf=1` alone would silently install the broken back-to-back-window behaviour on a pre-correction build. Requesting an unusable mode fails the apply with an upgrade error; the cleanup ("off") path only touches modes the build declares. +* **Capability reporting.** The compute exposes the probe through `GET /v3/compute/capabilities` as `ubridge_tc` (netem keyword list, `ebpf`/`cbpf` flags, and the usable `ebpf_modes` — legacy fallback applied), answered by a throwaway uBridge spawned as the server's own user and cached by binary identity — the runtime `ebpf` flag flips between users and kernels for the same binary, so a root-run probe would lie for a non-root server. The controller forwards it through `GET /v3/computes` and `available_filters` hides kernel-datapath filter types an endpoint compute reports it cannot run (both ends apply their own filters, so one incapable compute is enough). Computes that report nothing — older servers, failed probe — keep the full list; apply-time validation (409) stays the guard. ## Link-local frames (LACP, LLDP, 802.1X, STP) -The bridge stands in for a cable, but the kernel's `br_handle_frame()` does -not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by -default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while -the UDP relay carries them fine. uBridge therefore opens every port's -per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at -`brctl addif` time (Linux 4.15+, best-effort on older kernels). The -bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, -so LACP is reachable per port only. +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. -Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and -LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in -`br_handle_frame()` drops it unconditionally and no mask can enable it. -That is a documented limit, not a regression (PFC's hardware semantics are -out of emulation's reach anyway); a lab that needs PAUSE frames on the wire -must use a relay link. Verified live by -`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest -matrix); the contract and kernel references live in -`docs/design/ubridge-link-local-frame-forwarding-spec.md`. +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in `br_handle_frame()` drops it unconditionally and no mask can enable it. That is a documented limit, not a regression (PFC's hardware semantics are out of emulation's reach anyway); a lab that needs PAUSE frames on the wire must use a relay link. Verified live by `tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest matrix); the contract and kernel references live in `docs/design/ubridge-link-local-frame-forwarding-spec.md`. ## uBridge command surface -No uBridge changes were needed beyond the marker module's `*_kernel` commands -(the AF_PACKET capture module was pre-existing): +No uBridge changes were needed beyond the marker module's `*_kernel` commands (the AF_PACKET capture module was pre-existing): ``` docker create_veth / delete_veth / move_to_ns / set_mac_addr @@ -381,131 +326,35 @@ bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capt ```ini [Server] -# Wire eligible Docker-to-Docker links through kernel veth/bridge interfaces +# Wire eligible links through kernel interfaces instead of the uBridge UDP relay enable_kernel_datapath = True ``` -Intended to grow into the global datapath switch as IOU / Dynamips migrate to -kernel bridges. **QEMU has landed** on the same pattern (persistent TAP anchor -enslaved into the per-link bridge, zero relay) — see -`docs/features/qemu-kernel-datapath.md` — and **IOU after it** -(`docs/features/iou-kernel-datapath.md`; the fabric terminator in uBridge -gained a TAP-terminated port, `iol_bridge add_nio_tap`, specified in -`docs/design/ubridge-iol-tap-anchor-spec.md`). The shared half of this -document's compute machinery now lives in -`gns3server/compute/kernel_datapath.py`. +This is now the global datapath switch — `_kernel_datapath_eligible` consults it first, and every node type that can anchor a kernel link has landed: + +* **Docker** — veth host ends as anchors (this document) +* **QEMU** — persistent TAP anchors, see `docs/features/qemu-kernel-datapath.md` +* **IOU** — TAP-terminated fabric ports (`iol_bridge add_nio_tap`), see `docs/features/iou-kernel-datapath.md` +* **Dynamips** — uBridge-created TAPs the hypervisor opens, see `docs/features/dynamips-kernel-datapath.md` +* **IOL runner containers** — persistent `gx` TAPs through the bridge module's swappable TAP leg, see `docs/features/iol-docker-kernel-datapath.md` +* **Ethernet Switch** — its own uBridge `brctl` bridge that absorbs the peer's anchor as a fast path (switch-to-switch cascades over a veth pair), see `docs/features/ethernet-switch-kernel-datapath.md` + +A link rides the kernel when both endpoints sit on one compute, both can anchor (per-type uBridge capabilities), the switch is on and the ports are Ethernet; everything else falls back to the relay. The shared compute machinery behind all of these lives in `gns3server/compute/kernel_datapath.py`. ## Verification -End-to-end on a five-container FRR topology (mixed runtime-drawn and reloaded -links): 8/8 links on the kernel datapath, ping RTT ≈ 0.05 ms, FDB learning, -suspend = 100 % loss / resume restores, project reopen upgrades relay links, -runtime link creation on running containers, marker pcaps exact (tx/rx -pairs), capture freeze on stop, server restart reconciliation. +End-to-end on a five-container FRR topology (mixed runtime-drawn and reloaded links): 8/8 links on the kernel datapath, ping RTT ≈ 0.05 ms, FDB learning, suspend = 100 % loss / resume restores, project reopen upgrades relay links, runtime link creation on running containers, marker pcaps exact (tx/rx pairs), capture freeze on stop, server restart reconciliation. -Filters (two-container kernel link, 27/27 checks): baseline 0.06 ms → -`delay 100` = 200.2 ms RTT (2× per direction) → `delay 50` = 100.2 ms -(atomic replace) → clear = baseline (qdisc detached); `packet_loss 30` = -48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop` = 409 (at that -stage — superseded by the eBPF stage below); -suspend with delay active = 100 % loss, resume keeps the qdisc -(200.1 ms); node restart restores it (200.1 ms); link delete detaches it -(`/usr/sbin/tc qdisc show` — no netem left); re-created link has no residual -impairment. +Filters (two-container kernel link, 27/27 checks): baseline 0.06 ms → `delay 100` = 200.2 ms RTT (2× per direction) → `delay 50` = 100.2 ms (atomic replace) → clear = baseline (qdisc detached); `packet_loss 30` = 48 % round-trip loss (expected 51 % = 1 − 0.7²); `frequency_drop` = 409 (at that stage — superseded by the eBPF stage below); suspend with delay active = 100 % loss, resume keeps the qdisc (200.1 ms); node restart restores it (200.1 ms); link delete detaches it (`/usr/sbin/tc qdisc show` — no netem left); re-created link has no residual impairment. -bpf match-drop (same setup, 22/22 checks): `bpf "icmp"` = 100 % loss on both -ends' clsact; size-discriminating expression (`greater 150`) = small pings -pass / 300-byte pings dropped (real byte-level BPF match); two-line OR; -coexistence with netem (delay 50 → small pings 100.2 ms while big dropped); -clear removes clsact entirely; suspend/resume keeps the drops; link delete + -recreate leaves no residual filters; `frequency_drop` still 409 and hidden -(at that stage — superseded by the eBPF stage below); `available_filters` shows -`bpf`. Unit tests: -`tests/compute/docker/test_docker_kernel_datapath.py`, -`tests/controller/test_kernel_datapath_link.py`. +bpf match-drop (same setup, 22/22 checks): `bpf "icmp"` = 100 % loss on both ends' clsact; size-discriminating expression (`greater 150`) = small pings pass / 300-byte pings dropped (real byte-level BPF match); two-line OR; coexistence with netem (delay 50 → small pings 100.2 ms while big dropped); clear removes clsact entirely; suspend/resume keeps the drops; link delete + recreate leaves no residual filters; `frequency_drop` still 409 and hidden (at that stage — superseded by the eBPF stage below); `available_filters` shows `bpf`. Unit tests: `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. -netem extensions (same setup, 24/24 checks): `rate 512kbit` on both veths, -1400-byte pings measure 45.5 ms RTT = baseline + 2× the 22.2 ms -serialization (exact); `delay 100 20 paretonormal` applies delay+jitter and -clears the previous rate (leak detector for the kernel's merge-on-replace); -`reorder 25 gap 5` visible in the qdisc dump and traffic passes; -`gemodel 100/0/30` = 50 % bursty round-trip loss with the previous reorder -cleared; `duplicate 50` link alive; `packet_loss 30 correl 50` shows -`loss 30% 50%`; `seed 42` + `limit 5000` both visible in the dump, RTT -exactly 2×50 ms; suspend/resume and node restart keep the filters; a -docker↔ethernet-switch relay link rejects `rate` with 409 and its -`available_filters` hides the kernel-only types while the kernel link shows -them all (frequency_drop included since the eBPF stage); `reorder` without `delay` and -`gemodel`+`packet_loss` are 409s; teardown leaves no netem on either veth. -Also verified live: a relay attach (`bridge add_nio_ethernet`) on the -admin-down veth host end fails in libpcap — the relay path now brings the -interface up before attaching. +netem extensions (same setup, 24/24 checks): `rate 512kbit` on both veths, 1400-byte pings measure 45.5 ms RTT = baseline + 2× the 22.2 ms serialization (exact); `delay 100 20 paretonormal` applies delay+jitter and clears the previous rate (leak detector for the kernel's merge-on-replace); `reorder 25 gap 5` visible in the qdisc dump and traffic passes; `gemodel 100/0/30` = 50 % bursty round-trip loss with the previous reorder cleared; `duplicate 50` link alive; `packet_loss 30 correl 50` shows `loss 30% 50%`; `seed 42` + `limit 5000` both visible in the dump, RTT exactly 2×50 ms; suspend/resume and node restart keep the filters; a docker↔ethernet-switch relay link rejects `rate` with 409 and its `available_filters` hides the kernel-only types while the kernel link shows them all (frequency_drop included since the eBPF stage); `reorder` without `delay` and `gemodel`+`packet_loss` are 409s; teardown leaves no netem on either veth. Also verified live: a relay attach (`bridge add_nio_ethernet`) on the admin-down veth host end fails in libpcap — the relay path now brings the interface up before attaching. -eBPF stateful drops (same setup, 13/13 checks): `frequency_drop 3` installs -the tc_impair classifier on both veths (clsact egress pref 1, visible in -`tc filter show`); round-trip loss 57 % against the per-direction theory 1 − -(2/3)² = 55.6 % (each direction drops every 3rd independently — the relay's -single shared counter would give ≈ 1/3); `frequency_drop -1` = exactly -100 % loss; `quota 3000B/100 %` = pings pass until the byte cap then hard -cutoff; coexistence with `delay 50` (RTT 100.3 ms with every-4th drops); -clearing filters leaves no residual bpf filter; suspend = 100 % while down -with drops persisting after resume; node restart restores every-Nth on the -fresh veth; `available_filters` shows frequency_drop + quota on kernel -links and hides quota on relay links; a two-leg relay path through an -Ethernet switch still serves frequency_drop from the userspace filter -(56 % round-trip for N=3). The e2e also exposed and fixed a pre-existing -restart bug: the relay bridge-name registry was not cleared on stop, so a -node restart with a relay link skipped `bridge create` and failed. Unit tests: -`tests/utils/test_packet_filter_validation.py` (P6a class), -`tests/compute/docker/test_docker_kernel_datapath.py`, -`tests/controller/test_kernel_datapath_link.py`. +eBPF stateful drops (same setup, 13/13 checks): `frequency_drop 3` installs the tc_impair classifier on both veths (clsact egress pref 1, visible in `tc filter show`); round-trip loss 57 % against the per-direction theory 1 − (2/3)² = 55.6 % (each direction drops every 3rd independently — the relay's single shared counter would give ≈ 1/3); `frequency_drop -1` = exactly 100 % loss; `quota 3000B/100 %` = pings pass until the byte cap then hard cutoff; coexistence with `delay 50` (RTT 100.3 ms with every-4th drops); clearing filters leaves no residual bpf filter; suspend = 100 % while down with drops persisting after resume; node restart restores every-Nth on the fresh veth; `available_filters` shows frequency_drop + quota on kernel links and hides quota on relay links; a two-leg relay path through an Ethernet switch still serves frequency_drop from the userspace filter (56 % round-trip for N=3). The e2e also exposed and fixed a pre-existing restart bug: the relay bridge-name registry was not cleared on stop, so a node restart with a relay link skipped `bridge create` and failed. Unit tests: `tests/utils/test_packet_filter_validation.py` (P6a class), `tests/compute/docker/test_docker_kernel_datapath.py`, `tests/controller/test_kernel_datapath_link.py`. -window_drop (same setup, 15/15 checks, against uBridge `feature/tc-window`): -a single outage `[1000, 1500, 100]` delivers **per-seq evidence** of the B.2 -semantics — one ping per 200 ms arrives as `[0,1,2,3,4, 13…19]`: passing -before the window, dropped inside it, passing again after; a recurring flap -`[300, 1000, 100, 2000]` measures 52 % round-trip loss (the 1000/2000 duty -cycle); `period == outage` is a permanent outage (100 %); `period < outage` -is a 409 before uBridge sees it; a jittered flap -`[0, 1000, 100, 2000, 400]` keeps the ~50 % duty cycle (62 % sampled over -4 cycles — the per-cycle re-draw widens the variance by design); a 30 % -chance on a permanent window measures 53 % round trip against the -per-direction theory 1 − 0.7² = 51 % (the same arithmetic as -`packet_loss 30`); `start = 0` is stored as an *active* filter; the window -coexists with netem (`delay 50` + a permanent window = 100 % inside the -outage); clearing removes the classifier; a node restart restores the -schedule on the fresh veth; `available_filters` shows window_drop on kernel -links and a relay link both hides it and 409s it; teardown leaves no filter -or qdisc behind. The P6b e2e was re-run as a regression on the same uBridge -build (13/13 — nth/quota unaffected by the de-looping below). +window_drop (same setup, 15/15 checks, against uBridge `feature/tc-window`): a single outage `[1000, 1500, 100]` delivers **per-seq evidence** of the B.2 semantics — one ping per 200 ms arrives as `[0,1,2,3,4, 13…19]`: passing before the window, dropped inside it, passing again after; a recurring flap `[300, 1000, 100, 2000]` measures 52 % round-trip loss (the 1000/2000 duty cycle); `period == outage` is a permanent outage (100 %); `period < outage` is a 409 before uBridge sees it; a jittered flap `[0, 1000, 100, 2000, 400]` keeps the ~50 % duty cycle (62 % sampled over 4 cycles — the per-cycle re-draw widens the variance by design); a 30 % chance on a permanent window measures 53 % round trip against the per-direction theory 1 − 0.7² = 51 % (the same arithmetic as `packet_loss 30`); `start = 0` is stored as an *active* filter; the window coexists with netem (`delay 50` + a permanent window = 100 % inside the outage); clearing removes the classifier; a node restart restores the schedule on the fresh veth; `available_filters` shows window_drop on kernel links and a relay link both hides it and 409s it; teardown leaves no filter or qdisc behind. The P6b e2e was re-run as a regression on the same uBridge build (13/13 — nth/quota unaffected by the de-looping below). -In-repo live coverage: `tests/e2e/test_docker_kernel_datapath.py` (pytest -marker `e2e`) runs two real Alpine containers on an isolated instance — the -per-link bridge enslaving exactly the two veth host ends, real ICMP, the -L2-anchor spec §E assertions (no L3 identity, forwarding ports, idle -silence with the guests' links down), the filter matrix one type at a time -(netem core and extensions, cls_bpf match-drop, the eBPF classifier, each -asserted through real traffic), suspend, capture, link delete/re-create, -container stop/start, a relay negative control — whose own filters -(delay/frequency_drop/bpf) really shape the relay wire, whose capture and -markers ride the same relay engine (`bridge start_capture`, the bridge -`mark` filter) — and the relay→kernel upgrade on project reopen (the -datapath choice is flipped across a server restart). The container image is digest-pinned and pulled through the -server's own pull route on a cache miss, so collaborators test the same -bytes. +In-repo live coverage: `tests/e2e/test_docker_kernel_datapath.py` (pytest marker `e2e`) runs two real Alpine containers on an isolated instance — the per-link bridge enslaving exactly the two veth host ends, real ICMP, the L2-anchor spec §E assertions (no L3 identity, forwarding ports, idle silence with the guests' links down), the filter matrix one type at a time (netem core and extensions, cls_bpf match-drop, the eBPF classifier, each asserted through real traffic), suspend, capture, link delete/re-create, container stop/start, a relay negative control — whose own filters (delay/frequency_drop/bpf) really shape the relay wire, whose capture and markers ride the same relay engine (`bridge start_capture`, the bridge `mark` filter) — and the relay→kernel upgrade on project reopen (the datapath choice is flipped across a server restart). The container image is digest-pinned and pulled through the server's own pull route on a cache miss, so collaborators test the same bytes. -The uBridge-side window work took two fixes that are worth recording, both -found by *this* integration rather than by unit tests: the delivered -catch-up loop tripped the verifier's 8192 **jump-sequence** budget — on the -non-root path the verifier drops the loop counter's bound -(`R4=scalar(smax=umax32=0xfffff086)`), treats the loop as unbounded and -unrolls it until the pending-state limit, so the program was rejected -wholesale (`E2BIG`, `The sequence of 8193 jumps is too complex`) and -`tc capabilities` reported `ebpf=0` — taking `frequency_drop` and `quota` -down with it, since the classifier loads as one program; lowering the trip -cap did not help (128 trips failed identically), removing the loop did -(572ca0c: 16-step unroll + outer guard, zero back edges). A latent Makefile -header-dependency gap (53e34bb) had been shipping the *old* truncated -program object — the kernel answered `jump out of range from insn 9 to -421` with `processed 0 insns`. Both matter to anyone rebuilding uBridge: -`ebpf=1` from a direct probe is the only trustworthy signal. +The uBridge-side window work took two fixes that are worth recording, both found by *this* integration rather than by unit tests: the delivered catch-up loop tripped the verifier's 8192 **jump-sequence** budget — on the non-root path the verifier drops the loop counter's bound (`R4=scalar(smax=umax32=0xfffff086)`), treats the loop as unbounded and unrolls it until the pending-state limit, so the program was rejected wholesale (`E2BIG`, `The sequence of 8193 jumps is too complex`) and `tc capabilities` reported `ebpf=0` — taking `frequency_drop` and `quota` down with it, since the classifier loads as one program; lowering the trip cap did not help (128 trips failed identically), removing the loop did (572ca0c: 16-step unroll + outer guard, zero back edges). A latent Makefile header-dependency gap (53e34bb) had been shipping the *old* truncated program object — the kernel answered `jump out of range from insn 9 to 421` with `processed 0 insns`. Both matter to anyone rebuilding uBridge: `ebpf=1` from a direct probe is the only trustworthy signal. From e04552cdb05c6bb151f60c91410b5f7c5ef0f1ba Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 15:14:09 +0800 Subject: [PATCH 071/101] =?UTF-8?q?docs(iol):=20IOL=20runner=20kernel=20da?= =?UTF-8?q?tapath=20=E2=80=94=20diagram=20suite=20and=20glossary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the ASCII overview with mermaid diagrams per the documentation standard: a glossary table first, then the two-container architecture (port bridges inside their uBridge process blocks, the per-link Linux bridge circled, dashed sidebands split by direction — frame copies flow to uBridge, qdisc installs to the anchor), a copy-counted relay diagram (six kernel/user crossings each way, datapath-invariant — swapping datapaths only changes the middle two), the frame path with the reserved-range diamond, and the link creation sequence. Details moved from the old bullets into per-diagram prose; paragraphs reflowed to one per line. --- docs/features/iol-docker-kernel-datapath.md | 336 +++++++++++--------- 1 file changed, 191 insertions(+), 145 deletions(-) diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index 5b7b29aab..4df8974a6 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -3,184 +3,234 @@ SPDX-License-Identifier: CC-BY-SA-4.0 See LICENSE file for licensing information. --> -> This documentation is AI-generated with reference to actual code and verified -> against real-kernel testing. AI can make mistakes — please verify against -> the source code when in doubt. +> This documentation is AI-generated with reference to actual code and verified against real-kernel testing. AI can make mistakes — please verify against the source code when in doubt. # IOL runner container kernel datapath ## Overview -IOL runner containers (`IOLDockerVM`, selected by the `GNS3_IOL_RUNNER` -environment marker — the Cisco CML `iol-runner` images such as -`iol-xe/iol-xe:17-18-02`) have no container-namespace networking at all: -their guest leg is the runner's per-interface AF_UNIX datagram socket pairs. -A unix socket is not a kernel interface, so until now every link of such a -container rode the uBridge userspace relay (unix ↔ UDP). +IOL runner containers (`IOLDockerVM`, selected by the `GNS3_IOL_RUNNER` environment marker — the Cisco CML `iol-runner` images such as `iol-xe/iol-xe:17-18-02`) have no container-namespace networking at all: their guest leg is the runner's per-interface AF_UNIX datagram socket pairs. A unix socket is not a kernel interface, so until now every link of such a container rode the uBridge userspace relay (unix ↔ UDP). -The kernel datapath gives every Ethernet bay/unit a **persistent TAP anchor** -(`gx` names, born at node start exactly like IOU's) and wires the *link -segment* through the kernel. One userspace hop on the container leg is -irreducible — the socket pair is the only physical layer the runner exposes — -but the anchor → per-link bridge → peer anchor crossing, the kernel filters, -the suspend carrier and the AF_PACKET capture/markers are all native now. +The kernel datapath gives every Ethernet bay/unit a **persistent TAP anchor** (`gx` names, born at node start exactly like IOU's) and wires the *link segment* through the kernel. One userspace hop on the container leg is irreducible — the socket pair is the only physical layer the runner exposes — but the anchor → per-link bridge → peer anchor crossing, the kernel filters, the suspend carrier and the AF_PACKET capture/markers are all native now. -``` - container netns (iol-runner) root namespace - ┌────────────┐ unix datagram ┌────────────────────────────────┐ - │ netiomux │◄─ sNN/cNN.sock ──►│ uBridge port bridge │ - │ (sNN/cNN) │ (host-side dir) │ [nio_unix, nio_tap] relay │ - └────────────┘ │ │ holds the TAP fd │ - │ ▼ │ - │ gx{id}e{bay}p{unit} ─────────┼─► gns3{link_id} - │ (persistent TAP anchor) │ (per-link kernel - └────────────────────────────────┘ bridge port) +The port bridge is the deployment shape ubridge's `doc/gns3server-integration.md` § *bridge — generic NIO relay with a swappable TAP leg* froze: a per-node bridge created at start, the unix NIO in the first slot for the node's whole life, the topology leg in the second — `add_nio_udp` (relay) or `add_nio_tap` (kernel link) — swapped as links are attached, suspended, deleted or the project reopens, through stop → delete → add → start. The c-socket binding never re-binds across a swap: frames the container sends while the bridge is stopped queue on the socket and relay after `start` (regression-tested in ubridge's bridge suite). + +## Architecture + +Kernel link between two IOL runner containers on the same compute. Interface names follow the shared deterministic rules — `gx{node_id[:8]}e{bay}p{unit}` for the persistent TAP anchors (one per Ethernet bay/unit) and `gns3{link_id[:11]}` for the bridge; the concrete names in the diagrams use node ids `1a2b3c4d` / `99aa88bb`, Ethernet0/0 on both, and link id prefix `9a8b7c6d5e4`. Terms the diagrams use: + +| Term | Meaning | +|---|---| +| netiomux | the runner's network agent: one AF_UNIX datagram socket pair per interface, carrying raw Ethernet frames | +| `sNN.sock` / `cNN.sock` | the interface's socket pair (`NN = bay×4 + unit`): frames sent to `s` are injected into the guest, the guest's egress arrives on `c` | +| wiring dir | the host-side directory bind-mounted at the container's `/tmp` (`$XDG_RUNTIME_DIR/gns3/unixio/`) — the same socket files on both sides, short enough for the 107-byte `sun_path` cap | +| uBridge | the per-node helper process gns3-server spawns; its hypervisor command surface executes all host-side wiring (anchors, bridges, carrier, tc, taps) — and, unique to this node type, it also carries frames: the socket guest leg has no kernel form | +| port bridge | the per-bay/unit uBridge bridge (`bridge{bay}_{unit}`): `nio_unix` in its first slot for the node's whole life, the topology leg — `nio_tap` (kernel link) or `nio_udp` (relay) — in the second | +| anchor | the persistent TAP `gx…` a link attaches to: born at node start, DOWN, its fd held by the port bridge while a kernel link binds the port | +| per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | +| bridge port | an interface enslaved to a Linux bridge (`brctl addif`) — it no longer sends or receives on its own: frames arriving on it go to the bridge's forwarding logic, and the bridge transmits through it | +| FDB | the bridge's forwarding database (learned MAC → port); unknown, broadcast and multicast destinations are flooded | +| `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | +| tc / qdisc | traffic-control objects the kernel inserts into the anchor's egress path — stages frames pass through in flight, not external packet consumers; uBridge only installs them over netlink | +| `clsact` | the tc classifier carrier on the anchor; its egress chain is where the drop classifiers run | +| eBPF classifier | the stateful drop modes (every-Nth, quota, time window) at clsact egress prio 1 | +| `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | +| netem | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | +| `AF_PACKET tap` | uBridge's packet socket bound to the anchor; the kernel clones every frame the interface sends or receives to it (the tcpdump mechanism) — read-only copies for capture and markers, the original stays on the kernel path | +| carrier | the anchor's admin state (`link set up/down`); suspend is carrier down on both anchors = 100 % loss | +| `nio_bridge` | the NIO record the controller sends each compute: bridge name, filters, markers, suspend flag | + +```mermaid +flowchart TB + subgraph NetA["container A netns (iol-runner)"] + IOLA["IOL process"] + MUXA["netiomux"] + IOLA <-->|"Eth0/0"| MUXA + end + subgraph RootNS["root namespace"] + subgraph UBA["uBridge A process"] + PBA["port bridge A"] + end + gxA["gx1a2b3c4de0p0 — anchor"] + subgraph LBR["per-link Linux bridge"] + BR["gns39a8b7c6d5e4"] + end + gxB["gx99aa88bbe0p0 — anchor"] + subgraph UBB["uBridge B process"] + PBB["port bridge B"] + end + end + subgraph NetB["container B netns (iol-runner)"] + MUXB["netiomux"] + IOLB["IOL process"] + MUXB <-->|"Eth0/0"| IOLB + end + MUXA <-->|"nio_unix"| PBA + PBA <-->|"nio_tap"| gxA + gxA <-->|"bridge port"| BR + BR <-->|"bridge port"| gxB + gxB <-->|"nio_tap"| PBB + PBB <-->|"nio_unix"| MUXB + PBA -. "qdisc install" .-> gxA + gxA -. "frame copies" .-> PBA + PBB -. "qdisc install" .-> gxB + gxB -. "frame copies" .-> PBB ``` -The port bridge is the deployment shape ubridge's -`doc/gns3server-integration.md` § *bridge — generic NIO relay with a -swappable TAP leg* froze: a per-node bridge created at start, the unix NIO in -the first slot for the node's whole life, the topology leg in the second — -`add_nio_udp` (relay) or `add_nio_tap` (kernel link) — swapped as links are -attached, suspended, deleted or the project reopens, through -stop → delete → add → start. The c-socket binding never re-binds across a -swap: frames the container sends while the bridge is stopped queue on the -socket and relay after `start` (regression-tested in ubridge's bridge suite). +Everything in the root namespace is built the same way: gns3-server creates the anchor TAPs at node start, the per-link bridge and its memberships at link creation, and every later change (carrier, capture, markers, filters) by sending hypervisor commands to the node's uBridge process. That build flow is the Link operations sequence below; this diagram is the steady state that remains. + +The solid chain is the frame path, and it passes through the port bridges — the one userspace hop per container leg the socket pair forces, and the only one on the whole path; unlike the Docker veth datapath, where uBridge sits entirely off the forwarding path, here it is on it. The dashed edges are control and observation: `qdisc install` is the port bridge configuring the kernel over netlink (the kernel then runs those qdiscs in-flight on the anchor's egress — stages of the transmit path, not packet consumers), and `frame copies` is the AF_PACKET tap — read-only clones for capture and markers. Everything sideband is keyed on the anchor's interface name, which is why a TAP anchor and a veth host end are interchangeable in the shared mixin (`gns3server/compute/kernel_datapath.py`). + +Relay link (what a missing capability, a cross-compute peer or `enable_kernel_datapath = off` falls back to — same sockets, same port bridge, only the topology leg changes): + +```mermaid +flowchart LR + subgraph CTA["container A (iol-runner)"] + MUXA["netiomux A"] + end + subgraph KRN["kernel — sockets & loopback"] + SOCKA["c00.sock"] + UDP["UDP sockets (loopback)"] + SOCKB["s00.sock"] + ANCH["gx anchors — idle"] + end + subgraph UBA["uBridge A — userspace"] + UNA["unix nio"] + ENA["port bridge relay + userspace filters"] + NDA["udp nio"] + end + subgraph UBB["uBridge B — userspace"] + NDB["udp nio"] + ENB["port bridge relay + userspace filters"] + UNB["unix nio"] + end + subgraph CTB["container B (iol-runner)"] + MUXB["netiomux B"] + end + MUXA -->|"copy 1: user→kernel"| SOCKA + SOCKA -->|"copy 2: kernel→user"| UNA + UNA --> ENA --> NDA + NDA -->|"copy 3: user→kernel"| UDP + UDP -->|"copy 4: kernel→user"| NDB + NDB --> ENB --> UNB + UNB -->|"copy 5: user→kernel"| SOCKB + SOCKB -->|"copy 6: kernel→user"| MUXB +``` + +Drawn one-way; the reverse direction is the mirror image. The IOL relay crosses the kernel/user boundary **six times each way** — but four of them are the socket guest leg itself, paid identically on the kernel datapath (six there too): unlike Docker's four-against-zero, the crossing count is datapath-invariant here, and the choice swaps only the middle two — loopback UDP between the port bridges (filters userspace, at them) for the per-link bridge between the anchors (filters as tc, on them). + +Two further contrasts with Docker's relay: the IOL relay's local leg is the unix NIO itself, so the anchors never appear on the path (idle from birth — a Docker relay attaches to the *same* veth host end the kernel link enslaves); and the port bridge's first slot is shared with the kernel datapath — the unix NIO and its c-socket binding survive every leg swap, so frames the container sends while the bridge is stopped for a swap queue on the socket and relay after `start`. On this datapath suspend rides the synthetic frequency_drop filter (the kernel datapath's suspend is the anchor's admin state). + +Frame path for one A → B frame. The impairment chain fires on the egress of the *destination* anchor, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gxA — hence `delay 100` doubles the RTT (measured ≥ 150 ms): + +```mermaid +flowchart LR + S1["IOL A sends on Eth0/0"] --> S2["netiomux writes c00.sock"] + S2 --> S3["port bridge A relay"] + S3 --> S4["gxA ingress"] + S4 --> G1{"reserved 01:80:c2?"} + G1 -->|"PAUSE 0x01"| D["dropped"] + G1 -->|"0xfffd passes"| S5["FDB lookup / flood"] + S5 --> S6["clsact: eBPF prio 1"] + S6 --> S7["clsact: bpf_drop prio 10-99"] + S7 --> S8["netem qdisc"] + S8 --> S9["port bridge B relay"] + S9 --> S10["netiomux B via s00.sock"] + S10 --> S11["IOL B receives on Eth0/0"] +``` + +The reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The AF_PACKET taps observe both anchors but never a clsact-dropped frame. ## Datapath selection -`UDPLink._kernel_datapath_eligible` treats an IOL runner container like any -Docker node (same compute, Ethernet port) with one gate of its own in -`_kernel_endpoint_ready`: the compute must report both +`UDPLink._kernel_datapath_eligible` treats an IOL runner container like any Docker node (same compute, Ethernet port) with one gate of its own in `_kernel_endpoint_ready`: the compute must report both -* `ubridge_tap` — the tap module (`tap create`/`tap delete`), the same - capability QEMU and Dynamips gate on, and -* `ubridge_bridge_tap` — `bridge delete_nio_tap`, the swappable-leg command - (probed with one command against a bridge that cannot exist: 202 = old - build, 214 = new; no scratch objects, no capabilities, cached by binary - identity like every other probe). +* `ubridge_tap` — the tap module (`tap create`/`tap delete`), the same capability QEMU and Dynamips gate on, and +* `ubridge_bridge_tap` — `bridge delete_nio_tap`, the swappable-leg command (probed with one command against a bridge that cannot exist: 202 = old build, 214 = new; no scratch objects, no capabilities, cached by binary identity like every other probe). -Both strict `True`; anything unreported (old uBridge, failed probe) keeps the -link on the relay, where it always works. The generic `GNS3_UNIX_SOCKET_NIO` -containers stay excluded as before — that marker's semantics are -image-specific, while `GNS3_IOL_RUNNER` selects a class whose anchors this -server owns. +Both strict `True`; anything unreported (old uBridge, failed probe) keeps the link on the relay, where it always works. The generic `GNS3_UNIX_SOCKET_NIO` containers stay excluded as before — that marker's semantics are image-specific, while `GNS3_IOL_RUNNER` selects a class whose anchors this server owns. ## Anchor lifecycle -* **Birth (node start, before any link)** — `_start_ubridge` runs - `_prepare_tap_datapath` right after the hypervisor connects: probe both - capabilities, then for every (bay, unit 0-3) sweep a leftover - (`tap delete`, suppressed), `tap create` (hardened, starts DOWN) and - `link set down` — carrier off until a link attaches. The anchors exist - whether or not any link ever uses them: an anchor born with its link would - leave the Ethernet switch fast path's deferred join waiting forever. - No `tap set_owner`: uBridge itself holds these fds. -* **Life** — the anchor is never recreated. Link create/delete/switch, - suspend, capture and markers only change what is attached to it, and the - ensure-then-add contract guards the one window where the device could have - been swept (`/sys/class/net` existence check — `tap create` is strictly - create-only, `IFF_TUN_EXCL`, and refuses an existing device with EBUSY). -* **Death (node stop)** — the port bridges hold the anchor fds - (`bridge add_nio_tap`), so they are deleted first (`bridge delete` stops - their relay threads and frees the NIOs), then the anchors themselves - (`tap delete` answers EBADFD on a device another fd still holds), then the - per-link kernel bridges — all while the control channel lives, before the - hypervisor stops. A restart recreates everything from the NIO. +* **Birth (node start, before any link)** — `_start_ubridge` runs `_prepare_tap_datapath` right after the hypervisor connects: probe both capabilities, then for every (bay, unit 0-3) sweep a leftover (`tap delete`, suppressed), `tap create` (hardened, starts DOWN) and `link set down` — carrier off until a link attaches. The anchors exist whether or not any link ever uses them: an anchor born with its link would leave the Ethernet switch fast path's deferred join waiting forever. No `tap set_owner`: uBridge itself holds these fds. +* **Life** — the anchor is never recreated. Link create/delete/switch, suspend, capture and markers only change what is attached to it, and the ensure-then-add contract guards the one window where the device could have been swept (`/sys/class/net` existence check — `tap create` is strictly create-only, `IFF_TUN_EXCL`, and refuses an existing device with EBUSY). +* **Death (node stop)** — the port bridges hold the anchor fds (`bridge add_nio_tap`), so they are deleted first (`bridge delete` stops their relay threads and frees the NIOs), then the anchors themselves (`tap delete` answers EBADFD on a device another fd still holds), then the per-link kernel bridges — all while the control channel lives, before the hypervisor stops. A restart recreates everything from the NIO. ## Link operations -* **Kernel link create** (`_attach_kernel_link`): ensure the port bridge and - its unix NIO → ensure the anchor → `bridge add_nio_tap` (uBridge opens and - holds the fd) → the shared mixin flow (`brctl create`/`addif` the per-link - bridge, capture, markers, tc netem/cls_bpf/eBPF filters on the anchor) → - `bridge start` (the [unix ↔ tap] relay) → the carrier pass (which is also - what brings a born-down anchor up; a suspended NIO sets it back down). -* **Link delete** (`_release_port_tap`): `bridge stop` (delete_nio_tap - refuses while running — the relay threads hold the NIO pointers for their - whole life; freeing under them is a use-after-free) → - `bridge delete_nio_tap ""` (matched on the kernel-resolved - name) → the mixin teardown (markers, tc reset, `brctl delif`, per-link - bridge deletion). The anchor survives — the node owns it, not the link. -* **Relay link** — unchanged: the port bridge's second leg is `add_nio_udp`, - userspace filters at the port bridge. The unix NIO stays in its slot - across every datapath switch. +Both endpoints attach concurrently and compute the same bridge name independently (a pure function of the link id), so either may win the `brctl create` race — the loser verifies the bridge exists (`brctl show`) and continues: + +```mermaid +sequenceDiagram + participant API as REST API + participant C as Controller + participant CA as IOL node A + participant CB as IOL node B + participant UA as uBridge A + participant UB as uBridge B + + API->>C: POST /links + C->>C: eligibility check + C->>C: bridge = gns3 + link_id[:11] + C--)CA: nio_bridge + C--)CB: nio_bridge (same bridge) + par IOL node A + CA->>UA: ensure port bridge + CA->>UA: ensure anchor + CA->>UA: bridge add_nio_tap + CA->>UA: brctl create + CA->>UA: link set bridge up + CA->>UA: brctl addif gxA + CA->>UA: port fwd_mask 0xfffd + CA->>UA: markers / capture if any + CA->>UA: tc netem + bpf + eBPF + CA->>UA: bridge start + CA->>UA: link set gxA up + and IOL node B + CB->>UB: same on gxB + end + Note over UA,UB: steady state — kernel forwards between the anchors, port bridges relay to the containers +``` + +Every arrow to uBridge is one `_ubridge_send` command line on the node's local hypervisor console; uBridge executes it (netlink/ioctl) and replies OK or an error synchronously before the next command is sent — those replies are omitted for brevity. uBridge never initiates anything: it is a pure executor, and the only data ever flowing back (capture/marker frame copies) rides separate AF_PACKET sockets, not this console. + +The attach above is `_attach_kernel_link`; its ensures are the frozen ensure-then-add contract: the port bridge (`bridge create` + `add_nio_unix`) comes from the node's start loop or is created here, and `tap create` runs only when `/sys/class/net` shows the anchor missing — it is strictly create-only (`IFF_TUN_EXCL`) and answers EBUSY on the device the start loop just made. `bridge add_nio_tap` is what opens and holds the anchor fd, `bridge start` starts the [unix ↔ tap] relay, and the final `link set up` is the carrier pass that brings the born-down anchor up (a suspended NIO sets it back down). + +* **Link delete** (`_release_port_tap`): `bridge stop` (delete_nio_tap refuses while running — the relay threads hold the NIO pointers for their whole life; freeing under them is a use-after-free) → `bridge delete_nio_tap ""` (matched on the kernel-resolved name) → the mixin teardown (markers, tc reset, `brctl delif`, per-link bridge deletion). The anchor survives — the node owns it, not the link. +* **Relay link** — unchanged: the port bridge's second leg is `add_nio_udp`, userspace filters at the port bridge. The unix NIO stays in its slot across every datapath switch. ## Capture, markers, filters -Inherited from `KernelDatapathMixin` unchanged — everything is keyed on the -anchor interface name, which is why a TAP and a veth host end are -interchangeable there: `capture start_kernel` / `marker add_kernel` -(AF_PACKET), one tc netem qdisc (the full netem surface plus the -extensions), `bpf` as cls_bpf match-drop, `frequency_drop`/`quota`/ -`window_drop` as the eBPF stateful classifier — all capability-gated per -compute exactly like the Docker veth datapath. +Inherited from `KernelDatapathMixin` unchanged — everything is keyed on the anchor interface name, which is why a TAP and a veth host end are interchangeable there: `capture start_kernel` / `marker add_kernel` (AF_PACKET), one tc netem qdisc (the full netem surface plus the extensions), `bpf` as cls_bpf match-drop, `frequency_drop`/`quota`/`window_drop` as the eBPF stateful classifier — all capability-gated per compute exactly like the Docker veth datapath (the filter semantics live in `docker-kernel-datapath.md`). -Suspend semantics: anchor admin-down. The port bridge's TAP writes fail EIO -(tolerated by the delivered uBridge hardening) and its reads fall silent — -traffic stops both ways. The runner itself is unaware of the link state -(netiomux has no carrier signal), exactly as on the relay datapath where -suspend rode the synthetic frequency_drop filter instead. +Suspend semantics: anchor admin-down. The port bridge's TAP writes fail EIO (tolerated by the delivered uBridge hardening) and its reads fall silent — traffic stops both ways. The runner itself is unaware of the link state (netiomux has no carrier signal), exactly as on the relay datapath where suspend rode the synthetic frequency_drop filter instead. ## Link-local frames (LACP, LLDP, 802.1X, STP) -The bridge stands in for a cable, but the kernel's `br_handle_frame()` does -not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by -default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while -the UDP relay carries them fine. uBridge therefore opens every port's -per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at -`brctl addif` time (Linux 4.15+, best-effort on older kernels). The -bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, -so LACP is reachable per port only. +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. -Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and -LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in -`br_handle_frame()` drops it unconditionally and no mask can enable it. -That is a documented limit, not a regression (PFC's hardware semantics are -out of emulation's reach anyway); a lab that needs PAUSE frames on the wire -must use a relay link. Verified live by -`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest -matrix); the contract and kernel references live in -`docs/design/ubridge-link-local-frame-forwarding-spec.md`. +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in `br_handle_frame()` drops it unconditionally and no mask can enable it. That is a documented limit, not a regression (PFC's hardware semantics are out of emulation's reach anyway); a lab that needs PAUSE frames on the wire must use a relay link. Verified live by `tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest matrix); the contract and kernel references live in `docs/design/ubridge-link-local-frame-forwarding-spec.md`. ## Relay fallback -Missing either capability, the node runs relay-only: links ride -unix ↔ UDP, `kernel_datapath` stays false, and anchors are simply never -created (or, with `enable_kernel_datapath` off server-wide, created but -never attached — the same semantics as QEMU's and Dynamips' anchors). +Missing either capability, the node runs relay-only: links ride unix ↔ UDP, `kernel_datapath` stays false, and anchors are simply never created (or, with `enable_kernel_datapath` off server-wide, created but never attached — the same semantics as QEMU's and Dynamips' anchors). ## Verified -Live on the real server (`tests/e2e/test_iol_docker_kernel_datapath.py`, -isolated instance, real `iol-xe/iol-xe:17-18-02` images, real uBridge with -`bridge delete_nio_tap`): +Live on the real server (`tests/e2e/test_iol_docker_kernel_datapath.py`, isolated instance, real `iol-xe/iol-xe:17-18-02` images, real uBridge with `bridge delete_nio_tap`): -* kernel link reports `kernel_datapath`, anchors exist from node start, the - per-link bridge enslaves exactly the two of them; +* kernel link reports `kernel_datapath`, anchors exist from node start, the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [unix ↔ tap] port-bridge relay (100 % ping); -* the L2-anchor spec §E.2 silence window with the guests shut — the - behavioral half of the hardening on anchors created by uBridge's bridge - TAP module (`bridge add_nio_tap`); -* `delay 100` lands as netem on both anchors, measured RTT grows ≥ 150 ms - and returns < 50 ms when cleared; -* the classifier spot check on a TAP anchor (the Docker suite runs the full - matrix on veth host ends): `bpf "icmp"` drops everything (clsact on the - anchor), `frequency_drop 3` as the eBPF every-nth mode measured 56 % - round-trip loss (theory 55.6 %), both restoring on clear; +* the L2-anchor spec §E.2 silence window with the guests shut — the behavioral half of the hardening on anchors created by uBridge's bridge TAP module (`bridge add_nio_tap`); +* `delay 100` lands as netem on both anchors, measured RTT grows ≥ 150 ms and returns < 50 ms when cleared; +* the classifier spot check on a TAP anchor (the Docker suite runs the full matrix on veth host ends): `bpf "icmp"` drops everything (clsact on the anchor), `frequency_drop 3` as the eBPF every-nth mode measured 56 % round-trip loss (theory 55.6 %), both restoring on clear; * suspend admin-downs the anchor and kills the traffic; resume restores; * AF_PACKET capture writes a real pcap of the ICMP exchange; -* link delete/re-create swaps the port bridge's TAP leg out and back in - (unix binding never re-binds) with the ping returning; -* node stop tears the anchors down (no `gx` devices left), a restart - recreates the whole wiring from the NIO; -* project delete leaves no anchors and no bridges; the relay control on a - relay-configured instance keeps the link off the kernel and still pings. +* link delete/re-create swaps the port bridge's TAP leg out and back in (unix binding never re-binds) with the ping returning; +* node stop tears the anchors down (no `gx` devices left), a restart recreates the whole wiring from the NIO; +* project delete leaves no anchors and no bridges; the relay control on a relay-configured instance keeps the link off the kernel and still pings. -Also caught and fixed live: `tap create` is strictly create-only -(`IFF_TUN_EXCL`) — the ensure-then-add guard must check existence first -(`/sys/class/net`), not re-create unconditionally (EBUSY on the device the -start loop just made). +Also caught and fixed live: `tap create` is strictly create-only (`IFF_TUN_EXCL`) — the ensure-then-add guard must check existence first (`/sys/class/net`), not re-create unconditionally (EBUSY on the device the start loop just made). ## Configuration @@ -189,12 +239,8 @@ start loop just made). enable_kernel_datapath = True # the global switch (link attachment) ``` -The anchor lifecycle needs no configuration of its own — the capability -probes decide per compute, and an IOL runner container without them simply -never anchors. +The anchor lifecycle needs no configuration of its own — the capability probes decide per compute, and an IOL runner container without them simply never anchors. ## Roadmap -Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until -every node type is kernelized; with the IOL runner container anchored, that -condition is one project away from met. +Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized; with the IOL runner container anchored, that condition is one project away from met. From 8bca2c70f8d2546e5210fcec14c56aa0d6b5bf5e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 15:48:09 +0800 Subject: [PATCH 072/101] docs: qemu kernel datapath doc in the family diagram style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Term table, QEMU↔QEMU topology with role-suffixed anchors and frame-copies/qdisc-install dashed edges, relay copy-count diagram, boxed frame path, link creation sequence; prose reflowed one paragraph per line. --- docs/features/qemu-kernel-datapath.md | 349 +++++++++++++++----------- 1 file changed, 205 insertions(+), 144 deletions(-) diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 29f340a66..84dc8bf5f 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -3,43 +3,163 @@ SPDX-License-Identifier: CC-BY-SA-4.0 See LICENSE file for licensing information. --> +> This documentation is AI-generated with reference to actual code and verified against real-kernel testing. AI can make mistakes — please verify against the source code when in doubt. + # QEMU kernel datapath -QEMU adapters anchor on a **persistent TAP**, and links between two QEMU -nodes — or a QEMU node and a Docker container — on the same compute are wired -in the kernel (per-link Linux bridge) instead of the uBridge UDP relay. The -compute side is the same machinery Docker already uses: `KernelDatapathMixin` -(`gns3server/compute/kernel_datapath.py`) takes an *anchor interface* and does -not care whether it is a veth host end or a TAP. +QEMU adapters anchor on a **persistent TAP**, and links between two QEMU nodes — or a QEMU node and a Docker container — on the same compute are wired in the kernel (per-link Linux bridge) instead of the uBridge UDP relay. The compute side is the same machinery Docker already uses: `KernelDatapathMixin` (`gns3server/compute/kernel_datapath.py`) takes an *anchor interface* and does not care whether it is a veth host end or a TAP. ## Architecture -Kernel link between two QEMU nodes on one compute: +Kernel link between two QEMU nodes on the same compute. Interface names follow the shared contract of `gns3server/utils/kernel_anchor.py`: `gq{node_id[:8]}e{adapter}p{port}` for the persistent TAP (one per QEMU adapter, port always 0) and `gns3{link_id[:11]}` for the bridge; the concrete names in the diagrams use node ids `3f2a1b9c` / `7c9d4e2f` and link id prefix `5d3c2b1a4e6`. A mixed Docker↔QEMU link is the same bridge with the Docker end's `gv` veth host end as its other anchor — the shared mixin is anchor-agnostic; the pure-veth picture is `docker-kernel-datapath.md`. Terms the diagrams use: -``` - QEMU A (fd) root namespace QEMU B (fd) - ┌──────────┐ -netdev ┌──────────────────┐ -netdev ┌──────────┐ - │ e1000 │◄───tap──────┤ gq{idA}e0p0 ────┼──► gns3{link_id_prefix} - └──────────┘ │ (bridge port) │ (bridge port) │ - │ per-link kernel │ gq{idB}e0p0 │ - │ bridge = "cable"│ │ - └──────────────────┘ +| Term | Meaning | +|---|---| +| uBridge | the per-node helper process gns3-server spawns with elevated capabilities; its hypervisor command surface executes all host-side wiring (TAPs, bridges, carrier, tc, taps) — on a kernel link it builds and observes the path but never carries frames | +| anchor | the host-side interface a link attaches to — here the `gq` persistent TAP | +| persistent TAP | the TAP uBridge creates per QEMU adapter at node start (`tap create`, `tap set_owner`); it outlives restarts and crashes, and QEMU opens it by name as its `-netdev tap` | +| per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | +| bridge port | an interface enslaved to a Linux bridge (`brctl addif`) — it no longer sends or receives on its own: frames arriving on it go to the bridge's forwarding logic, and the bridge transmits through it | +| tap fd | a TAP's file descriptor: reads return frames the kernel received on the device, writes inject frames into it — on this datapath QEMU itself holds it | +| FDB | the bridge's forwarding database (learned MAC → port); unknown, broadcast and multicast destinations are flooded | +| `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | +| tc / qdisc | traffic-control objects the kernel inserts into the anchor's egress path — stages frames pass through in flight, not external packet consumers; uBridge only installs them over netlink | +| `clsact` | the tc classifier carrier on the anchor; its egress chain is where the drop classifiers run | +| eBPF classifier | the stateful drop modes (every-Nth, quota, time window) at clsact egress prio 1 | +| `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | +| netem | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | +| `AF_PACKET tap` | uBridge's packet socket bound to the anchor; the kernel clones every frame the interface sends or receives to it (the tcpdump mechanism) — read-only copies for capture and markers, the original stays on the kernel path | +| carrier | the anchor's admin state (`link set up/down`); suspend is carrier down plus QMP `set_link off` so the guest notices | +| QMP | the QEMU machine protocol — how the server drives a running QEMU (link state on suspend/resume) without restarting it | +| `nio_bridge` | the NIO record the controller posts to each endpoint node: bridge name, filters, markers, suspend flag | + +```mermaid +flowchart LR + subgraph GA["guest A (in the VM)"] + nA["NIC 0"] + end + subgraph RootNS["root namespace"] + gqA["gq3f2a1b9ce0p0 — anchor"] + BR["gns35d3c2b1a4e6 — per-link bridge"] + gqB["gq7c9d4e2fe0p0 — anchor"] + UBA["uBridge A"] + UBB["uBridge B"] + end + subgraph QPA["QEMU A process"] + ndA["netdev gns3-0 (tap fd)"] + end + subgraph QPB["QEMU B process"] + ndB["netdev gns3-0 (tap fd)"] + end + subgraph GB["guest B (in the VM)"] + nB["NIC 0"] + end + nA <-->|"virtio / e1000"| ndA + ndA <-->|"tap fd"| gqA + gqA <-->|"bridge port"| BR + BR <-->|"bridge port"| gqB + gqB <-->|"tap fd"| ndB + ndB <-->|"virtio / e1000"| nB + gqA -. "frame copies" .-> UBA + UBA -. "qdisc install" .-> gqA + gqB -. "frame copies" .-> UBB + UBB -. "qdisc install" .-> gqB ``` -Relay link on the same anchor (cross-compute, mixed node types, or -`enable_kernel_datapath` off): +Everything in the root namespace is built the same way: gns3-server creates the persistent TAPs at node start (before QEMU is launched), the per-link bridge and its memberships at link creation, and every later change (carrier, capture, markers, filters) by sending hypervisor commands to the node's uBridge process — the privileged agent that turns them into netlink/ioctl operations. That build flow is the Link creation sequence below; this diagram is the steady state that remains, where the only uBridge relationships left are the two dashed kinds. -``` - ┌──────────┐ tap ┌────────────────────────────┐ UDP ┌──────────┐ - │ QEMU A │◄────────►│ uBridge relay bridge │◄───────►│ peer │ - │ (fd) │ gq…e0p0 │ add_nio_ethernet ↔ add_ │ tunnel │ (any) │ - └──────────┘ │ nio_udp │ └──────────┘ - └────────────────────────────┘ +uBridge sits off the forwarding path — the dashed edges are control and observation, never data flow. The `qdisc install` edge is uBridge configuring the kernel over netlink; the kernel then runs those qdiscs in-flight on the anchor's egress (stages of the transmit path, not packet consumers). The `frame copies` edge is the AF_PACKET tap: the kernel clones every frame the TAP sends or receives to uBridge's packet socket (the tcpdump mechanism) — read-only copies for capture and markers; the originals never leave the kernel. The only userspace on the path is the two QEMU processes pumping their own tap fds — the irreducible VM leg, one fd crossing at each end per direction. + +Relay link between the same two QEMU nodes on the same single compute — what every link becomes when `enable_kernel_datapath` is off (mixed node types and cross-compute wiring fall back too; there the UDP hop rides the real network instead of loopback, the copy count unchanged): + +```mermaid +flowchart LR + subgraph GA["guest A (in the VM)"] + EA["NIC sends"] + end + subgraph QA["QEMU A — userspace"] + NDA["netdev tap fd"] + end + subgraph KERN["kernel — root ns"] + TA["gq3f2a1b9ce0p0 — anchor"] + UDP["UDP sockets (loopback)"] + TB["gq7c9d4e2fe0p0 — anchor"] + end + subgraph UBA["uBridge A — userspace"] + NEA["AF_PACKET nio"] + ENA["relay + userspace filters"] + NUA["UDP nio"] + end + subgraph UBB["uBridge B — userspace"] + NUB["UDP nio"] + ENB["relay + userspace filters"] + NEB["AF_PACKET nio"] + end + subgraph QB["QEMU B — userspace"] + NDB["netdev tap fd"] + end + subgraph GB["guest B (in the VM)"] + EB["NIC receives"] + end + EA -->|"virtio"| NDA + NDA -->|"copy 1: user→kernel"| TA + TA -->|"copy 2: kernel→user"| NEA + NEA --> ENA --> NUA + NUA -->|"copy 3: user→kernel"| UDP + UDP -->|"copy 4: kernel→user"| NUB + NUB --> ENB --> NEB + NEB -->|"copy 5: user→kernel"| TB + TB -->|"copy 6: kernel→user"| NDB + NDB -->|"virtio IRQ"| EB ``` -The relay attaches the TAP with `bridge add_nio_ethernet` (AF_PACKET, which -does not take the device over) — **never** `bridge add_nio_tap`, which would -claim the fd QEMU itself holds. +Drawn one-way; the reverse direction is the mirror image. A relayed frame crosses the kernel/user boundary **six times each way** (twelve per round trip): the two halves of the irreducible VM leg — QEMU's tap-fd write at the sending end, its read at the receiving end — plus the four the relay adds around them (AF_PACKET read, UDP send, UDP receive, AF_PACKET inject). The kernel datapath's only crossings are the VM leg's two; the relay's filters run in userspace, on the frames between the copies. + +The relay attaches the TAP with `bridge add_nio_ethernet` (AF_PACKET, which does not take the device over) — **never** `bridge add_nio_tap`, which would claim the fd QEMU itself holds. And unlike the IOL runner container, whose anchors idle on the relay, the anchor here is the netdev itself: it stays up and carries as the relay's AF_PACKET endpoint — QEMU has no other leg (the legacy socket-netdev fallback below is the one shape without the TAP). + +Frame path for one A → B frame. The impairment chain fires on the egress of the *destination* end's TAP, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gqA — hence `delay 100` ≈200 ms round trip (measured 100.3 ms one-way). Each one-way trip crosses the kernel/user boundary exactly twice — the sending QEMU's tap-fd write and the receiving QEMU's tap-fd read, the irreducible VM leg: + +```mermaid +flowchart LR + subgraph ANS["guest A (in the VM)"] + S1["NIC sends"] + end + subgraph QPA["QEMU A — userspace"] + S2["tap fd to kernel"] + end + subgraph ROOT["root namespace"] + subgraph GQA["gqA — ingress (its egress chain serves B→A)"] + S3["ingress"] + end + subgraph BRP["per-link bridge"] + G1{"reserved 01:80:c2?"} + D["dropped"] + S4["FDB lookup / flood"] + end + subgraph GQB["gqB — egress chain (clsact, then netem)"] + S5["eBPF prio 1"] + S6["bpf_drop prio 10-99"] + S7["netem"] + end + end + subgraph QPB["QEMU B — userspace"] + S8["tap fd to QEMU"] + end + subgraph BNS["guest B (in the VM)"] + S9["NIC receives"] + end + S1 -->|"virtio"| S2 + S2 --> S3 + S3 --> G1 + G1 -->|"PAUSE 0x01"| D + G1 -->|"0xfffd passes"| S4 + S4 --> S5 + S5 --> S6 --> S7 + S7 --> S8 + S8 -->|"virtio IRQ"| S9 +``` + +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The AF_PACKET taps observe both anchors but never a clsact-dropped frame. ## Why a TAP works like a veth host end @@ -50,48 +170,60 @@ claim the fd QEMU itself holds. | `tc` egress qdisc impairs | traffic entering the container | traffic entering the VM | | Capture/marker anchor | the interface | the same interface | -So every filter, capture and marker primitive carries over verbatim, and -`delay 100` on both anchors still measures ≈200 ms round trip while each -direction is impaired exactly once. +So every filter, capture and marker primitive carries over verbatim, and `delay 100` on both anchors still measures ≈200 ms round trip while each direction is impaired exactly once. ## Datapath selection -`UDPLink._kernel_datapath_eligible` needs both endpoints on one compute and -both able to anchor: +`UDPLink._kernel_datapath_eligible` needs both endpoints on one compute and both able to anchor: * Docker always can (adapters are born as veth pairs); -* QEMU can when that compute's uBridge reports the `tap` module — asked per - compute through `/capabilities` (`ubridge_tap`, probed by creating and - deleting one throwaway TAP, cached by binary identity), because an old - uBridge leaves QEMU on the legacy socket-netdev datapath and cannot anchor - a kernel link. An unknown/failed probe keeps the link on the relay. +* QEMU can when that compute's uBridge reports the `tap` module — asked per compute through `/capabilities` (`ubridge_tap`, probed by creating and deleting one throwaway TAP, cached by binary identity), because an old uBridge leaves QEMU on the legacy socket-netdev datapath and cannot anchor a kernel link. An unknown/failed probe keeps the link on the relay. -Because the `-netdev` type is fixed when QEMU is launched, this is -all-or-nothing per VM: on the TAP datapath *every* link can attach to (or -detach from) a running VM, including switching a link between the two -datapaths (delete + re-create, or a project reopen). +Because the `-netdev` type is fixed when QEMU is launched, this is all-or-nothing per VM: on the TAP datapath *every* link can attach to (or detach from) a running VM, including switching a link between the two datapaths (delete + re-create, or a project reopen). + +## Link creation sequence + +Both endpoints attach concurrently and compute the same bridge name independently (a pure function of the link id), so either may win the `brctl create` race — the loser verifies the bridge exists (`brctl show`) and continues. The topology above's link, at attach time: + +```mermaid +sequenceDiagram + participant API as REST API + participant C as Controller + participant CA as QEMU node A + participant CB as QEMU node B + participant UA as uBridge A + participant UB as uBridge B + + API->>C: POST /links + C->>C: eligibility check + C->>C: bridge = gns3 + link_id[:11] + C--)CA: nio_bridge + C--)CB: nio_bridge (same bridge) + par QEMU node A + CA->>UA: brctl create + CA->>UA: link set bridge up + CA->>UA: brctl addif gq3f2a1b9ce0p0 + CA->>UA: port fwd_mask 0xfffd + CA->>UA: markers / capture if any + CA->>UA: tc netem + bpf + eBPF + CA->>UA: link set gq3f2a1b9ce0p0 up + and QEMU node B + CB->>UB: same on gq7c9d4e2fe0p0 + end + Note over UA,UB: steady state — kernel forwards, uBridge taps +``` + +Every arrow to uBridge is one `_ubridge_send` command line on the node's local hypervisor console; uBridge executes it (netlink/ioctl) and replies OK or an error synchronously before the next command is sent — those replies are omitted for brevity. uBridge never initiates anything: it is a pure executor, and the only data ever flowing back (capture/marker frame copies) rides separate AF_PACKET sockets, not this console. + +The TAP already exists at attach time — created at node start, before QEMU launched (Adapter lifecycle below) — which is what makes the attach runtime-safe on a running VM: anchors, bridge memberships and qdiscs all change around the `-netdev tap` QEMU already holds, and nothing talks to QEMU itself outside suspend/resume (QMP `set_link`, so the guest's carrier follows the link). + +A mixed Docker↔QEMU link runs the identical sequence per end — each node's uBridge receives the same commands on its own anchor, the Docker end's `gv` veth host end in place of the `gq` TAP; only the anchor name differs (the mixin is anchor-agnostic, which is the point of the shared `gns3server/compute/kernel_datapath.py`). ## Adapter lifecycle -* **Birth (node start, before QEMU is launched)** — `_prepare_tap_datapath` - probes the tap module, then `_create_taps` sweeps any leftover (a persistent - TAP outlives a crash) and creates one TAP per adapter: - `tap create gq{node_id[:8]}e{adapter}p{port}` → `tap set_owner ` - (so the unprivileged QEMU process can open it) → `link set down` - (carrier off until a link attaches). QEMU then opens it as - `-netdev tap,id=gns3-{adapter},ifname=,script=no,downscript=no`. -* **Life (running)** — the TAP is never created or destroyed again. Link - create/delete/switch, suspend, capture and markers only change *what is - attached to it*. -* **Death (node stop)** — the QEMU process is stopped **first**, then - `_remove_taps` un-persists the TAPs (`tap delete`) and deletes the - per-link kernel bridges the node still holds. The order matters: uBridge's - `tap delete` refuses a device another process holds open ("Device or - resource busy") and that best-effort delete is suppressed, so deleting the - TAPs while QEMU still ran leaked every persistent TAP (process side first, - then the devices — the same lesson as IOU's reverse stop order). Deleting - a node never runs the link-teardown path, so those bridges would otherwise - stay behind as empty orphans; a restart rebuilds them from the NIO. +* **Birth (node start, before QEMU is launched)** — `_prepare_tap_datapath` probes the tap module, then `_create_taps` sweeps any leftover (a persistent TAP outlives a crash) and creates one TAP per adapter: `tap create gq{node_id[:8]}e{adapter}p{port}` → `tap set_owner ` (so the unprivileged QEMU process can open it) → `link set down` (carrier off until a link attaches). QEMU then opens it as `-netdev tap,id=gns3-{adapter},ifname=,script=no,downscript=no`. +* **Life (running)** — the TAP is never created or destroyed again. Link create/delete/switch, suspend, capture and markers only change *what is attached to it*. +* **Death (node stop)** — the QEMU process is stopped **first**, then `_remove_taps` un-persists the TAPs (`tap delete`) and deletes the per-link kernel bridges the node still holds. The order matters: uBridge's `tap delete` refuses a device another process holds open ("Device or resource busy") and that best-effort delete is suppressed, so deleting the TAPs while QEMU still ran leaked every persistent TAP (process side first, then the devices — the same lesson as IOU's reverse stop order). Deleting a node never runs the link-teardown path, so those bridges would otherwise stay behind as empty orphans; a restart rebuilds them from the NIO. ## Link operations @@ -104,73 +236,31 @@ datapaths (delete + re-create, or a project reopen). ## Capture, markers, filters -Identical to Docker's kernel links, on the TAP: `capture start_kernel ` -(driven by the NIO type, not by a link flag), `marker add_kernel …` -(only the capture node's end), and one `tc netem set` per anchor so each -direction is impaired once. `frequency_drop`/`quota`/`window_drop` run as the -eBPF classifier on the anchor's clsact egress. +Identical to Docker's kernel links, on the TAP: `capture start_kernel ` (driven by the NIO type, not by a link flag), `marker add_kernel …` (only the capture node's end), and one `tc netem set` per anchor so each direction is impaired once. `frequency_drop`/`quota`/`window_drop` run as the eBPF classifier on the anchor's clsact egress. ## Link-local frames (LACP, LLDP, 802.1X, STP) -The bridge stands in for a cable, but the kernel's `br_handle_frame()` does -not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by -default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while -the UDP relay carries them fine. uBridge therefore opens every port's -per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at -`brctl addif` time (Linux 4.15+, best-effort on older kernels). The -bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, -so LACP is reachable per port only. +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. -Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and -LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in -`br_handle_frame()` drops it unconditionally and no mask can enable it. -That is a documented limit, not a regression (PFC's hardware semantics are -out of emulation's reach anyway); a lab that needs PAUSE frames on the wire -must use a relay link. Verified live by -`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest -matrix); the contract and kernel references live in -`docs/design/ubridge-link-local-frame-forwarding-spec.md`. +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in `br_handle_frame()` drops it unconditionally and no mask can enable it. That is a documented limit, not a regression (PFC's hardware semantics are out of emulation's reach anyway); a lab that needs PAUSE frames on the wire must use a relay link. Verified live by `tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest matrix); the contract and kernel references live in `docs/design/ubridge-link-local-frame-forwarding-spec.md`. ## Legacy fallback -A uBridge build without the `tap` module (or without `CAP_NET_ADMIN`) keeps -the previous datapath untouched: local UDP tunnels into uBridge, QEMU launched -with `-netdev socket`, and the node logs a warning that it cannot carry kernel -links. Such a node is never offered a kernel link, so nothing fails. +A uBridge build without the `tap` module (or without `CAP_NET_ADMIN`) keeps the previous datapath untouched: local UDP tunnels into uBridge, QEMU launched with `-netdev socket`, and the node logs a warning that it cannot carry kernel links. Such a node is never offered a kernel link, so nothing fails. ## Verified -End-to-end through a real server (isolated instance, two QEMU nodes with a -stub binary, the test playing the guest by holding the TAP fds and pushing -frames): **32/32 checks** on the kernel datapath and **29/29** on the relay -variant (`enable_kernel_datapath = false`). Kernel run highlights: +End-to-end through a real server (isolated instance, two QEMU nodes with a stub binary, the test playing the guest by holding the TAP fds and pushing frames): **32/32 checks** on the kernel datapath and **29/29** on the relay variant (`enable_kernel_datapath = false`). Kernel run highlights: -* anchors created persistent and owned by the server user, QEMU's netdev is - the TAP, no local UDP tunnel left; `link_list` reports `kernel_datapath: true` - and one per-link bridge holds both taps; -* frames cross in both directions (~0.1 ms); `delay 100` ⇒ netem on both - anchors and a measured **100.3 ms** one-way; +* anchors created persistent and owned by the server user, QEMU's netdev is the TAP, no local UDP tunnel left; `link_list` reports `kernel_datapath: true` and one per-link bridge holds both taps; +* frames cross in both directions (~0.1 ms); `delay 100` ⇒ netem on both anchors and a measured **100.3 ms** one-way; * suspend ⇒ anchor DOWN, no traffic, resume ⇒ 100.3 ms again (filter kept); * capture ⇒ a 252-byte pcap written by `capture start_kernel`; -* link delete ⇒ bridge gone, qdisc clean, anchor alive; re-create ⇒ 0.1 ms - with no residual impairment; node stop ⇒ no taps, no bridges left. +* link delete ⇒ bridge gone, qdisc clean, anchor alive; re-create ⇒ 0.1 ms with no residual impairment; node stop ⇒ no taps, no bridges left. -Relay run the same script with the flag off: same lifecycle, filter measured -at 109.8 ms (applied by uBridge's bridge — no netem on the anchor, as -expected), no kernel bridge holding the tap. +Relay run the same script with the flag off: same lifecycle, filter measured at 109.8 ms (applied by uBridge's bridge — no netem on the anchor, as expected), no kernel bridge holding the tap. -The repo suite `tests/e2e/test_qemu_kernel_datapath.py` covers the same -ground with a *real* guest: two IOSv routers (a real qcow2 image, linked -clone, the real IOS CLI on the telnet console) — the full kernel lifecycle -(anchors at start, per-link bridge membership, pure-L2 anchors, the §E.2 -idle-silence window with the guests shut, real ICMP, netem delay, suspend, -capture, link delete/re-create, node stop/start rewiring, zero residue on -project delete) plus the relay negative control, -where the anchor is up and carrying as the relay's AF_PACKET endpoint (QEMU -has no other leg — the TAP is the netdev on the relay too). It needs an L3 -IOSv image in the images directory's `QEMU` subfolder; a raw node must also -name `hda_disk_interface` (the default `none` attaches no frontend device -and the VM comes up diskless — the IOSv appliance boots virtio). +The repo suite `tests/e2e/test_qemu_kernel_datapath.py` covers the same ground with a *real* guest: two IOSv routers (a real qcow2 image, linked clone, the real IOS CLI on the telnet console) — the full kernel lifecycle (anchors at start, per-link bridge membership, pure-L2 anchors, the §E.2 idle-silence window with the guests shut, real ICMP, netem delay, suspend, capture, link delete/re-create, node stop/start rewiring, zero residue on project delete) plus the relay negative control, where the anchor is up and carrying as the relay's AF_PACKET endpoint (QEMU has no other leg — the TAP is the netdev on the relay too). It needs an L3 IOSv image in the images directory's `QEMU` subfolder; a raw node must also name `hda_disk_interface` (the default `none` attaches no frontend device and the VM comes up diskless — the IOSv appliance boots virtio). ## Configuration @@ -183,41 +273,12 @@ enable_kernel_datapath = True ## Known caveat -Anchors are host-side netdevs, so the kernel gives them an IPv6 link-local -address and emits MLD/DAD from them; those frames flood into the emulated -segment. The hardening requirement (anchors must be pure L2, no L3 identity) -is specified for uBridge in `docs/design/ubridge-l2-anchor-spec.md` and is -**delivered**: anchors and per-link bridges come up with `addrgenmode none`, -no addresses, and an idle anchor is silent (asserted in the e2e after a 2.5 s -settle). +Anchors are host-side netdevs, so the kernel gives them an IPv6 link-local address and emits MLD/DAD from them; those frames flood into the emulated segment. The hardening requirement (anchors must be pure L2, no L3 identity) is specified for uBridge in `docs/design/ubridge-l2-anchor-spec.md` and is **delivered**: anchors and per-link bridges come up with `addrgenmode none`, no addresses, and an idle anchor is silent (asserted in the e2e after a 2.5 s settle). -One residual is accepted and documented there: enslaving a port makes the -*kernel* announce the bridge's multicast memberships once — an IGMPv3 plus one -or two MLDv2 reports in the first second — which floods into the segment like -any L2 control frame. It stops; it is outside the L2-only command's reach (no -addresses involved, IPv4 has no per-device IGMP switch). +One residual is accepted and documented there: enslaving a port makes the *kernel* announce the bridge's multicast memberships once — an IGMPv3 plus one or two MLDv2 reports in the first second — which floods into the segment like any L2 control frame. It stops; it is outside the L2-only command's reach (no addresses involved, IPv4 has no per-device IGMP switch). ## Roadmap -IOU (its fabric terminator in uBridge needs a TAP-terminated port — -`iol_bridge add_nio_tap`, frozen in -`docs/design/ubridge-iol-tap-anchor-spec.md`; unlike QEMU, one userspace hop -on the IOU leg is irreducible, the fabric is Unix sockets), Dynamips -(hypervisor-created taps, enslavable as they are), and the Ethernet switch / -cloud paths (their anchors are ubridge-owned by design) are still on the -relay; cross-compute kernel links need VXLAN/GENEVE encapsulation, which is -a separate project (it would serve Docker links the same way). +IOU (its fabric terminator in uBridge needs a TAP-terminated port — `iol_bridge add_nio_tap`, frozen in `docs/design/ubridge-iol-tap-anchor-spec.md`; unlike QEMU, one userspace hop on the IOU leg is irreducible, the fabric is Unix sockets), Dynamips (hypervisor-created taps, enslavable as they are), and the Ethernet switch / cloud paths (their anchors are ubridge-owned by design) are still on the relay; cross-compute kernel links need VXLAN/GENEVE encapsulation, which is a separate project (it would serve Docker links the same way). -Update: **IOU has landed** on the same mixin — see -`docs/features/iou-kernel-datapath.md` (Ethernet bays anchor on persistent -TAPs bound to the IOL fabric; serial links stay relay). **Dynamips has -landed** too — see `docs/features/dynamips-kernel-datapath.md` (the -hypervisor opens uBridge-created TAPs with `nio create_tap`: the same -external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay). -**The Ethernet switch has landed** — see -`docs/features/ethernet-switch-kernel-datapath.md` (the switch absorbs a -peer's anchor into its own kernel bridge; switch-to-switch cascades too). -**The IOL runner container has landed** — see -`docs/features/iol-docker-kernel-datapath.md` (persistent TAP anchors -through the generic bridge module's swappable TAP leg; the unix-socket guest -leg keeps one userspace hop, like IOU's fabric). +Update: **IOU has landed** on the same mixin — see `docs/features/iou-kernel-datapath.md` (Ethernet bays anchor on persistent TAPs bound to the IOL fabric; serial links stay relay). **Dynamips has landed** too — see `docs/features/dynamips-kernel-datapath.md` (the hypervisor opens uBridge-created TAPs with `nio create_tap`: the same external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay). **The Ethernet switch has landed** — see `docs/features/ethernet-switch-kernel-datapath.md` (the switch absorbs a peer's anchor into its own kernel bridge; switch-to-switch cascades too). **The IOL runner container has landed** — see `docs/features/iol-docker-kernel-datapath.md` (persistent TAP anchors through the generic bridge module's swappable TAP leg; the unix-socket guest leg keeps one userspace hop, like IOU's fabric). From d0773aabeafb3374727adf90a08f23d7b84b8b1e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 22:24:06 +0800 Subject: [PATCH 073/101] docs(iou): align the kernel-datapath doc with the delivered implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sequence diagram: drop the phantom "port fwd_mask 0xfffd" step — uBridge sets the per-port mask inside brctl addif and gns3-server never sends it. Frame path: explain the ingress/egress mechanism in prose (the TAP-fd write lands at the sender's anchor ingress, where nothing is installed; the bridge's forward out the far anchor is the egress transmission the tc chain sits on), and group the netio mailboxes into a kernel subgraph matching the relay diagram. Suspend: a down anchor answers EIO on the TAP fd in both directions (the section B hardening), not reads-block. Term table: the anchor shape is per Ethernet bay (4 units per bay), not the default 2x4. Link operations: carrier up is applied last, matching _attach_kernel_nio. Link-local: note the verification matrix runs on the Docker datapath. Reflow paragraphs to the soft-wrap convention. --- docs/features/iou-kernel-datapath.md | 350 +++++++++++++++------------ 1 file changed, 199 insertions(+), 151 deletions(-) diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index eea88f783..60844968b 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -5,193 +5,249 @@ See LICENSE file for licensing information. # IOU kernel datapath -IOU's only physical layer is the Unix-datagram IOL fabric (`/tmp/netio/`, -8-byte IOL headers, NETMAP), so it has no host netdev to enslave — a kernel -link instead gives each **Ethernet bay/unit** a persistent TAP anchor -(`gi{node_id[:8]}e{bay}p{unit}`), and uBridge's `iol_bridge` (the fake IOL -instance the NETMAP maps to) binds the port to that TAP with -`iol_bridge add_nio_tap`. Links then attach to the anchor exactly as they do -for Docker (veth) and QEMU (TAP): a per-link Linux bridge enslaves it, tc -impairments run on it, capture and markers anchor on it. The uBridge -capability is specified in `docs/design/ubridge-iol-tap-anchor-spec.md` and -delivered by the `feature/iol-tap-anchor` branch of the ubridge fork. +IOU's only physical layer is the Unix-datagram IOL fabric (`/tmp/netio/`, 8-byte IOL headers, NETMAP), so it has no host netdev to enslave — a kernel link instead gives each **Ethernet bay/unit** a persistent TAP anchor (`gi{node_id[:8]}e{bay}p{unit}`), and uBridge's `iol_bridge` (the fake IOL instance the NETMAP maps to) binds the port to that TAP with `iol_bridge add_nio_tap`. Links then attach to the anchor exactly as they do for Docker (veth) and QEMU (TAP): a per-link Linux bridge enslaves it, tc impairments run on it, capture and markers anchor on it. The uBridge capability is specified in `docs/design/ubridge-iol-tap-anchor-spec.md` and delivered by the `feature/iol-tap-anchor` branch of the ubridge fork. -Serial bays never anchor (a TAP carries Ethernet frames): serial links stay -on the relay datapath whatever the compute's capabilities. +Serial bays never anchor (a TAP carries Ethernet frames): serial links stay on the relay datapath whatever the compute's capabilities. ## Architecture -``` - IOU process root namespace - ┌──────────┐ unix datagram ┌──────────────────────────────┐ - │ IOL │◄──── /tmp/netio ──►│ uBridge iol_bridge (listener)│ - │ fabric │ (8-byte hdr) │ │ holds the TAP fd │ - └──────────┘ │ ▼ │ - │ gi{id}e{bay}p{unit} ──────┼──► gns3{link_id} - │ (persistent TAP anchor) │ (per-link kernel - └──────────────────────────────┘ bridge port) +The diagram shows the native shape — two IOU devices linked on the kernel datapath, the scenario the e2e suite drives. A Docker or QEMU peer on the far end is the same link segment with a different anchor (a `gv` veth host end or a `gq` TAP — see `docker-kernel-datapath.md`); each IOU still terminates its own fabric in its own uBridge. Interface names follow the deterministic rules `gi{node_id[:8]}e{bay}p{unit}` for the anchors and `gns3{link_id[:11]}` for the per-link bridge; the concrete names in the diagrams use node ids `a1b2c3d4` / `5e6f7a8b` (IOU A/B), link id prefix `5d3c2b1a4e6`, and application ids 1 / 2 (so the fake instances are `IOL-BRIDGE-513` / `IOL-BRIDGE-514`). Terms the diagrams use: + +| Term | Meaning | +|---|---| +| IOL fabric | the `/tmp/netio/` unix-datagram mailboxes — one directory shared by every IOU node on the host, namespaced by IOL id; every frame carries an 8-byte IOL header | +| NETMAP | the per-node redirect table pointing a port's fabric traffic at another IOL id — here uBridge's fake instance | +| `iol_bridge` | uBridge's fake IOL instance — one per node, named `IOL-BRIDGE-{application_id + 512}` (the NETMAP redirects the node's own id to it); one listener thread per bound port, and it holds the anchor TAP fds | +| anchor | the host-side interface a link attaches to — here the `gi` persistent TAP; a Docker peer's `gv` veth host end or a QEMU `gq` TAP is interchangeable as the other port | +| persistent TAP | the TAP created per Ethernet bay/unit at node start (`tap create`): born DOWN, address-free, 4 units per bay (as many bays as the node has Ethernet adapters) | +| per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | +| tap fd | a TAP's file descriptor: reads return frames the kernel received on the device, writes inject frames into it — on this datapath the `iol_bridge` listener holds it | +| FDB | the bridge's forwarding database (learned MAC → port); unknown, broadcast and multicast destinations are flooded | +| `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | +| `clsact` | the tc classifier carrier on the anchor; its egress chain is where the drop classifiers run | +| eBPF classifier | the stateful drop modes (every-Nth, quota, time window) at clsact egress prio 1 | +| `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | +| netem | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | +| `AF_PACKET tap` | uBridge's packet socket on the anchor — how capture and markers observe frames off the forwarding path | +| carrier | the anchor's admin state (`link set up/down`); suspend is carrier down — a down TAP answers EIO in both fd directions (writes drop the frame; the §B hardening tolerates both) | + +```mermaid +flowchart LR + subgraph IOSPA["IOU A process"] + iosA["IOS (IOL binary)"] + end + subgraph RootNS["root namespace"] + subgraph UBA["uBridge A"] + brA["IOL-BRIDGE-513"] + end + giA["gia1b2c3de0p0"] + BR["gns35d3c2b1a4e6"] + giB["gi5e6f7a8be0p0"] + subgraph UBB["uBridge B"] + brB["IOL-BRIDGE-514"] + end + end + subgraph IOSPB["IOU B process"] + iosB["IOS (IOL binary)"] + end + iosA <-->|"IOL fabric (NETMAP)"| brA + brA <-->|"TAP fd 0/0"| giA + giA <-->|"bridge port"| BR + BR <-->|"bridge port"| giB + giB <-->|"TAP fd 0/0"| brB + brB <-->|"IOL fabric (NETMAP)"| iosB + brA -.->|"AF_PACKET tap"| giA + brA -.->|"tc qdisc"| giA + brB -.->|"AF_PACKET tap"| giB + brB -.->|"tc qdisc"| giB ``` -One userspace hop on the IOU leg is **irreducible** — the fabric is Unix -sockets, so frames always cross fabric → listener thread → TAP fd. What the -kernel datapath buys is the **link segment**: anchor → per-link bridge → -peer anchor crosses entirely in the kernel (including IOU ↔ Docker/QEMU -links), impairments run as kernel tc instead of the userspace delay line -(the known blocking-nanosleep RTT defect), and suspend maps to anchor -admin-down (writes to a down TAP fail EIO and are dropped by the delivered -§B hardening). Direction semantics on the anchor match QEMU's exactly: -uBridge holds the fd like the QEMU process does. +Unlike the Docker veth datapath, where uBridge sits off the forwarding path and only taps, here each node's uBridge is **on** it: the `iol_bridge` listener terminates the fabric and relays frames between the mailboxes and the anchor TAP fd. That one userspace hop per leg is **irreducible** — the fabric is Unix sockets, so frames always cross fabric → listener thread → TAP fd. What the kernel datapath buys is the **link segment**: anchor → per-link bridge → peer anchor crosses entirely in the kernel (including IOU ↔ Docker/QEMU links), impairments run as kernel tc instead of the userspace delay line (the known blocking-nanosleep RTT defect), and suspend maps to anchor admin-down (writes to a down TAP fail EIO and are dropped by the delivered §B hardening). Direction semantics on the anchor match QEMU's exactly: uBridge holds the fd like the QEMU process does. + +Relay link between the same two nodes (what an old uBridge build, a serial port or `enable_kernel_datapath` off falls back to; a cross-compute peer is the same tunnel with the far socket on the real network instead of loopback): each port rides a UDP NIO inside its `iol_bridge` and no kernel interface exists — anchors untouched, filters/capture/markers on the port's userspace list: + +```mermaid +flowchart LR + subgraph IOPA["IOU A process — userspace"] + iosA["IOS (IOL binary)"] + end + subgraph KERN["kernel"] + mbxA["netio sockets"] + UDP["UDP sockets (loopback)"] + mbxB["netio sockets"] + end + subgraph UBA["uBridge A — userspace"] + ioA["IOL nio"] + enA["relay + userspace filters"] + nuA["UDP nio"] + end + subgraph UBB["uBridge B — userspace"] + nuB["UDP nio"] + enB["relay + userspace filters"] + ioB["IOL nio"] + end + subgraph IOPB["IOU B process — userspace"] + iosB["IOS (IOL binary)"] + end + iosA -->|"copy 1: user→kernel"| mbxA + mbxA -->|"copy 2: kernel→user"| ioA + ioA --> enA --> nuA + nuA -->|"copy 3: user→kernel"| UDP + UDP -->|"copy 4: kernel→user"| nuB + nuB --> enB --> ioB + ioB -->|"copy 5: user→kernel"| mbxB + mbxB -->|"copy 6: kernel→user"| iosB +``` + +Drawn one-way; the reverse direction is the mirror image. A relayed frame crosses the kernel/user boundary **six times each way** (twelve per round trip — each `sendto` copies user→kernel, each `recvfrom` kernel→user), the UDP hop additionally traversing the protocol stack twice; the relay's filters run in userspace on the frames between the copies. + +Frame path for one A → B frame (`giA` = `gia1b2c3de0p0`, `giB` = `gi5e6f7a8be0p0`). The controller ships the same filter dict to **both** endpoints, so the link carries **two impairment points** — each anchor's egress chain — but a one-way frame crosses exactly one of them: the destination anchor's, in order clsact → netem (a classifier drop means netem never sees the frame). The asymmetry is the trace's, not the configuration's — both anchors carry the identical chain: the frame enters the kernel at the sender's anchor **ingress** (the TAP-fd write; this datapath installs nothing on ingress), and the bridge's forward out of the far anchor is an **egress** transmission, which is where the tc chain sits. B → A is symmetric on giA. Capture and markers are the observation point: one AF_PACKET tap at whichever end captures, seeing **both** directions through its anchor: + +```mermaid +flowchart LR + subgraph IOPA["IOU A process — userspace"] + S1["IOS A sends on e0/0"] + end + subgraph KERN["kernel"] + mbxA["netio sockets"] + mbxB["netio sockets"] + end + subgraph UBA["uBridge A — userspace"] + S2["relay"] + end + subgraph GIA["giA — ingress"] + S4["ingress"] + chA["clsact + netem chain"] + tapA["AF_PACKET tap: capture + markers"] + end + subgraph BRP["per-link bridge"] + G1{"reserved 01:80:c2?"} + X1["dropped"] + S5["FDB lookup / flood"] + end + subgraph GIB["giB — egress chain (clsact, then netem)"] + S6["eBPF prio 1"] + S7["bpf_drop prio 10-99"] + S8["netem"] + end + subgraph UBB["uBridge B — userspace"] + S9["relay"] + end + subgraph IOPB["IOU B process — userspace"] + S11["IOS B receives on e0/0"] + end + S1 -->|"copy 1: user→kernel"| mbxA + mbxA -->|"copy 2: kernel→user"| S2 + S2 -->|"copy 3: user→kernel"| S4 + S4 --> G1 + G1 -->|"PAUSE 0x01"| X1 + G1 -->|"0xfffd passes"| S5 + S5 --> S6 + S6 --> S7 --> S8 + S8 -->|"copy 4: kernel→user"| S9 + S9 -->|"copy 5: user→kernel"| mbxB + mbxB -->|"copy 6: kernel→user"| S11 + S4 -.->|"serves B→A"| chA + S4 -.->|"observes tx + rx"| tapA +``` + +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below); the AF_PACKET tap — capture and markers ride it, installed at whichever end captures — observes both directions of the link through its anchor (tx/rx signal pairs) but never a clsact-dropped frame. The crossings do not disappear on this datapath: each leg still crosses the kernel/user boundary three times (`sendto`/`recvfrom` on the fabric, then the TAP write — the B leg mirrors them, six per one-way frame, the same count as the relay); what moves into the kernel is the link segment — no UDP tunnel, no protocol-stack pass — and with it the impairment chain. A fabric → anchor → bridge → anchor → fabric round trip measures ≈0.2 ms (Verified below). ## Datapath selection `UDPLink._kernel_datapath_eligible` extends the Docker/QEMU rules with: -* the compute must report **`ubridge_iol_tap`** (its own capability: the tap - module and `iol_bridge add_nio_tap` land independently, and either can be - missing on an old build). Probed by running one add/delete cycle on a - scratch bridge + scratch TAP (`probe_iol_tap_support`, cached by binary - identity, scratch bridge id 1050 — above the per-node id space 513..1024 - the fabric locks); any failure means unknown → relay; -* **both linked ports must be Ethernet** — an IOU serial port disqualifies - the kernel path per-port, not per-node, so serial links between capable - IOU nodes silently stay on the relay instead of failing at the compute. +* the compute must report **`ubridge_iol_tap`** (its own capability: the tap module and `iol_bridge add_nio_tap` land independently, and either can be missing on an old build). Probed by running one add/delete cycle on a scratch bridge + scratch TAP (`probe_iol_tap_support`, cached by binary identity, scratch bridge id 1050 — above the per-node id space 513..1024 the fabric locks); any failure means unknown → relay; +* **both linked ports must be Ethernet** — an IOU serial port disqualifies the kernel path per-port, not per-node, so serial links between capable IOU nodes silently stay on the relay instead of failing at the compute. -The NIO schema on the compute NIO routes accepts `nio_bridge` alongside the -UDP/TAP/Ethernet NIOs (the same widening QEMU needed — without it the -controller's NIO POST is 422'd before the node ever sees it). +The NIO schema on the compute NIO routes accepts `nio_bridge` alongside the UDP/TAP/Ethernet NIOs (the same widening QEMU needed — without it, the controller's NIO POST is 422'd before the node ever sees it). + +## Link creation sequence + +Both endpoints sit on the **same compute** (a kernel-link prerequisite). The controller derives the bridge name once from the link id and ships one `nio_bridge` record to each endpoint — same bridge name, each side's own filters/markers and the suspend flag — the two POSTs overlapping (`asyncio.gather`, either failure rolls the peer back). The bridge has no owner: both sides race `brctl create` (EEXIST tolerated, the loser verifies via `brctl show`). The anchors already exist, born at node start (Anchor lifecycle below), so attaching only binds and enslaves: + +```mermaid +sequenceDiagram + participant API as REST API + participant C as Controller + participant NA as Node A handler + participant NB as Node B handler + participant UA as uBridge A + participant UB as uBridge B + + API->>C: POST /links + C->>C: eligibility check + C->>C: bridge = gns3 + link_id[:11] + par POST nio_bridge to node A + C->>NA: nio_bridge (bridge, A's filters/markers, suspend) + NA->>UA: iol_bridge add_nio_tap + NA->>UA: brctl create + NA->>UA: link set bridge up + NA->>UA: brctl addif giA + NA->>UA: markers / capture if any + NA->>UA: tc netem + bpf + eBPF + NA->>UA: link set giA up + and POST nio_bridge to node B + C->>NB: nio_bridge (same bridge, B's filters/markers) + NB->>UB: same on giB + end + Note over UA,UB: steady state — kernel forwards between the anchors, iol_bridge relays to the IOU processes +``` + +`iol_bridge add_nio_tap ""` is the one IOU-specific step: it binds the fabric port to the anchor and hands uBridge the TAP fd. Deletion reverses it in miniature — markers, tc reset, delif, `brctl delete`, then `iol_bridge delete_nio_tap` releases the fd while the control channel lives (Link operations below). ## Anchor lifecycle — and why the stop order is QEMU's reverse -* **Birth (node start)** — `_prepare_tap_datapath` probes the capability and - creates one persistent TAP per Ethernet bay/unit (4 units per bay, swept - for stale leftovers first, born DOWN). uBridge holds a device's fd only - while a kernel link binds the port; until then the TAP has no holder. -* **Life** — the TAP is never recreated. A kernel link adds - `iol_bridge add_nio_tap ` then - enslaves the anchor (`_kernel_attach`); a relay link keeps - `iol_bridge add_nio_udp` and never touches the anchor. Switching a port - between datapaths is just an NIO swap — the IOL port frees its previous - NIO (fd or socket) on attach. -* **Death (node stop)** — `_stop_ubridge` runs the cleanup **in the reverse - order of QEMU's**, because here uBridge itself holds the anchor fds: - `iol_bridge delete` (releases every port's TAP fd) → `tap delete` per - anchor (a held TAP answers EBADFD/207) → `brctl delete` for per-link - bridges still held → stop the hypervisor. Removing a single kernel link - does the same in miniature: `_remove_kernel_nio` (markers, tc reset, - delif, brctl delete) then `iol_bridge delete_nio_tap` — releasing the fd - while the control channel lives. +* **Birth (node start)** — `_prepare_tap_datapath` probes the capability and creates one persistent TAP per Ethernet bay/unit (4 units per bay, swept for stale leftovers first, born DOWN). uBridge holds a device's fd only while a kernel link binds the port; until then the TAP has no holder. +* **Life** — the TAP is never recreated. A kernel link adds `iol_bridge add_nio_tap ` then enslaves the anchor (`_kernel_attach`); a relay link keeps `iol_bridge add_nio_udp` and never touches the anchor. Switching a port between datapaths is just an NIO swap — the IOL port frees its previous NIO (fd or socket) on attach. +* **Death (node stop)** — `_stop_ubridge` runs the cleanup **in the reverse order of QEMU's**, because here uBridge itself holds the anchor fds: `iol_bridge delete` (releases every port's TAP fd) → `tap delete` per anchor (a held TAP answers EBADFD/207) → `brctl delete` for per-link bridges still held → stop the hypervisor. Removing a single kernel link does the same in miniature: `_remove_kernel_nio` (markers, tc reset, delif, brctl delete) then `iol_bridge delete_nio_tap` — releasing the fd while the control channel lives. ## Link operations | Operation | Kernel-datapath action | |---|---| -| Create / attach | `iol_bridge add_nio_tap` (bind port ↔ anchor, uBridge holds the fd) + `brctl create` (EEXIST tolerated) + `link set up` + `brctl addif` + carrier up + markers + `tc netem set` | +| Create / attach | `iol_bridge add_nio_tap` (bind port ↔ anchor, uBridge holds the fd) + `brctl create` (EEXIST tolerated) + `link set up` + `brctl addif` + markers + `tc netem set` + carrier up | | Delete / detach | marker teardown → `tc reset ` → carrier off → `brctl delif`/`brctl delete` (last endpoint wins) → `iol_bridge delete_nio_tap` (release the fd; the device survives) | | Update (filters/markers changed) | reconcile on the anchor only — no re-binding, no re-enslaving | -| Suspend | anchor admin-down (IOL relay writes fail EIO → dropped; reads block) + resume restores | +| Suspend | anchor admin-down (TAP-fd writes fail EIO → dropped; reads answer EIO too — the §B hardening tolerates both) + resume restores | ## Capture, markers, filters -Capture on a kernel link is `capture start_kernel ` (driven by the NIO -type); markers ride the base reconcile with the anchor as the location — -IOU's marker primitives are polymorphic (`marker add_kernel/enable_kernel/ -delete_kernel` on an anchor, `iol_bridge … mark` on an IOL location), which -let the old IOU-specific reconcile override be deleted outright. Filters are -one tc netem qdisc per anchor plus cls_bpf/eBPF drops — the port's -userspace `iol_bridge add_packet_filter` filters carry relay links only and -are never combined with tc on the same port. +Capture on a kernel link is `capture start_kernel ` (driven by the NIO type); markers ride the base reconcile with the anchor as the location — IOU's marker primitives are polymorphic (`marker add_kernel/enable_kernel/delete_kernel` on an anchor, `iol_bridge … mark` on an IOL location), which let the old IOU-specific reconcile override be deleted outright. Filters are one tc netem qdisc per anchor plus cls_bpf/eBPF drops — the port's userspace `iol_bridge add_packet_filter` filters carry relay links only and are never combined with tc on the same port. ## Link-local frames (LACP, LLDP, 802.1X, STP) -The bridge stands in for a cable, but the kernel's `br_handle_frame()` does -not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by -default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while -the UDP relay carries them fine. uBridge therefore opens every port's -per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at -`brctl addif` time (Linux 4.15+, best-effort on older kernels). The -bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, -so LACP is reachable per port only. +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. -Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and -LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in -`br_handle_frame()` drops it unconditionally and no mask can enable it. -That is a documented limit, not a regression (PFC's hardware semantics are -out of emulation's reach anyway); a lab that needs PAUSE frames on the wire -must use a relay link. Verified live by -`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest -matrix); the contract and kernel references live in -`docs/design/ubridge-link-local-frame-forwarding-spec.md`. +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in `br_handle_frame()` drops it unconditionally and no mask can enable it. That is a documented limit, not a regression (PFC's hardware semantics are out of emulation's reach anyway); a lab that needs PAUSE frames on the wire must use a relay link. Verified live by `tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest matrix, driven on the Docker datapath — the same per-link-bridge mechanism; the IOU e2e does not repeat it); the contract and kernel references live in `docs/design/ubridge-link-local-frame-forwarding-spec.md`. ## Relay fallback -A uBridge without `iol_bridge add_nio_tap` (old build) keeps the node on the -relay datapath unchanged: every link rides `iol_bridge add_nio_udp`, the -node starts fine, and the warning says it cannot carry kernel links. The -controller never offers such a node a kernel link (capability unreported). +A uBridge without `iol_bridge add_nio_tap` (old build) keeps the node on the relay datapath unchanged: every link rides `iol_bridge add_nio_udp`, the node starts fine, and the warning says it cannot carry kernel links. The controller never offers such a node a kernel link (capability unreported). ## Verified -Unit level: anchor lifecycle and naming, kernel/relay binding on add, -update and remove, the stop order (iol_bridge delete before tap delete -before brctl delete before hypervisor stop), `_networking` on restart with -mixed kernel/relay ports, capture and marker command shapes on both -datapaths, the capability probe (cycle, old-build fallback, cleanup, -caching), capability plumbing, and controller eligibility (per-compute -capability, its independence from `ubridge_tap`, serial-port exclusion). +Unit level: anchor lifecycle and naming, kernel/relay binding on add, update and remove, the stop order (iol_bridge delete before tap delete before brctl delete before hypervisor stop), `_networking` on restart with mixed kernel/relay ports, capture and marker command shapes on both datapaths, the capability probe (cycle, old-build fallback, cleanup, caching), capability plumbing, and controller eligibility (per-compute capability, its independence from `ubridge_tap`, serial-port exclusion). -End-to-end on a live server (isolated instance, two IOU nodes behind a fake -ELF image, the test playing the IOL fabric on `/tmp/netio/` and pushing -frames through it): **64/64 checks** on the kernel datapath and **62/62** on -the relay (`enable_kernel_datapath = false`). Kernel highlights: +End-to-end on a live server (isolated instance, two IOU nodes behind a fake ELF image, the test playing the IOL fabric on `/tmp/netio/` and pushing frames through it): **64/64 checks** on the kernel datapath and **62/62** on the relay (`enable_kernel_datapath = false`). Kernel highlights: -* 8 anchors per node (2 bays × 4 units), persistent, DOWN, address-free; - `link_list` reports `kernel_datapath: true` and one per-link bridge holds - both anchors with `brport/state = 3`; -* fabric → anchor → bridge → anchor → fabric in both directions at ~0.2 ms; - `delay 100` ⇒ netem on both anchors and a measured **100.5 ms** one-way; +* 8 anchors per node (2 bays × 4 units), persistent, DOWN, address-free; `link_list` reports `kernel_datapath: true` and one per-link bridge holds both anchors with `brport/state = 3`; +* fabric → anchor → bridge → anchor → fabric in both directions at ~0.2 ms; `delay 100` ⇒ netem on both anchors and a measured **100.5 ms** one-way; * suspend ⇒ anchor DOWN, no traffic, resume ⇒ 100.4 ms with the qdisc intact; * capture ⇒ a 328-byte pcap written by `capture start_kernel`; * link delete ⇒ bridge gone, qdisc clean, anchor alive, re-create at 0.6 ms; -* a **serial** link between the same two capable nodes stays on the relay - (`kernel_datapath: false`) — per-port exclusion, not per-node; -* single-sided restart ⇒ stopped node's anchors removed, peer's kept, link - re-attached from the NIO (0.3 ms); stop ⇒ no stray anchors, bridges or - fabric sockets. +* a **serial** link between the same two capable nodes stays on the relay (`kernel_datapath: false`) — per-port exclusion, not per-node; +* single-sided restart ⇒ stopped node's anchors removed, peer's kept, link re-attached from the NIO (0.3 ms); stop ⇒ no stray anchors, bridges or fabric sockets. -The relay run exercises the same lifecycle with the IOL UDP NIOs (no tc -qdiscs, userspace `delay` at 100.7 ms, suspend via the synthetic -frequency_drop, `iol_bridge start_capture`), which is also the regression -net for the port-coordinate capture restore this work touched. +The relay run exercises the same lifecycle with the IOL UDP NIOs (no tc qdiscs, userspace `delay` at 100.7 ms, suspend via the synthetic frequency_drop, `iol_bridge start_capture`), which is also the regression net for the port-coordinate capture restore this work touched. -The repository's live e2e suite carries the same scenario with **real -IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker -`e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on -the server's telnet console, driving anchors-born-with-the-node, the §E.2 -idle-silence window with the guests shut, real ICMP through the port -bridge, netem delay, the TAP-anchor classifier spot check -(bpf match-drop; the eBPF every-nth mode at 56 % measured round-trip loss, -55.6 % expected), suspend, capture, link delete/re-create, node -stop/start rewiring with a delay filter restored on the fresh anchor -(tc and traffic both), the serial link staying on the relay, and a relay -negative control whose real `delay`/`frequency_drop` filters ride the -port's `iol_bridge add_packet_filter` list — capture and markers on the -same engine (`iol_bridge start_capture` / the port's `mark` filter). +The repository's live e2e suite carries the same scenario with **real IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker `e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on the server's telnet console, driving anchors-born-with-the-node, the §E.2 idle-silence window with the guests shut, real ICMP through the port bridge, netem delay, the TAP-anchor classifier spot check (bpf match-drop; the eBPF every-nth mode at 56 % measured round-trip loss, 55.6 % expected), suspend, capture, link delete/re-create, node stop/start rewiring with a delay filter restored on the fresh anchor (tc and traffic both), the serial link staying on the relay, and a relay negative control whose real `delay`/`frequency_drop` filters ride the port's `iol_bridge add_packet_filter` list — capture and markers on the same engine (`iol_bridge start_capture` / the port's `mark` filter). -The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the -fork), including the DOWN-anchor resilience and kernel-bridge interop this -design depends on. +The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the fork), including the DOWN-anchor resilience and kernel-bridge interop this design depends on. -One pre-existing bug surfaced here and is fixed: the IOU NIO **update** route -never copied the body's `suspend` onto the NIO (Docker and QEMU routes do). -Harmless on the relay, where suspend is emulated by the synthetic -frequency_drop filter, but fatal for a kernel link, whose suspend *is* the -anchor's admin state — a "suspended" link kept forwarding. +One pre-existing bug surfaced here and is fixed: the IOU NIO **update** route never copied the body's `suspend` onto the NIO (Docker and QEMU routes do). Harmless on the relay, where suspend is emulated by the synthetic frequency_drop filter, but fatal for a kernel link, whose suspend *is* the anchor's admin state — a "suspended" link kept forwarding. -**Measuring seeded impairments on IOU**: IOL emits CDP/keepalive frames of -its own, and they draw from the netem RNG stream — a `seed`-reproducible run -never matches while they are on (measured: three rounds of mismatches, then -bit-identical pcaps after `no cdp run`). Turn the platform chatter off before -comparing seeded runs; the extra frames are visible in the capture. +**Measuring seeded impairments on IOU**: IOL emits CDP/keepalive frames of its own, and they draw from the netem RNG stream — a `seed`-reproducible run never matches while they are on (measured: three rounds of mismatches, then bit-identical pcaps after `no cdp run`). Turn the platform chatter off before comparing seeded runs; the extra frames are visible in the capture. ## Configuration @@ -204,12 +260,4 @@ enable_kernel_datapath = True ## Roadmap -Dynamips has landed (see `dynamips-kernel-datapath.md` — the hypervisor -opens uBridge-created TAPs, the same shape QEMU uses), and so has the -Ethernet switch (see `ethernet-switch-kernel-datapath.md` — it absorbs the -peer's anchor into its own kernel bridge). Switch-to-switch cascades (a veth -pair between the two kernel bridges) have landed, and so has the IOL runner -container — see `iol-docker-kernel-datapath.md` (same persistent-TAP shape, -through the generic bridge module's swappable TAP leg instead of the IOL -bridge). Cross-compute kernel links need VXLAN/GENEVE encapsulation — -deferred until every node type is kernelized. +Dynamips has landed (see `dynamips-kernel-datapath.md` — the hypervisor opens uBridge-created TAPs, the same shape QEMU uses), and so has the Ethernet switch (see `ethernet-switch-kernel-datapath.md` — it absorbs the peer's anchor into its own kernel bridge). Switch-to-switch cascades (a veth pair between the two kernel bridges) have landed, and so has the IOL runner container — see `iol-docker-kernel-datapath.md` (same persistent-TAP shape, through the generic bridge module's swappable TAP leg instead of the IOL bridge). Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. From 74010b9abdcedbda4bcc45d8fe270d3b43333dd4 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sat, 3 Oct 2026 23:27:55 +0800 Subject: [PATCH 074/101] docs: dynamips kernel-datapath diagram suite, unified frame-path form across the family The dynamips doc gets the full suite: anatomy (anchors reached by the hypervisors' TAP fds, uBridge's AF_PACKET-tap / tc-qdisc sidebands), a one-way copy-numbered relay comparison (six kernel/user crossings per way against the kernel datapath's two TAP-fd legs), a staged walkthrough (reserved-range diamond, egress clsact chain, capture/markers observation sideband, copies 1-2) and a control sequence of real commands in real order (hypervisor fd first, capture before markers, carrier last; router handlers, not computes, as the per-node participants). The four sibling frame paths are unified on the same form: netem qdisc node labels, one capture/markers node outside the stages with AF_PACKET tap edges anchored at ingress and post-chain egress, copy numbering wherever the trace crosses the kernel/user boundary (qemu's two TAP-fd legs, iol-docker's six socket/TAP legs), stage subgraphs with role-noted titles everywhere (iol-docker restructured from its flat chain, iou's chA/tapA variants folded into the canonical shapes). The documentation skill records the rules as v3.1.0: four diagram kinds (anatomy / comparison / control sequence / walkthrough), one step per node, dashed sidebands per role with meaningful direction, numbered crossings when copies are the point, glossary before the first diagram. --- .claude/skills/gns3-documentation/SKILL.md | 24 +- docs/features/docker-kernel-datapath.md | 7 +- docs/features/dynamips-kernel-datapath.md | 299 ++++++++++++-------- docs/features/iol-docker-kernel-datapath.md | 57 +++- docs/features/iou-kernel-datapath.md | 11 +- docs/features/qemu-kernel-datapath.md | 11 +- 6 files changed, 264 insertions(+), 145 deletions(-) diff --git a/.claude/skills/gns3-documentation/SKILL.md b/.claude/skills/gns3-documentation/SKILL.md index 8772e1f07..485de4945 100644 --- a/.claude/skills/gns3-documentation/SKILL.md +++ b/.claude/skills/gns3-documentation/SKILL.md @@ -1,7 +1,7 @@ --- name: documentation description: Use this skill when creating or updating technical documentation under docs/ directory for GNS3 server -version: 3.0.0 +version: 3.1.0 --- # GNS3 Server Technical Documentation Standard @@ -39,7 +39,7 @@ Code details are left to the codebase — readers can use AI to find implementat ## What to Include -- Mermaid diagrams for architecture and flow (GitHub renders natively) +- Mermaid diagrams for architecture and flow (GitHub renders natively) — see Mermaid Diagram Rules below - `graph` for component relationships - `sequenceDiagram` for request/response flows - `flowchart` for data processing steps @@ -47,6 +47,26 @@ Code details are left to the codebase — readers can use AI to find implementat - Request/response JSON examples (for interfaces) - Performance data (only measured numbers, no estimates) +## Mermaid Diagram Rules + +Draw only the diagram kinds the story needs — each kind answers exactly one reader question, in reading order: + +- **Anatomy — "what exists, and who owns it?"** — `flowchart LR` with the fewest nodes that show ownership: one subgraph per ownership boundary (a process, a namespace, userspace vs kernel), anything shared between owners a bare node inside the enclosing boundary. The steady-state path is bidirectional solid (`<-->`) edges labeled with the carrying technology, never with a direction; each observer/configurator gets one dashed sideband edge per role. +- **Comparison — "how does X differ from Y?"** — the mirror of its counterpart diagram: same scenario, same node ids, same concrete names, one variable changed. Draw one direction with unidirectional `-->` and state in prose that the reverse is the mirror image. +- **Control sequence — "how is it built / torn down?"** — `sequenceDiagram` spanning every layer (API → coordinator → per-node handlers → the managed process): local steps are self-messages, concurrent fan-out is a `par / and / end` block, each message to the bottom layer is one real command in real order, and a closing `Note` names the steady state. +- **Walkthrough — "how does one item traverse it?"** — `flowchart LR` as a linear pipeline: one subgraph per stage, titled with that stage's role in this trace (`X — ingress`), sideband dashed edges (reverse-direction role, observation point) anchored at the stage they belong to. + +Whatever the kind: + +- **One step per node, one short line per label.** If a label needs `
` lines to explain itself, split the node into smaller steps instead of packing text — needing many words means the flow needs further decomposition. +- **Details belong to prose.** Parentheticals, exceptions, and parameter lists go to the surrounding text or the section that owns the topic; a diagram never duplicates what prose already says. +- **Define jargon before the first diagram.** A Term | Meaning table placed above the diagrams covers the domain terms they use (interface names, kernel mechanisms), so readers don't have to scan forward. +- **Edge style is semantics.** Solid arrows carry the main flow (data plane); dashed `-.->` edges are sideband relationships (taps, control installs). A node with multiple sideband roles gets one dashed edge per role, each labeled with a single function — and edge *direction* matters too: a config edge points from the configurere to its target, an observation edge from the target to the observer. +- **Decision points are diamonds with labeled outcomes**, not text-packed rectangles spelling out every branch inside one node. +- **A comparison diagram changes exactly one variable.** When two diagrams contrast approaches (kernel vs relay, old vs new), everything else — scenario, topology, node names — stays identical between them; a difference that isn't the point belongs in a prose aside, not in the picture. +- **Numbered crossings stay consistent.** When kernel/user or process-boundary copies are the point of a diagram, number the crossing edges (`copy N: user→kernel`) and keep one numbering scheme across every diagram in the document. +- **Concrete names instantiate stated rules.** Every interface/bridge/instance name appearing in a diagram instantiates a deterministic naming rule stated in prose before the first diagram — never invent ad-hoc names. + ## What to Skip - Code snippets — let readers search the codebase diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 423189a2c..711d856c9 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -155,8 +155,9 @@ flowchart LR subgraph GVB["gvB — egress chain (clsact, then netem)"] S5["eBPF prio 1"] S6["bpf_drop prio 10-99"] - S7["netem"] + S7["netem qdisc"] end + T["capture / markers"] end subgraph BNS["container B netns"] S9["eth0 B receives"] @@ -168,9 +169,11 @@ flowchart LR S4 --> S5 S5 --> S6 --> S7 S7 -->|"veth pair"| S9 + S3 -. "AF_PACKET tap" .-> T + S7 -. "AF_PACKET tap" .-> T ``` -What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Packet filters below). The AF_PACKET taps observe both host ends but never a clsact-dropped frame. +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Packet filters below). Whichever end hosts capture / markers, its AF_PACKET tap — the dashed sideband above — clones the frame at that anchor: a tap on gvA sees the frame on arrival, a tap on gvB only if the clsact chain let it through. All dozen-plus GNS3 filter types ride these three kernel objects — nine of them (delay, packet_loss, corrupt, duplicate, rate, reorder, gemodel, seed, limit) are parameters of the single netem qdisc, `bpf` expressions are the cls_bpf filters, and frequency_drop / quota / window_drop are the three modes inside the one eBPF program (evaluated nth → quota → window). The complete type-to-object mapping is the Packet filters table below. diff --git a/docs/features/dynamips-kernel-datapath.md b/docs/features/dynamips-kernel-datapath.md index baed487c7..5f844017d 100644 --- a/docs/features/dynamips-kernel-datapath.md +++ b/docs/features/dynamips-kernel-datapath.md @@ -5,74 +5,185 @@ See LICENSE file for licensing information. # Dynamips kernel datapath -Dynamips routers emulate every port inside the Dynamips process; today a link -reaches a router through a UDP tunnel (a Dynamips `nio create_udp`, a -loopback port pair, the node's uBridge relay, and the link's UDP NIOs). The -kernel datapath replaces that segment with the same anchor model Docker, -QEMU and IOU use: every **Ethernet slot/port** owns a persistent TAP -(`gd{node_id[:8]}e{slot}p{port}`), created by the server through uBridge's -tap module, and links attach to the anchor — a per-link Linux bridge -enslaves it, tc impairments run on it, capture and markers anchor on it. -The Dynamips **hypervisor** opens the TAP with `nio create_tap` (QEMU's -move: an external process holds the fd), unprivileged because the server -handed the device's ownership to its own user. +Dynamips routers emulate every port inside the Dynamips process; on the relay datapath a link reaches a router through a UDP tunnel (a Dynamips `nio create_udp`, a loopback port pair, the node's uBridge relay, and the link's UDP NIOs). The kernel datapath replaces that segment with the same anchor model Docker, QEMU and IOU use: every **Ethernet slot/port** owns a persistent TAP (`gd{node_id[:8]}e{slot}p{port}`), created by the server through uBridge's tap module, and links attach to the anchor — a per-link Linux bridge enslaves it, tc impairments run on it, capture and markers anchor on it. The Dynamips **hypervisor** opens the TAP with `nio create_tap` (QEMU's move: an external process holds the fd), unprivileged because the server handed the device's ownership to its own user. -``` - Dynamips hypervisor root namespace - ┌──────────────┐ nio create_tap ┌────────────────────────────────┐ - │ IOS router │◄────────────────►│ gd{id}e{slot}p{port} (TAP) │ - │ (emulated │ holds the fd │ │ per-link kernel bridge│ - │ ports) │ │ ▼ gns3{link_id} │ - └──────────────┘ └────────────────────────────────┘ +## Architecture + +The diagram shows the e2e shape — two c7200 routers on one link (slot 0 port 0 each), the topology `tests/e2e/test_dynamips_kernel_datapath.py` drives. A Dynamips↔Docker/QEMU/IOU link is the same shape with the peer's own anchor on the other side; a routed transit is two such links with IOS forwarding between them. Interface names follow the deterministic rules `gd{node_id[:8]}e{slot}p{port}` for the anchor TAPs and `gns3{link_id[:11]}` for the bridge; the concrete names in the diagrams use node ids `7c1d4e2f` / `9e8b6a3f` and link id prefix `5d3c2b1a4e6` (`gdA`/`gdB` shorthand in the sequence below). Terms the diagrams use: + +| Term | Meaning | +|---|---| +| hypervisor | the Dynamips process emulating the router; it holds every anchor TAP's fd | +| anchor | the host-side interface a link attaches to — for Dynamips the `gd` persistent TAP, for Docker the `gv` veth host end | +| persistent TAP | the TAP created per Ethernet slot/port at node start (`tap create` + `tap set_owner`, ownership handed to the server user so the unprivileged hypervisor can open it); born DOWN, it outlives `vm stop` and hypervisor crashes | +| per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | +| tap fd | a TAP's file descriptor: reads return frames the kernel received on the device, writes inject frames into it — the hypervisor opens it with `nio create_tap` and holds it for the port's lifetime | +| FDB | the bridge's forwarding database (learned MAC → port); unknown, broadcast and multicast destinations are flooded | +| `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | +| `clsact` | the tc classifier carrier on the anchor; its egress chain is where the drop classifiers run | +| eBPF classifier | the stateful drop modes (every-Nth, quota, time window) at clsact egress prio 1 | +| `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | +| netem | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | +| `AF_PACKET tap` | uBridge's packet socket on the anchor — how capture and markers observe frames off the forwarding path | +| carrier | the anchor's admin state (`link set up/down`); suspend is carrier down — traffic dead both ways | +| `nio_bridge` | the NIO record the controller posts to each endpoint node: bridge name, filters, markers, suspend flag | +| uBridge | the per-node helper process — on a kernel link it created the anchor, enslaves it and taps it, but never carries frames | + +```mermaid +flowchart LR + subgraph H1["Dynamips hypervisor R1"] + ios1["IOS router 1"] + end + subgraph RootNS["root namespace"] + gdA["gd7c1d4e2fe0p0"] + BR["gns35d3c2b1a4e6"] + gdB["gd9e8b6a3fe0p0"] + ub1["uBridge R1"] + ub2["uBridge R2"] + end + subgraph H2["Dynamips hypervisor R2"] + ios2["IOS router 2"] + end + ios1 <-->|"TAP fd"| gdA + gdA <-->|"bridge port"| BR + BR <-->|"bridge port"| gdB + gdB <-->|"TAP fd"| ios2 + gdA -.->|"AF_PACKET tap"| ub1 + ub1 -.->|"tc qdisc"| gdA + gdB -.->|"AF_PACKET tap"| ub2 + ub2 -.->|"tc qdisc"| gdB ``` -The legacy relay is untouched: a port without a kernel link keeps its -Dynamips UDP NIO and the node's uBridge relay tunnel exactly as before, and -a kernel link binds the same port to its anchor instead — per port, no -node-wide switch. +The only userspace crossings left are the Dynamips processes themselves — the routers *are* the emulation, and the hypervisor reads and writes the TAP directly (QEMU's external-fd-holder shape, no extra relay hop on the node leg unlike IOU's fabric). What the kernel datapath buys is the link segment: anchor → per-link bridge → peer anchor crosses entirely in the kernel, impairments run as kernel tc instead of userspace filters, and suspend maps to anchor admin-down. Each side's uBridge does its own tap-and-qdisc work on its node's anchor — but neither dashed edge is a steady fixture: the qdisc exists only while the link carries filters (deletion `tc reset`s it), the tap only while capture or markers run, and by convention only on the capture node's end, so an unfiltered, uncaptured link shows neither. The anchors themselves were created through uBridge at node start (Anchor lifecycle below). + +The legacy relay is untouched: a port without a kernel link keeps its Dynamips UDP NIO and the node's uBridge relay tunnel exactly as before, and a kernel link binds the same port to its anchor instead — per port, no node-wide switch. The same R1 ↔ R2 link on the relay — one direction drawn, the reverse is the mirror image; this is the exact shape the negative control keeps when `enable_kernel_datapath = false` (Verified below). Each relay bridge is named `DYNAMIPS-{lport}-{rport}` after the loopback port pair its NIO allocates at link time: + +```mermaid +flowchart LR + subgraph H1["Dynamips hypervisor R1"] + ios1["IOS router 1"] + nio1["UDP NIO"] + end + subgraph ub1["uBridge R1"] + rl1["relay bridge"] + end + subgraph KERN["kernel"] + lp1["loopback UDP"] + tun["UDP tunnel"] + lp2["loopback UDP"] + end + subgraph ub2["uBridge R2"] + rl2["relay bridge"] + end + subgraph H2["Dynamips hypervisor R2"] + nio2["UDP NIO"] + ios2["IOS router 2"] + end + ios1 --> nio1 + nio1 -->|"copy 1: user→kernel"| lp1 + lp1 -->|"copy 2: kernel→user"| rl1 + rl1 -->|"copy 3: user→kernel"| tun + tun -->|"copy 4: kernel→user"| rl2 + rl2 -->|"copy 5: user→kernel"| lp2 + lp2 -->|"copy 6: kernel→user"| nio2 + nio2 --> ios2 +``` + +Six kernel/user crossings each way, against the kernel datapath's two — and those two are the TAP-fd legs that are the emulation itself, so the relay's four extra copies are pure overhead. Userspace filters run inside the relay bridges, between copies 2-3 and 4-5. On a same-compute link the UDP tunnel rides loopback as drawn; cross-compute it rides the real network with the copy count unchanged. + +Frame path for one R1 → R2 frame. The impairment chain fires on the egress of the *destination* end's anchor, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and R2 → R1 is symmetric on gdA: + +```mermaid +flowchart LR + subgraph H1["Dynamips hypervisor R1"] + S1["IOS port sends"] + end + subgraph ROOT["root namespace"] + subgraph GA["gd7c1d4e2fe0p0 — ingress (egress chain serves R2→R1)"] + S3["ingress"] + end + subgraph BRP["per-link bridge"] + G1{"reserved 01:80:c2?"} + D["dropped"] + S4["FDB lookup / flood"] + end + subgraph GB["gd9e8b6a3fe0p0 — egress chain (clsact, then netem)"] + S5["eBPF prio 1"] + S6["bpf_drop prio 10-99"] + S7["netem qdisc"] + end + T["capture / markers"] + end + subgraph H2["Dynamips hypervisor R2"] + S9["IOS port receives"] + end + S1 -->|"tap fd, copy 1: user→kernel"| S3 + S3 --> G1 + G1 -->|"PAUSE 0x01"| D + G1 -->|"0xfffd passes"| S4 + S4 --> S5 + S5 --> S6 --> S7 + S7 -->|"tap fd, copy 2: kernel→user"| S9 + S3 -. "AF_PACKET tap" .-> T + S7 -. "AF_PACKET tap" .-> T +``` + +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The R2 → R1 reply is symmetric — its impairment points are `gd9e8b6a3fe0p0`'s egress (entering R2) and `gd7c1d4e2fe0p0`'s egress (entering R1) — so every direction is impaired exactly once, and `delay 100` measures ≈200 ms round trip (the e2e-verified number). Copies 1-2 are the whole kernel/user bill — both are the TAP-fd legs of the emulation itself; the same frame on the relay datapath crosses six times (diagram above). The dashed sideband is uBridge's observation: capture and markers run as AF_PACKET taps on whichever end hosts them, cloning this frame at that anchor's stage — gdA's rx or gdB's tx — and a clsact-dropped frame never reaches the tap. ## Datapath selection `UDPLink._kernel_datapath_eligible` extends the common rules with: -* the compute must report **`ubridge_tap`** — the anchors are uBridge-created - TAPs the hypervisor merely opens, the same capability QEMU gates on; - there is no Dynamips-specific one; -* **both ports must be Ethernet**. The controller's port matrix decides - per port; serial, ATM and POS ports keep the relay whatever the - capabilities (the adapter registry in - `compute/dynamips/adapters/adapter.py` names the Ethernet models — - `ETHERNET_ADAPTERS` / `ETHERNET_WICS`, WIC-1ENET included, at its - Dynamips port number `16 * (wic_slot + 1)`). +* the compute must report **`ubridge_tap`** — the anchors are uBridge-created TAPs the hypervisor merely opens, the same capability QEMU gates on; there is no Dynamips-specific one; +* **both ports must be Ethernet**. The controller's port matrix decides per port; serial, ATM and POS ports keep the relay whatever the capabilities (the adapter registry in `compute/dynamips/adapters/adapter.py` names the Ethernet models — `ETHERNET_ADAPTERS` / `ETHERNET_WICS`, WIC-1ENET included, at its Dynamips port number `16 * (wic_slot + 1)`). + +## Link creation sequence + +Both routers sit on the same compute (a kernel-link precondition) and each drives its own uBridge process; the hypervisor is one process per router. The two node handlers attach concurrently and compute the same bridge name independently (a pure function of the link id), so either may win the `brctl create` race — the loser verifies the bridge exists (`brctl show`) and continues: + +```mermaid +sequenceDiagram + participant API as REST API + participant C as Controller + participant R1 as Router R1 + participant HY1 as Hypervisor R1 + participant UB1 as uBridge R1 + participant R2 as Router R2 + participant HY2 as Hypervisor R2 + participant UB2 as uBridge R2 + + API->>C: POST /links + C->>C: eligibility check + C->>C: bridge = gns3 + link_id[:11] + C--)R1: nio_bridge + C--)R2: nio_bridge (same bridge) + par Router R1 + R1->>HY1: nio create_tap + R1->>HY1: vm slot_add_nio_binding + R1->>HY1: vm slot_enable_nio + R1->>UB1: brctl create + R1->>UB1: link set gns35d3c2b1a4e6 up + R1->>UB1: brctl addif gns35d3c2b1a4e6 gdA + R1->>UB1: capture start_kernel gdA + R1->>UB1: marker add_kernel gdA + R1->>UB1: tc netem set gdA + R1->>UB1: link set gdA up + and Router R2 + R2->>HY2: nio create_tap + R2->>UB2: same on gdB + end + Note over HY1,UB2: steady state — kernel forwards between the anchors, the hypervisors hold the tap fds +``` + +`nio create_tap`, `vm slot_add_nio_binding` and `vm slot_enable_nio` are the Dynamips-specific steps: the hypervisor opens the anchor and binds it to the slot/port before any bridge work, so a half-wired port never appears on the link. Capture, markers and `tc netem set` run only when the NIO carries them — a marker's real form is `marker add_kernel ""`, `bpf` expressions arrive as `tc bpf_drop add`, the stateful modes as `tc nth/quota/window_drop` — and `brctl addif` is itself what opens the port's `group_fwd_mask` to `0xfffd` (no separate command exists). The anchor's carrier stays down while the link is suspended. A link created while the router is **stopped** is stored and wired by the next start — the NIO simply waits in the slot adapter (deferred wiring, Anchor lifecycle below). ## Anchor lifecycle -* **Birth (node start)** — `_prepare_tap_datapath` probes the tap module - (one create/delete cycle) and creates one persistent TAP per Ethernet - slot/port, ownership handed to the server user, born DOWN. A stale - leftover from a previous run is swept first; an anchor that already - exists (restart) is kept — an anchor is never recreated under a live - link. Serial/ATM/POS ports simply have no anchor. -* **Life** — a kernel link opens the anchor in the hypervisor - (`nio create_tap`), binds it to the slot/port and enslaves it into the - per-link bridge; removing the link unbinds the port, deletes the - hypervisor's TAP NIO (releasing the fd) and tears the anchor out of the - bridge. A link bound while the router is **stopped** is stored and wired - by the next start (deferred wiring — the NIO simply waits in the slot - adapter). -* **Death (node close)** — `_stop_ubridge` runs in **QEMU's order**, not - IOU's: here the Dynamips hypervisor holds the anchor fds, so its TAP - NIOs go first (`nio delete` closes the fd), then `tap delete` per anchor, - then the per-link kernel bridges, then uBridge itself. +* **Birth (node start)** — `_prepare_tap_datapath` probes the tap module (one create/delete cycle) and creates one persistent TAP per Ethernet slot/port, ownership handed to the server user, born DOWN. A stale leftover from a previous run is swept first; an anchor that already exists (restart) is kept — an anchor is never recreated under a live link. Serial/ATM/POS ports simply have no anchor. +* **Life** — a kernel link opens the anchor in the hypervisor (`nio create_tap`), binds it to the slot/port and enslaves it into the per-link bridge; removing the link unbinds the port, deletes the hypervisor's TAP NIO (releasing the fd) and tears the anchor out of the bridge. A link bound while the router is **stopped** is stored and wired by the next start (deferred wiring — the NIO simply waits in the slot adapter). +* **Death (node close)** — `_stop_ubridge` runs in **QEMU's order**, not IOU's: here the Dynamips hypervisor holds the anchor fds, so its TAP NIOs go first (`nio delete` closes the fd), then `tap delete` per anchor, then the per-link kernel bridges, then uBridge itself. -**A stop is not a close.** `stop` only halts the emulated router: the -hypervisor process — and with it the TAP fds, the port bindings and the -per-link bridges — survives `vm stop` (the same reason a relay link's -tunnel always survived a stop). Links therefore keep working through a -stop/start with no re-wiring at all; the restart's attach pass skips every -port that still has its TAP NIO. If the hypervisor died mid-flight (crash -or kill), the persistent TAPs remain and the next start re-attaches from -the stored NIOs. +**A stop is not a close.** `stop` only halts the emulated router: the hypervisor process — and with it the TAP fds, the port bindings and the per-link bridges — survives `vm stop` (the same reason a relay link's tunnel always survived a stop). Links therefore keep working through a stop/start with no re-wiring at all; the restart's attach pass skips every port that still has its TAP NIO. If the hypervisor died mid-flight (crash or kill), the persistent TAPs remain and the next start re-attaches from the stored NIOs. ## Link operations @@ -86,78 +197,35 @@ the stored NIOs. ## Capture, markers, filters -Capture on a kernel link is `capture start_kernel `; markers ride -the base reconcile with the anchor as the location (`marker add_kernel` — -the shared KernelDatapathMixin machinery, no Dynamips-specific code); -filters are one tc netem qdisc per anchor plus cls_bpf/eBPF drops. A relay -port keeps its Dynamips NIO and the node's uBridge relay bridge -(`DYNAMIPS-*`) with userspace filters, exactly as before. +Capture on a kernel link is `capture start_kernel `; markers ride the base reconcile with the anchor as the location (`marker add_kernel` — the shared KernelDatapathMixin machinery, no Dynamips-specific code); filters are one tc netem qdisc per anchor plus cls_bpf/eBPF drops. A relay port keeps its Dynamips NIO and the node's uBridge relay bridge (`DYNAMIPS-*`) with userspace filters, exactly as before. ## Link-local frames (LACP, LLDP, 802.1X, STP) -The bridge stands in for a cable, but the kernel's `br_handle_frame()` does -not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by -default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while -the UDP relay carries them fine. uBridge therefore opens every port's -per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at -`brctl addif` time (Linux 4.15+, best-effort on older kernels). The -bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, -so LACP is reachable per port only. +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. -Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and -LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in -`br_handle_frame()` drops it unconditionally and no mask can enable it. -That is a documented limit, not a regression (PFC's hardware semantics are -out of emulation's reach anyway); a lab that needs PAUSE frames on the wire -must use a relay link. Verified live by -`tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest -matrix); the contract and kernel references live in -`docs/design/ubridge-link-local-frame-forwarding-spec.md`. +Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in `br_handle_frame()` drops it unconditionally and no mask can enable it. That is a documented limit, not a regression (PFC's hardware semantics are out of emulation's reach anyway); a lab that needs PAUSE frames on the wire must use a relay link. Verified live by `tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest matrix); the contract and kernel references live in `docs/design/ubridge-link-local-frame-forwarding-spec.md`. ## Relay fallback -A uBridge without the tap module keeps the node relay-only: `_prepare_tap_datapath` -warns, no anchors are created, the warning says the node cannot carry -kernel links, and every link keeps its `nio create_udp` tunnel. The -controller never offers such a node a kernel link (capability unreported). +A uBridge without the tap module keeps the node relay-only: `_prepare_tap_datapath` warns, no anchors are created, the warning says the node cannot carry kernel links, and every link keeps its `nio create_udp` tunnel. The controller never offers such a node a kernel link (capability unreported). ## Verified -Unit level (`tests/compute/dynamips/test_dynamips_kernel_datapath.py`): -anchor topology (Ethernet models and WIC-1ENET port numbering only), the -lifecycle (create/sweep/skip-existing), deferred wiring of NIOs bound -while stopped, attach/update/remove command shapes on both datapaths, the -stop order (hypervisor NIO delete before tap delete before brctl delete), -capture and marker shapes, hot-added adapters, and the manager's -`nio_bridge` construction. Plus the controller-eligibility cases (mixable -with docker/qemu/iou, tap capability required, serial excluded) in -`tests/controller/test_kernel_datapath_link.py`. +Unit level (`tests/compute/dynamips/test_dynamips_kernel_datapath.py`): anchor topology (Ethernet models and WIC-1ENET port numbering only), the lifecycle (create/sweep/skip-existing), deferred wiring of NIOs bound while stopped, attach/update/remove command shapes on both datapaths, the stop order (hypervisor NIO delete before tap delete before brctl delete), capture and marker shapes, hot-added adapters, and the manager's `nio_bridge` construction. Plus the controller-eligibility cases (mixable with docker/qemu/iou, tap capability required, serial excluded) in `tests/controller/test_kernel_datapath_link.py`. -End-to-end (`tests/e2e/test_dynamips_kernel_datapath.py`, pytest marker -`e2e`) on a live server with **two real c7200 routers on a real IOS image**, -driven through the REST API and the IOS consoles with real ICMP: +End-to-end (`tests/e2e/test_dynamips_kernel_datapath.py`, pytest marker `e2e`) on a live server with **two real c7200 routers on a real IOS image**, driven through the REST API and the IOS consoles with real ICMP: -* a link created while both routers are stopped selects the kernel - datapath and is wired by node start (deferred wiring); -* the per-link kernel bridge on the host has exactly the two anchor TAPs - enslaved; `show ip int brief` is up/up and ping R1→R2 succeeds; -* `delay 100` ⇒ netem qdisc visible on both anchors and the ping RTT - grows by ~200 ms (one-way 100 per direction, each anchor impairing its - own ingress side); clearing resets the qdiscs and the RTT; +* a link created while both routers are stopped selects the kernel datapath and is wired by node start (deferred wiring); +* the per-link kernel bridge on the host has exactly the two anchor TAPs enslaved; `show ip int brief` is up/up and ping R1→R2 succeeds; +* `delay 100` ⇒ netem qdisc visible on both anchors and the ping RTT grows by ~200 ms (one-way 100 per direction, each anchor impairing its own ingress side); clearing resets the qdiscs and the RTT; * suspend ⇒ anchor administratively down, ping 0%; resume restores both; * capture writes a pcap full of ICMP-over-Ethernet records; -* a **serial** link between the same two routers stays on the relay - (per-port exclusion) and pings over it — the relay regression net; -* link delete/re-create: the bridge goes and comes back, the anchors - survive, traffic resumes; -* router stop/start keeps the wiring intact (bridge membership unchanged, - no duplicate NIOs) and traffic resumes after the reboot; +* a **serial** link between the same two routers stays on the relay (per-port exclusion) and pings over it — the relay regression net; +* link delete/re-create: the bridge goes and comes back, the anchors survive, traffic resumes; +* router stop/start keeps the wiring intact (bridge membership unchanged, no duplicate NIOs) and traffic resumes after the reboot; * deleting the project removes every anchor and bridge. -`test_dynamips_relay_control` is the negative control: the same topology on -an isolated relay-configured instance (`enable_kernel_datapath = false`) -has no per-link bridge and nothing enslaved, and still pings — so the -kernel objects above can only come from the kernel datapath. +`test_dynamips_relay_control` is the negative control: the same topology on an isolated relay-configured instance (`enable_kernel_datapath = false`) has no per-link bridge and nothing enslaved, and still pings — so the kernel objects above can only come from the kernel datapath. ## Configuration @@ -170,9 +238,4 @@ enable_kernel_datapath = True ## Roadmap -The Ethernet switch has landed (see `ethernet-switch-kernel-datapath.md` — -it absorbs the peer's anchor into its own kernel bridge; switch-to-switch -cascades and the IOL runner container have landed too — see -`ethernet-switch-kernel-datapath.md` and -`iol-docker-kernel-datapath.md`). Cross-compute kernel links need -VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. +The Ethernet switch has landed (see `ethernet-switch-kernel-datapath.md` — it absorbs the peer's anchor into its own kernel bridge; switch-to-switch cascades and the IOL runner container have landed too — see `ethernet-switch-kernel-datapath.md` and `iol-docker-kernel-datapath.md`). Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index 4df8974a6..2a544d3f5 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -121,22 +121,53 @@ Drawn one-way; the reverse direction is the mirror image. The IOL relay crosses Two further contrasts with Docker's relay: the IOL relay's local leg is the unix NIO itself, so the anchors never appear on the path (idle from birth — a Docker relay attaches to the *same* veth host end the kernel link enslaves); and the port bridge's first slot is shared with the kernel datapath — the unix NIO and its c-socket binding survive every leg swap, so frames the container sends while the bridge is stopped for a swap queue on the socket and relay after `start`. On this datapath suspend rides the synthetic frequency_drop filter (the kernel datapath's suspend is the anchor's admin state). -Frame path for one A → B frame. The impairment chain fires on the egress of the *destination* anchor, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gxA — hence `delay 100` doubles the RTT (measured ≥ 150 ms): +Frame path for one A → B frame, with the relay diagram's copy numbering: copies 1, 2, 5 and 6 are the socket guest legs and are paid identically on both datapaths — only the middle two swap (TAP fd write/read here, loopback UDP there). The impairment chain fires on the egress of the *destination* anchor, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gxA's egress chain — hence `delay 100` doubles the RTT (measured ≥ 150 ms): ```mermaid flowchart LR - S1["IOL A sends on Eth0/0"] --> S2["netiomux writes c00.sock"] - S2 --> S3["port bridge A relay"] - S3 --> S4["gxA ingress"] - S4 --> G1{"reserved 01:80:c2?"} - G1 -->|"PAUSE 0x01"| D["dropped"] - G1 -->|"0xfffd passes"| S5["FDB lookup / flood"] - S5 --> S6["clsact: eBPF prio 1"] - S6 --> S7["clsact: bpf_drop prio 10-99"] - S7 --> S8["netem qdisc"] - S8 --> S9["port bridge B relay"] - S9 --> S10["netiomux B via s00.sock"] - S10 --> S11["IOL B receives on Eth0/0"] + subgraph CTA["IOL A container — userspace"] + S1["IOL A sends on Eth0/0"] + end + subgraph KERN["kernel"] + SOCKA["c00.sock"] + SOCKB["s00.sock"] + end + subgraph UBA["uBridge port bridge A — userspace"] + S2["relay"] + end + subgraph GXA["gxA — ingress (egress chain serves B→A)"] + S4["ingress"] + end + subgraph BRP["per-link bridge"] + G1{"reserved 01:80:c2?"} + D["dropped"] + S5["FDB lookup / flood"] + end + subgraph GXB["gxB — egress chain (clsact, then netem)"] + S6["eBPF prio 1"] + S7["bpf_drop prio 10-99"] + S8["netem qdisc"] + end + subgraph UBB["uBridge port bridge B — userspace"] + S9["relay"] + end + subgraph CTB["IOL B container — userspace"] + S11["IOL B receives on Eth0/0"] + end + tap["capture / markers"] + S1 -->|"copy 1: user→kernel"| SOCKA + SOCKA -->|"copy 2: kernel→user"| S2 + S2 -->|"copy 3: user→kernel"| S4 + S4 --> G1 + G1 -->|"PAUSE 0x01"| D + G1 -->|"0xfffd passes"| S5 + S5 --> S6 + S6 --> S7 --> S8 + S8 -->|"copy 4: kernel→user"| S9 + S9 -->|"copy 5: user→kernel"| SOCKB + SOCKB -->|"copy 6: kernel→user"| S11 + S4 -.->|"AF_PACKET tap"| tap + S8 -.->|"AF_PACKET tap"| tap ``` The reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The AF_PACKET taps observe both anchors but never a clsact-dropped frame. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 60844968b..5dbeeaf41 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -115,10 +115,8 @@ flowchart LR subgraph UBA["uBridge A — userspace"] S2["relay"] end - subgraph GIA["giA — ingress"] + subgraph GIA["giA — ingress (its egress chain serves B→A)"] S4["ingress"] - chA["clsact + netem chain"] - tapA["AF_PACKET tap: capture + markers"] end subgraph BRP["per-link bridge"] G1{"reserved 01:80:c2?"} @@ -128,7 +126,7 @@ flowchart LR subgraph GIB["giB — egress chain (clsact, then netem)"] S6["eBPF prio 1"] S7["bpf_drop prio 10-99"] - S8["netem"] + S8["netem qdisc"] end subgraph UBB["uBridge B — userspace"] S9["relay"] @@ -147,8 +145,9 @@ flowchart LR S8 -->|"copy 4: kernel→user"| S9 S9 -->|"copy 5: user→kernel"| mbxB mbxB -->|"copy 6: kernel→user"| S11 - S4 -.->|"serves B→A"| chA - S4 -.->|"observes tx + rx"| tapA + tap["capture / markers"] + S4 -.->|"AF_PACKET tap"| tap + S8 -.->|"AF_PACKET tap"| tap ``` What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below); the AF_PACKET tap — capture and markers ride it, installed at whichever end captures — observes both directions of the link through its anchor (tx/rx signal pairs) but never a clsact-dropped frame. The crossings do not disappear on this datapath: each leg still crosses the kernel/user boundary three times (`sendto`/`recvfrom` on the fabric, then the TAP write — the B leg mirrors them, six per one-way frame, the same count as the relay); what moves into the kernel is the link segment — no UDP tunnel, no protocol-stack pass — and with it the impairment chain. A fabric → anchor → bridge → anchor → fabric round trip measures ≈0.2 ms (Verified below). diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 84dc8bf5f..703bd1164 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -139,8 +139,9 @@ flowchart LR subgraph GQB["gqB — egress chain (clsact, then netem)"] S5["eBPF prio 1"] S6["bpf_drop prio 10-99"] - S7["netem"] + S7["netem qdisc"] end + T["capture / markers"] end subgraph QPB["QEMU B — userspace"] S8["tap fd to QEMU"] @@ -149,17 +150,19 @@ flowchart LR S9["NIC receives"] end S1 -->|"virtio"| S2 - S2 --> S3 + S2 -->|"tap fd, copy 1: user→kernel"| S3 S3 --> G1 G1 -->|"PAUSE 0x01"| D G1 -->|"0xfffd passes"| S4 S4 --> S5 S5 --> S6 --> S7 - S7 --> S8 + S7 -->|"tap fd, copy 2: kernel→user"| S8 S8 -->|"virtio IRQ"| S9 + S3 -. "AF_PACKET tap" .-> T + S7 -. "AF_PACKET tap" .-> T ``` -What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The AF_PACKET taps observe both anchors but never a clsact-dropped frame. +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The AF_PACKET taps observe both anchors but never a clsact-dropped frame. Copies 1–2 are the whole path's only kernel/user crossings — both halves of the emulation's own TAP-fd leg (QEMU A's write, QEMU B's read); every stage between them runs in the kernel. ## Why a TAP works like a veth host end From 7a9a514ea313c5074a54b7309d9e3e208bef39f3 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sun, 4 Oct 2026 00:34:47 +0800 Subject: [PATCH 075/101] =?UTF-8?q?docs:=20kernel-datapath=20family=20revi?= =?UTF-8?q?ew=20pass=20=E2=80=94=20accuracy=20fixes=20and=20pinned=20claim?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A correctness read-through of the five delivered datapath documents, plus the two code comments that carried the same stale vocabulary. Datapath selection: the VendorDockerVM selector is `console_type=docker_exec` alone (`GNS3_SKIP_INIT` and friends are behavior knobs of the chosen class); the cross-compute and anchor-less fallbacks are named for what they are; the Ethernet-switch endpoint is the documented exception to the anchoring rule, and the IOL runner container is called out as the Docker node that does not anchor on a veth. Filter vocabulary: the stateful eBPF program evaluates four modes (nth → quota → window → flow), GNS3 exposes three of them, and window_drop joins the kernel-only list; the correl suffixes gate on the `rate` token as the extension-build marker. group_fwd_mask: the 0xfffd port mask is applied inside uBridge's `brctl addif` itself, not as a separate command — the link-creation sequences and the operation tables no longer show an arrow the server never sends. Same class of fix for `link set up` on the IOL anchor (addif already brought it up; the command only pushes a suspended anchor back down) and for the QEMU carrier replication (QMP `set_link`, gated by `replicate_network_connection_state`). Lifecycles and stop orders re-worded around the invariant rather than the node type: the fd holder must release before the devices are deleted, which is QEMU's process-first order, IOU's uBridge-first order and Dynamips' hypervisor-NIO-first order. Deferred wiring for a stopped endpoint is documented as the common path it is, and the Dynamips "stop is not a close" note now says what recovery from a dead hypervisor actually requires. Dynamips specifics: a c7200 needs an adapter in slot 1 (slot 0 is the fixed I/O slot), the tap-fd direction is stated correctly (writes inject into the kernel, reads return what the kernel transmits), and the anchor registry is name-keyed against the controller's port matrix by convention, not derivation. IOL port bridge naming: `bridge{bay}` on unit 0, `bridge{bay}_{unit}` elsewhere; the leg swap happens on link delete/re-create or a datapath switch, while suspend is an in-place NIO update and a project reopen is a fresh build. Verified and Roadmap sections: the in-repo e2e suite is distinguished from the isolated-instance development harness (IOU), assertions are stated with their bounds and capability gates, and the roadmap entries collapse into the one remaining project — VXLAN/GENEVE for cross-compute links. Code comments: `flow` added to the documented mode order in the eBPF apply path, and the `nio_bridge` module docstring corrected on which filter types run on the kernel datapath (bpf via cls_bpf, frequency_drop/quota/window_drop via the eBPF classifier — neither is relay-only any more). --- docs/features/docker-kernel-datapath.md | 21 +++++++------- docs/features/dynamips-kernel-datapath.md | 32 ++++++++++----------- docs/features/iol-docker-kernel-datapath.md | 16 +++++------ docs/features/iou-kernel-datapath.md | 24 +++++++++------- docs/features/qemu-kernel-datapath.md | 23 +++++++-------- gns3server/compute/kernel_datapath.py | 2 +- gns3server/compute/nios/nio_bridge.py | 7 +++-- 7 files changed, 63 insertions(+), 62 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 711d856c9..8e36cd1ce 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -39,11 +39,11 @@ Which interface a Docker node's adapters use is decided by the node class (`Dock | Node class | Selected by | Adapter interface | Kernel links | |---|---|---|---| | `DockerVM` | default | veth pair | yes | -| `VendorDockerVM` (non-unix-socket) | `console_type=docker_exec`, `GNS3_SKIP_INIT`, … (XRd, SR Linux, …) | veth pair (same path via `super()`) | yes | +| `VendorDockerVM` (non-unix-socket) | `console_type=docker_exec` (XRd, SR Linux, …) | veth pair (same path via `super()`) | yes | | `IOLDockerVM` | `GNS3_IOL_RUNNER=1` | AF_UNIX datagram socket pairs (`sNN`/`cNN`) — the container netns is unused; kernel links ride persistent `gx` TAP anchors through the port bridge's swappable TAP leg (see `iol-docker-kernel-datapath.md`) | yes (gated on `ubridge_bridge_tap` + `ubridge_tap`) | | `VendorDockerVM` (unix-socket) | `GNS3_UNIX_SOCKET_NIO=1` | same socket contract, generic capability for vendor NOS images | rejected | -Unix-socket containers are detected both in the controller (`_is_unix_socket_docker`, link eligibility) and on the compute side (NIO rejection) — a missed detection surfaces as a clearer-late error. +The other `GNS3_*` markers a vendor image sets (`GNS3_SKIP_INIT`, `GNS3_INTERFACE_NAMES`, …) are behavior knobs of the selected class, not class selectors — an environment carrying only `GNS3_SKIP_INIT` stays on `DockerVM`. Unix-socket containers are detected both in the controller (`_is_unix_socket_docker`, link eligibility) and on the compute side (NIO rejection) — a missed detection surfaces as a clearer-late error, except on the Ethernet-switch fast path, where an absent anchor is deferred forever by design (the join must not fail while the peer is stopped), leaving a silently dead cable. ## Architecture @@ -61,7 +61,7 @@ Kernel link between two containers on the same compute. Interface names follow t | `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | | tc / qdisc | traffic-control objects the kernel inserts into the host end's egress path — stages frames pass through in flight, not external packet consumers; uBridge only installs them over netlink | | `clsact` | the tc classifier carrier on the host end; its egress chain is where the drop classifiers run | -| eBPF classifier | the stateful drop modes (every-Nth, quota, time window) at clsact egress prio 1 | +| eBPF classifier | the stateful drop classifier at clsact egress prio 1 — modes every-Nth, quota, time window and flow (GNS3 exposes the first three as filter types) | | `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | | `netem` | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | | `AF_PACKET tap` | uBridge's packet socket bound to the host end; the kernel clones every frame the interface sends or receives to it (the tcpdump mechanism) — read-only copies, the original stays on the kernel path | @@ -97,7 +97,7 @@ Everything in the root namespace is built the same way: gns3-server creates the uBridge sits off the forwarding path — the dashed edges are control and observation, never data flow. The `qdisc install` edge is uBridge configuring the kernel over netlink; the kernel then runs those qdiscs in-flight on the host end's egress (they are stages of the transmit path, not packet consumers). The `frame copies` edge is the AF_PACKET tap: bound to the host end, the kernel clones every frame the interface sends or receives to uBridge's packet socket (the tcpdump mechanism) — read-only copies; the originals never leave the kernel. -Relay link on the same unified veth and the same single compute as above — what the link becomes when `enable_kernel_datapath` is off (mixed node types and cross-compute wiring fall back too; there the UDP hop rides the real network instead of loopback, the copy count unchanged): +Relay link on the same unified veth and the same single compute as above — what the link becomes when `enable_kernel_datapath` is off (cross-compute wiring and an endpoint that cannot anchor — VPCS, a cloud node, … — fall back too; across computes the UDP hop rides the real network instead of loopback, the copy count unchanged): ```mermaid flowchart LR @@ -175,7 +175,7 @@ flowchart LR What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Packet filters below). Whichever end hosts capture / markers, its AF_PACKET tap — the dashed sideband above — clones the frame at that anchor: a tap on gvA sees the frame on arrival, a tap on gvB only if the clsact chain let it through. -All dozen-plus GNS3 filter types ride these three kernel objects — nine of them (delay, packet_loss, corrupt, duplicate, rate, reorder, gemodel, seed, limit) are parameters of the single netem qdisc, `bpf` expressions are the cls_bpf filters, and frequency_drop / quota / window_drop are the three modes inside the one eBPF program (evaluated nth → quota → window). The complete type-to-object mapping is the Packet filters table below. +All dozen-plus GNS3 filter types ride these three kernel objects — nine of them (delay, packet_loss, corrupt, duplicate, rate, reorder, gemodel, seed, limit) are parameters of the single netem qdisc, `bpf` expressions are the cls_bpf filters, and frequency_drop / quota / window_drop map to three of the one eBPF program's four modes (evaluated nth → quota → window → flow; the flow-hash mode has no GNS3 filter type). The complete type-to-object mapping is the Packet filters table below. ## Datapath selection @@ -212,8 +212,7 @@ sequenceDiagram CA->>UA: brctl create CA->>UA: link set bridge up CA->>UA: brctl addif gvA - CA->>UA: port fwd_mask 0xfffd - CA->>UA: markers / capture if any + CA->>UA: capture / markers if any CA->>UA: tc netem + bpf + eBPF CA->>UA: link set gvA up and Docker node B @@ -237,7 +236,7 @@ The veth pairs already exist at attach time — every adapter is born as a veth | Operation | Kernel-datapath action | |---|---| -| Create / attach | `brctl create` (EEXIST tolerated via `brctl show` verify) + `link set up` + `brctl addif` both ends (concurrent, race-tolerant) + carrier up + markers + `tc netem set` (filters) | +| Create / attach | `brctl create` (EEXIST tolerated via `brctl show` verify) + `link set up` + `brctl addif` both ends (concurrent, race-tolerant) + capture / markers if any + the tc chain (netem, bpf, eBPF) + carrier up | | Delete / detach | marker teardown → `tc reset` (netem teardown — the veth survives the link) → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The veth survives — unlike a relay bridge, whose death dropped its filters, an orphaned AF_PACKET marker would keep sniffing, hence the explicit teardown | | Reset (`POST /links/{id}/reset`) | delete + create (re-evaluates eligibility) | | Suspend (`PUT /links/{id}` `{"suspend": true}`) | veth host end admin-state down both ends — 100 % loss, no synthetic filter needed; resume restores (netem qdisc survives the carrier flap) | @@ -284,10 +283,10 @@ Semantics: * **Both endpoints** receive the filter dict and attach one qdisc to *their* veth host end. The qdisc's egress covers traffic entering that container, so every direction of the link is impaired exactly once — the same net effect as the relay, where both directions cross the single filtered bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the per-packet serialization time to the RTT. * **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES optional attributes (rate, correlation, reorder, corrupt, gemodel, distribution: an absent attr keeps its previous value), so re-applying a filter set with a parameter *removed* would silently keep the old value. Every apply therefore starts with `tc reset` (also drops clsact and its bpf_drop filters — re-added right after in the same flow) followed by one `netem set` built from the current filters; no per-parameter diffing. An empty filter set is the reset alone (ENOENT-tolerated). -* **Extension gating.** The netem-extension keywords (rate, reorder, gemodel, dist, seed, limit, and the correl suffixes) probe `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. -* **kernel-only vs relay-available.** The extension types and `quota` have no uBridge relay equivalent: `available_filters` hides them on relay links, setting one on a created relay link returns 409, and a project loaded with such filters on a link that cannot be kernel-wired drops them with a warning (invalid filters are dropped the same way at load). `frequency_drop` runs on both datapaths (relay userspace filter / eBPF every-Nth). +* **Extension gating.** The netem-extension keywords (rate, reorder, gemodel, dist, seed, limit — plus the correl suffixes, which gate on the `rate` token as the extension-build marker) probe `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. +* **kernel-only vs relay-available.** The extension types, `quota` and `window_drop` have no uBridge relay equivalent: `available_filters` hides them on relay links, setting one on a created relay link returns 409, and a project loaded with such filters on a link that cannot be kernel-wired drops them with a warning (invalid filters are dropped the same way at load). `frequency_drop` runs on both datapaths (relay userspace filter / eBPF every-Nth). * **frequency_drop counting semantics differ per datapath.** The relay's single filtered bridge counts packets of BOTH directions through one counter; the kernel attaches one classifier per veth end, so each direction drops every Nth independently. For a round-trip measurement with every-Nth N: relay ≈ 1/N of frames (phase-locked), kernel = 1 − ((N−1)/N)² (e.g. N=3 → 55.6 % round-trip loss, measured 57 %). The kernel count is exact (atomic counter), unlike netem's stochastic loss. -* **Classifier order** on clsact egress: the eBPF stateful filter runs at prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the stateful modes never reaches the expression drops, and dropped packets never reach netem. Within the stateful program the modes evaluate in the fixed order nth → quota → window. +* **Classifier order** on clsact egress: the eBPF stateful filter runs at prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the stateful modes never reaches the expression drops, and dropped packets never reach netem. Within the stateful program the modes evaluate in the fixed order nth → quota → window → flow. * **window_drop semantics.** `start` is relative to the moment the filter is applied — and because reconcile re-sets the classifier on every apply (any filter change, node restart, link reset reloads the program after the netem reset), every such event **restarts the schedule**. 3 parameters = one single outage (`[start, start+outage)`), traffic passes before and after — the classic "link dies at T, comes back at T+outage". 4 parameters = recurring flap: `period ≥ outage` (validated), so the outage occupies `outage/period` of each cycle. 5 parameters = randomized flap: each cycle's outage and period are re-drawn uniformly in nominal ± `jitter` (integer-ms grid; jitter 0 draws nothing and equals the fixed schedule). Inside a window each packet drops with the given chance (100 = full outage, lower = degraded service during the window). Both endpoints run their own schedule, started ~simultaneously by the controller — a round trip survives only when **both** directions are outside their windows. Note `start = 0` (outage begins immediately) is a valid, active configuration — unlike other filters, a zero first parameter does not mean "disabled". A PUT re-sending an unchanged filters dict is normally a controller no-op, but one still carrying `window_drop` is reconciled anyway, so re-applying the same window re-arms a one-shot outage that has already passed (arm the filter immediately before generating traffic, or use a period for a repeatable event). * **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + unit ≤ 100gbit. * **Node restart** restores the qdisc from the NIO (like capture and markers); **link deletion** detaches it explicitly — the veth survives the link and an orphaned qdisc would keep impairing the next one. diff --git a/docs/features/dynamips-kernel-datapath.md b/docs/features/dynamips-kernel-datapath.md index 5f844017d..4d433e779 100644 --- a/docs/features/dynamips-kernel-datapath.md +++ b/docs/features/dynamips-kernel-datapath.md @@ -9,7 +9,7 @@ Dynamips routers emulate every port inside the Dynamips process; on the relay da ## Architecture -The diagram shows the e2e shape — two c7200 routers on one link (slot 0 port 0 each), the topology `tests/e2e/test_dynamips_kernel_datapath.py` drives. A Dynamips↔Docker/QEMU/IOU link is the same shape with the peer's own anchor on the other side; a routed transit is two such links with IOS forwarding between them. Interface names follow the deterministic rules `gd{node_id[:8]}e{slot}p{port}` for the anchor TAPs and `gns3{link_id[:11]}` for the bridge; the concrete names in the diagrams use node ids `7c1d4e2f` / `9e8b6a3f` and link id prefix `5d3c2b1a4e6` (`gdA`/`gdB` shorthand in the sequence below). Terms the diagrams use: +The diagram shows the e2e shape — two c7200 routers on one link (slot 1 port 0 each, a PA-2FE-TX — slot 0 is the c7200's fixed I/O slot), the topology `tests/e2e/test_dynamips_kernel_datapath.py` drives. A Dynamips↔Docker/QEMU/IOU link is the same shape with the peer's own anchor on the other side; a routed transit is two such links with IOS forwarding between them. Interface names follow the deterministic rules `gd{node_id[:8]}e{slot}p{port}` for the anchor TAPs and `gns3{link_id[:11]}` for the bridge; the concrete names in the diagrams use node ids `7c1d4e2f` / `9e8b6a3f` and link id prefix `5d3c2b1a4e6` (`gdA`/`gdB` shorthand in the sequence below). Terms the diagrams use: | Term | Meaning | |---|---| @@ -17,7 +17,7 @@ The diagram shows the e2e shape — two c7200 routers on one link (slot 0 port 0 | anchor | the host-side interface a link attaches to — for Dynamips the `gd` persistent TAP, for Docker the `gv` veth host end | | persistent TAP | the TAP created per Ethernet slot/port at node start (`tap create` + `tap set_owner`, ownership handed to the server user so the unprivileged hypervisor can open it); born DOWN, it outlives `vm stop` and hypervisor crashes | | per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | -| tap fd | a TAP's file descriptor: reads return frames the kernel received on the device, writes inject frames into it — the hypervisor opens it with `nio create_tap` and holds it for the port's lifetime | +| tap fd | a TAP's file descriptor: writes inject frames into the kernel as if the device received them, reads return frames the kernel transmits through the device (bridge-forwarded, toward the router) — the hypervisor opens it with `nio create_tap` and holds it for the port's lifetime | | FDB | the bridge's forwarding database (learned MAC → port); unknown, broadcast and multicast destinations are flooded | | `01:80:c2` / `group_fwd_mask` | the IEEE 802.1D reserved multicast range (STP, LACP, LLDP, 802.1X, PAUSE); the bridge forwards it only per the port's `group_fwd_mask` — `0xfffd` passes everything except PAUSE/PFC | | `clsact` | the tc classifier carrier on the anchor; its egress chain is where the drop classifiers run | @@ -35,9 +35,9 @@ flowchart LR ios1["IOS router 1"] end subgraph RootNS["root namespace"] - gdA["gd7c1d4e2fe0p0"] + gdA["gd7c1d4e2fe1p0"] BR["gns35d3c2b1a4e6"] - gdB["gd9e8b6a3fe0p0"] + gdB["gd9e8b6a3fe1p0"] ub1["uBridge R1"] ub2["uBridge R2"] end @@ -99,7 +99,7 @@ flowchart LR S1["IOS port sends"] end subgraph ROOT["root namespace"] - subgraph GA["gd7c1d4e2fe0p0 — ingress (egress chain serves R2→R1)"] + subgraph GA["gd7c1d4e2fe1p0 — ingress (egress chain serves R2→R1)"] S3["ingress"] end subgraph BRP["per-link bridge"] @@ -107,7 +107,7 @@ flowchart LR D["dropped"] S4["FDB lookup / flood"] end - subgraph GB["gd9e8b6a3fe0p0 — egress chain (clsact, then netem)"] + subgraph GB["gd9e8b6a3fe1p0 — egress chain (clsact, then netem)"] S5["eBPF prio 1"] S6["bpf_drop prio 10-99"] S7["netem qdisc"] @@ -128,18 +128,18 @@ flowchart LR S7 -. "AF_PACKET tap" .-> T ``` -What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The R2 → R1 reply is symmetric — its impairment points are `gd9e8b6a3fe0p0`'s egress (entering R2) and `gd7c1d4e2fe0p0`'s egress (entering R1) — so every direction is impaired exactly once, and `delay 100` measures ≈200 ms round trip (the e2e-verified number). Copies 1-2 are the whole kernel/user bill — both are the TAP-fd legs of the emulation itself; the same frame on the relay datapath crosses six times (diagram above). The dashed sideband is uBridge's observation: capture and markers run as AF_PACKET taps on whichever end hosts them, cloning this frame at that anchor's stage — gdA's rx or gdB's tx — and a clsact-dropped frame never reaches the tap. +What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Capture, markers, filters below). The R2 → R1 reply is symmetric — its impairment points are `gd9e8b6a3fe1p0`'s egress (entering R2) and `gd7c1d4e2fe1p0`'s egress (entering R1) — so every direction is impaired exactly once, and `delay 100` adds one-way 100 ms per direction, ≈200 ms round trip (the e2e pins the lower bound: RTT ≥ 150 ms with 100% success). Copies 1-2 are the whole kernel/user bill — both are the TAP-fd legs of the emulation itself; the same frame on the relay datapath crosses six times (diagram above). The dashed sideband is uBridge's observation: capture and markers run as AF_PACKET taps on whichever end hosts them, cloning this frame at that anchor's stage — gdA's rx or gdB's tx — and a clsact-dropped frame never reaches the tap. ## Datapath selection `UDPLink._kernel_datapath_eligible` extends the common rules with: * the compute must report **`ubridge_tap`** — the anchors are uBridge-created TAPs the hypervisor merely opens, the same capability QEMU gates on; there is no Dynamips-specific one; -* **both ports must be Ethernet**. The controller's port matrix decides per port; serial, ATM and POS ports keep the relay whatever the capabilities (the adapter registry in `compute/dynamips/adapters/adapter.py` names the Ethernet models — `ETHERNET_ADAPTERS` / `ETHERNET_WICS`, WIC-1ENET included, at its Dynamips port number `16 * (wic_slot + 1)`). +* **both ports must be Ethernet**. The controller's port matrix decides per port (the port's `link_type`); serial, ATM and POS ports keep the relay whatever the capabilities. Anchor creation on the compute side uses its own registry of Ethernet models — `ETHERNET_ADAPTERS` / `ETHERNET_WICS` in `compute/dynamips/adapters/adapter.py`, WIC-1ENET included at its Dynamips port number `16 * (wic_slot + 1)` — a name-keyed list kept in step with the controller's matrix by convention, not derivation. ## Link creation sequence -Both routers sit on the same compute (a kernel-link precondition) and each drives its own uBridge process; the hypervisor is one process per router. The two node handlers attach concurrently and compute the same bridge name independently (a pure function of the link id), so either may win the `brctl create` race — the loser verifies the bridge exists (`brctl show`) and continues: +Both routers sit on the same compute (a kernel-link precondition) and each drives its own uBridge process; the hypervisor is one process per router. The sequence shows the running-router case — a router that is stopped stores the NIO and defers every step here to its next start (deferred wiring, below). The two node handlers attach concurrently and compute the same bridge name independently (a pure function of the link id), so either may win the `brctl create` race — the loser verifies the bridge exists (`brctl show`) and continues: ```mermaid sequenceDiagram @@ -175,15 +175,15 @@ sequenceDiagram Note over HY1,UB2: steady state — kernel forwards between the anchors, the hypervisors hold the tap fds ``` -`nio create_tap`, `vm slot_add_nio_binding` and `vm slot_enable_nio` are the Dynamips-specific steps: the hypervisor opens the anchor and binds it to the slot/port before any bridge work, so a half-wired port never appears on the link. Capture, markers and `tc netem set` run only when the NIO carries them — a marker's real form is `marker add_kernel ""`, `bpf` expressions arrive as `tc bpf_drop add`, the stateful modes as `tc nth/quota/window_drop` — and `brctl addif` is itself what opens the port's `group_fwd_mask` to `0xfffd` (no separate command exists). The anchor's carrier stays down while the link is suspended. A link created while the router is **stopped** is stored and wired by the next start — the NIO simply waits in the slot adapter (deferred wiring, Anchor lifecycle below). +`nio create_tap`, `vm slot_add_nio_binding` and `vm slot_enable_nio` are the Dynamips-specific steps: the hypervisor opens the anchor and binds it to the slot/port before any bridge work, so a half-wired port never appears on the link. Capture, markers and `tc netem set` run only when the NIO carries them — a marker's real form is `marker add_kernel ""`, `bpf` expressions arrive as `tc bpf_drop add`, the stateful modes as `tc nth/quota/window_drop` — and `brctl addif` is itself what opens the port's `group_fwd_mask` to `0xfffd` — the server sends no separate command for it (uBridge does have one, `brctl setportgroupfwd`, last-writer-wins against addif's default, which re-applies on every re-attach). The anchor's carrier stays down while the link is suspended. A link created while the router is **stopped** is stored and wired by the next start — the NIO simply waits in the slot adapter (deferred wiring, Anchor lifecycle below). ## Anchor lifecycle * **Birth (node start)** — `_prepare_tap_datapath` probes the tap module (one create/delete cycle) and creates one persistent TAP per Ethernet slot/port, ownership handed to the server user, born DOWN. A stale leftover from a previous run is swept first; an anchor that already exists (restart) is kept — an anchor is never recreated under a live link. Serial/ATM/POS ports simply have no anchor. * **Life** — a kernel link opens the anchor in the hypervisor (`nio create_tap`), binds it to the slot/port and enslaves it into the per-link bridge; removing the link unbinds the port, deletes the hypervisor's TAP NIO (releasing the fd) and tears the anchor out of the bridge. A link bound while the router is **stopped** is stored and wired by the next start (deferred wiring — the NIO simply waits in the slot adapter). -* **Death (node close)** — `_stop_ubridge` runs in **QEMU's order**, not IOU's: here the Dynamips hypervisor holds the anchor fds, so its TAP NIOs go first (`nio delete` closes the fd), then `tap delete` per anchor, then the per-link kernel bridges, then uBridge itself. +* **Death (node close)** — the fd holder releases before the devices are deleted, the order every node type uses: here the Dynamips hypervisor (not uBridge) holds the anchor fds, so its TAP NIOs go first (`nio delete` closes the fd), then `tap delete` per anchor, then the per-link kernel bridges, then uBridge itself. -**A stop is not a close.** `stop` only halts the emulated router: the hypervisor process — and with it the TAP fds, the port bindings and the per-link bridges — survives `vm stop` (the same reason a relay link's tunnel always survived a stop). Links therefore keep working through a stop/start with no re-wiring at all; the restart's attach pass skips every port that still has its TAP NIO. If the hypervisor died mid-flight (crash or kill), the persistent TAPs remain and the next start re-attaches from the stored NIOs. +**A stop is not a close.** `stop` only halts the emulated router: the hypervisor process — and with it the TAP fds, the port bindings and the per-link bridges — survives `vm stop` (the same reason a relay link's tunnel always survived a stop). Links therefore keep working through a stop/start with no re-wiring at all; the restart's attach pass skips every port that still has its TAP NIO. If the hypervisor died mid-flight (crash or kill), the persistent TAPs remain, but the node cannot simply be started again — `start()` speaks to the dead hypervisor and nothing respawns it; recovery is re-creating the node (a project reopen does it), where the stale anchors are swept and rebuilt and the controller re-posts the link NIOs. ## Link operations @@ -211,18 +211,18 @@ A uBridge without the tap module keeps the node relay-only: `_prepare_tap_datapa ## Verified -Unit level (`tests/compute/dynamips/test_dynamips_kernel_datapath.py`): anchor topology (Ethernet models and WIC-1ENET port numbering only), the lifecycle (create/sweep/skip-existing), deferred wiring of NIOs bound while stopped, attach/update/remove command shapes on both datapaths, the stop order (hypervisor NIO delete before tap delete before brctl delete), capture and marker shapes, hot-added adapters, and the manager's `nio_bridge` construction. Plus the controller-eligibility cases (mixable with docker/qemu/iou, tap capability required, serial excluded) in `tests/controller/test_kernel_datapath_link.py`. +Unit level (`tests/compute/dynamips/test_dynamips_kernel_datapath.py`): anchor topology (Ethernet models and WIC-1ENET port numbering only), the lifecycle (create/sweep/skip-existing), deferred wiring of NIOs bound while stopped, attach/update/remove command shapes on both datapaths, the idempotent re-attach across a restart, the stop order (hypervisor NIO delete before tap delete before brctl delete), capture and marker shapes, hot-added adapters, and the manager's `nio_bridge` construction. Plus the controller-eligibility cases (mixable with docker/qemu/iou, tap capability required, serial excluded) in `tests/controller/test_kernel_datapath_link.py`. End-to-end (`tests/e2e/test_dynamips_kernel_datapath.py`, pytest marker `e2e`) on a live server with **two real c7200 routers on a real IOS image**, driven through the REST API and the IOS consoles with real ICMP: * a link created while both routers are stopped selects the kernel datapath and is wired by node start (deferred wiring); * the per-link kernel bridge on the host has exactly the two anchor TAPs enslaved; `show ip int brief` is up/up and ping R1→R2 succeeds; -* `delay 100` ⇒ netem qdisc visible on both anchors and the ping RTT grows by ~200 ms (one-way 100 per direction, each anchor impairing its own ingress side); clearing resets the qdiscs and the RTT; +* `delay 100` ⇒ netem qdisc visible on both anchors and the ping RTT grows by ~200 ms (one-way 100 per direction, each anchor's egress chain impairing the direction entering its own router); clearing resets the qdiscs and the RTT; * suspend ⇒ anchor administratively down, ping 0%; resume restores both; * capture writes a pcap full of ICMP-over-Ethernet records; * a **serial** link between the same two routers stays on the relay (per-port exclusion) and pings over it — the relay regression net; * link delete/re-create: the bridge goes and comes back, the anchors survive, traffic resumes; -* router stop/start keeps the wiring intact (bridge membership unchanged, no duplicate NIOs) and traffic resumes after the reboot; +* router stop/start keeps the wiring intact (bridge membership unchanged — the no-duplicate-NIO half is the unit-tested idempotent re-attach) and traffic resumes after the reboot; * deleting the project removes every anchor and bridge. `test_dynamips_relay_control` is the negative control: the same topology on an isolated relay-configured instance (`enable_kernel_datapath = false`) has no per-link bridge and nothing enslaved, and still pings — so the kernel objects above can only come from the kernel datapath. @@ -238,4 +238,4 @@ enable_kernel_datapath = True ## Roadmap -The Ethernet switch has landed (see `ethernet-switch-kernel-datapath.md` — it absorbs the peer's anchor into its own kernel bridge; switch-to-switch cascades and the IOL runner container have landed too — see `ethernet-switch-kernel-datapath.md` and `iol-docker-kernel-datapath.md`). Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized. +The Ethernet switch has landed (see `ethernet-switch-kernel-datapath.md` — it absorbs the peer's anchor into its own kernel bridge, and switch-to-switch cascades with it), and so has the IOL runner container (`iol-docker-kernel-datapath.md`). Every router and switch node type now anchors; VPCS, the hub, the cloud and the NAT node stay on the relay by design. Cross-compute kernel links need VXLAN/GENEVE encapsulation — a separate, still-deferred project. diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index 2a544d3f5..26d89f8b7 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -13,7 +13,7 @@ IOL runner containers (`IOLDockerVM`, selected by the `GNS3_IOL_RUNNER` environm The kernel datapath gives every Ethernet bay/unit a **persistent TAP anchor** (`gx` names, born at node start exactly like IOU's) and wires the *link segment* through the kernel. One userspace hop on the container leg is irreducible — the socket pair is the only physical layer the runner exposes — but the anchor → per-link bridge → peer anchor crossing, the kernel filters, the suspend carrier and the AF_PACKET capture/markers are all native now. -The port bridge is the deployment shape ubridge's `doc/gns3server-integration.md` § *bridge — generic NIO relay with a swappable TAP leg* froze: a per-node bridge created at start, the unix NIO in the first slot for the node's whole life, the topology leg in the second — `add_nio_udp` (relay) or `add_nio_tap` (kernel link) — swapped as links are attached, suspended, deleted or the project reopens, through stop → delete → add → start. The c-socket binding never re-binds across a swap: frames the container sends while the bridge is stopped queue on the socket and relay after `start` (regression-tested in ubridge's bridge suite). +The port bridge is the deployment shape ubridge's `doc/gns3server-integration.md` § *bridge — generic NIO relay with a swappable TAP leg* froze: one bridge per bay/unit, all created at start, the unix NIO in the first slot for the node's whole life, the topology leg in the second — `add_nio_udp` (relay) or `add_nio_tap` (kernel link) — swapped through stop → delete → add → start whenever a link is deleted, re-created or switched between datapaths. Suspend swaps no leg (an in-place NIO update), and a project reopen is a fresh build from the NIO, not a swap. The c-socket binding never re-binds across a swap: frames the container sends while the bridge is stopped queue on the socket and relay after `start` (regression-tested in ubridge's bridge suite). ## Architecture @@ -25,7 +25,7 @@ Kernel link between two IOL runner containers on the same compute. Interface nam | `sNN.sock` / `cNN.sock` | the interface's socket pair (`NN = bay×4 + unit`): frames sent to `s` are injected into the guest, the guest's egress arrives on `c` | | wiring dir | the host-side directory bind-mounted at the container's `/tmp` (`$XDG_RUNTIME_DIR/gns3/unixio/`) — the same socket files on both sides, short enough for the 107-byte `sun_path` cap | | uBridge | the per-node helper process gns3-server spawns; its hypervisor command surface executes all host-side wiring (anchors, bridges, carrier, tc, taps) — and, unique to this node type, it also carries frames: the socket guest leg has no kernel form | -| port bridge | the per-bay/unit uBridge bridge (`bridge{bay}_{unit}`): `nio_unix` in its first slot for the node's whole life, the topology leg — `nio_tap` (kernel link) or `nio_udp` (relay) — in the second | +| port bridge | the per-bay/unit uBridge bridge (`bridge{bay}` on unit 0, `bridge{bay}_{unit}` on units 1-3 — the historical `_bridge_name` naming): `nio_unix` in its first slot for the node's whole life, the topology leg — `nio_tap` (kernel link) or `nio_udp` (relay) — in the second | | anchor | the persistent TAP `gx…` a link attaches to: born at node start, DOWN, its fd held by the port bridge while a kernel link binds the port | | per-link bridge | the Linux bridge `gns3{link_id[:11]}` that stands in for the cable; the two anchors are its only ports | | bridge port | an interface enslaved to a Linux bridge (`brctl addif`) — it no longer sends or receives on its own: frames arriving on it go to the bridge's forwarding logic, and the bridge transmits through it | @@ -38,7 +38,7 @@ Kernel link between two IOL runner containers on the same compute. Interface nam | netem | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | | `AF_PACKET tap` | uBridge's packet socket bound to the anchor; the kernel clones every frame the interface sends or receives to it (the tcpdump mechanism) — read-only copies for capture and markers, the original stays on the kernel path | | carrier | the anchor's admin state (`link set up/down`); suspend is carrier down on both anchors = 100 % loss | -| `nio_bridge` | the NIO record the controller sends each compute: bridge name, filters, markers, suspend flag | +| `nio_bridge` | the NIO record the controller sends each compute: bridge name, filters, markers, suspend flag — the bridge name is None when an Ethernet-switch peer absorbs the anchor into its own kernel bridge (then no per-link bridge exists on this side) | ```mermaid flowchart TB @@ -121,7 +121,7 @@ Drawn one-way; the reverse direction is the mirror image. The IOL relay crosses Two further contrasts with Docker's relay: the IOL relay's local leg is the unix NIO itself, so the anchors never appear on the path (idle from birth — a Docker relay attaches to the *same* veth host end the kernel link enslaves); and the port bridge's first slot is shared with the kernel datapath — the unix NIO and its c-socket binding survive every leg swap, so frames the container sends while the bridge is stopped for a swap queue on the socket and relay after `start`. On this datapath suspend rides the synthetic frequency_drop filter (the kernel datapath's suspend is the anchor's admin state). -Frame path for one A → B frame, with the relay diagram's copy numbering: copies 1, 2, 5 and 6 are the socket guest legs and are paid identically on both datapaths — only the middle two swap (TAP fd write/read here, loopback UDP there). The impairment chain fires on the egress of the *destination* anchor, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gxA's egress chain — hence `delay 100` doubles the RTT (measured ≥ 150 ms): +Frame path for one A → B frame, with the relay diagram's copy numbering: copies 1, 2, 5 and 6 are the socket guest legs and are paid identically on both datapaths — only the middle two swap (TAP fd write/read here, loopback UDP there). The impairment chain fires on the egress of the *destination* anchor, in order clsact → netem (a classifier drop means netem never sees the frame); each direction is impaired exactly once and B → A is symmetric on gxA's egress chain — `delay 100` is therefore collected once per direction, ~200 ms on the RTT (measured ≥ 150 ms): ```mermaid flowchart LR @@ -225,7 +225,7 @@ sequenceDiagram Every arrow to uBridge is one `_ubridge_send` command line on the node's local hypervisor console; uBridge executes it (netlink/ioctl) and replies OK or an error synchronously before the next command is sent — those replies are omitted for brevity. uBridge never initiates anything: it is a pure executor, and the only data ever flowing back (capture/marker frame copies) rides separate AF_PACKET sockets, not this console. -The attach above is `_attach_kernel_link`; its ensures are the frozen ensure-then-add contract: the port bridge (`bridge create` + `add_nio_unix`) comes from the node's start loop or is created here, and `tap create` runs only when `/sys/class/net` shows the anchor missing — it is strictly create-only (`IFF_TUN_EXCL`) and answers EBUSY on the device the start loop just made. `bridge add_nio_tap` is what opens and holds the anchor fd, `bridge start` starts the [unix ↔ tap] relay, and the final `link set up` is the carrier pass that brings the born-down anchor up (a suspended NIO sets it back down). +The attach above is `_attach_kernel_link`; its ensures are the frozen ensure-then-add contract: the port bridge (`bridge create` + `add_nio_unix`) comes from the node's start loop or is created here, and `tap create` runs only when `/sys/class/net` shows the anchor missing — it is strictly create-only (`IFF_TUN_EXCL`) and answers EBUSY on the device the start loop just made. `bridge add_nio_tap` is what opens and holds the anchor fd, `bridge start` starts the [unix ↔ tap] relay, and the final `link set up` refines the anchor's admin state to the NIO's suspend flag — `brctl addif` has already brought the port up (addif auto-UPs it along with its fwd_mask), so on an unsuspended link the command changes nothing, and a suspended one is what pushes the anchor back down. * **Link delete** (`_release_port_tap`): `bridge stop` (delete_nio_tap refuses while running — the relay threads hold the NIO pointers for their whole life; freeing under them is a use-after-free) → `bridge delete_nio_tap ""` (matched on the kernel-resolved name) → the mixin teardown (markers, tc reset, `brctl delif`, per-link bridge deletion). The anchor survives — the node owns it, not the link. * **Relay link** — unchanged: the port bridge's second leg is `add_nio_udp`, userspace filters at the port bridge. The unix NIO stays in its slot across every datapath switch. @@ -252,9 +252,9 @@ Live on the real server (`tests/e2e/test_iol_docker_kernel_datapath.py`, isolate * kernel link reports `kernel_datapath`, anchors exist from node start, the per-link bridge enslaves exactly the two of them; * real ICMP crosses the [unix ↔ tap] port-bridge relay (100 % ping); -* the L2-anchor spec §E.2 silence window with the guests shut — the behavioral half of the hardening on anchors created by uBridge's bridge TAP module (`bridge add_nio_tap`); +* the L2-anchor spec on the anchors: FORWARDING ports always, and on a uBridge with `link l2only` §E.1's pure-L2 checks plus §E.2's silence window with the guests shut — the behavioral half of the hardening on anchors created by uBridge's bridge TAP module (`bridge add_nio_tap`); * `delay 100` lands as netem on both anchors, measured RTT grows ≥ 150 ms and returns < 50 ms when cleared; -* the classifier spot check on a TAP anchor (the Docker suite runs the full matrix on veth host ends): `bpf "icmp"` drops everything (clsact on the anchor), `frequency_drop 3` as the eBPF every-nth mode measured 56 % round-trip loss (theory 55.6 %), both restoring on clear; +* the classifier spot check on a TAP anchor (the Docker suite runs the full matrix on veth host ends), each behind its capability gate: `bpf "icmp"` drops everything (clsact on the anchor, needs cbpf), `frequency_drop 3` as the eBPF every-nth mode (needs the nth token) — round-trip loss asserted 20-90 % around the per-direction theory 1-(2/3)² = 55.6 %, both restoring on clear; * suspend admin-downs the anchor and kills the traffic; resume restores; * AF_PACKET capture writes a real pcap of the ICMP exchange; * link delete/re-create swaps the port bridge's TAP leg out and back in (unix binding never re-binds) with the ping returning; @@ -274,4 +274,4 @@ The anchor lifecycle needs no configuration of its own — the capability probes ## Roadmap -Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized; with the IOL runner container anchored, that condition is one project away from met. +Cross-compute kernel links need VXLAN/GENEVE encapsulation — deferred until every node type is kernelized; with the IOL runner container anchored, every node type in scope can now anchor (hub, cloud, NAT and VPCS stay on the relay by decision), so the encapsulation work is all that remains. diff --git a/docs/features/iou-kernel-datapath.md b/docs/features/iou-kernel-datapath.md index 5dbeeaf41..5e83507b1 100644 --- a/docs/features/iou-kernel-datapath.md +++ b/docs/features/iou-kernel-datapath.md @@ -11,7 +11,7 @@ Serial bays never anchor (a TAP carries Ethernet frames): serial links stay on t ## Architecture -The diagram shows the native shape — two IOU devices linked on the kernel datapath, the scenario the e2e suite drives. A Docker or QEMU peer on the far end is the same link segment with a different anchor (a `gv` veth host end or a `gq` TAP — see `docker-kernel-datapath.md`); each IOU still terminates its own fabric in its own uBridge. Interface names follow the deterministic rules `gi{node_id[:8]}e{bay}p{unit}` for the anchors and `gns3{link_id[:11]}` for the per-link bridge; the concrete names in the diagrams use node ids `a1b2c3d4` / `5e6f7a8b` (IOU A/B), link id prefix `5d3c2b1a4e6`, and application ids 1 / 2 (so the fake instances are `IOL-BRIDGE-513` / `IOL-BRIDGE-514`). Terms the diagrams use: +The diagram shows the native shape — two IOU devices linked on the kernel datapath, the scenario the e2e suite drives. A Docker or QEMU peer on the far end is the same link segment with a different anchor (a `gv` veth host end or a `gq` TAP — see `docker-kernel-datapath.md`); each IOU still terminates its own fabric in its own uBridge. Interface names follow the deterministic rules `gi{node_id[:8]}e{bay}p{unit}` for the anchors and `gns3{link_id[:11]}` for the per-link bridge (ids stripped of dashes before the slice); the concrete names in the diagrams use node ids `a1b2c3d4` / `5e6f7a8b` (IOU A/B), link id prefix `5d3c2b1a4e6`, and application ids 1 / 2 (so the fake instances are `IOL-BRIDGE-513` / `IOL-BRIDGE-514`). Terms the diagrams use: | Term | Meaning | |---|---| @@ -163,7 +163,7 @@ The NIO schema on the compute NIO routes accepts `nio_bridge` alongside the UDP/ ## Link creation sequence -Both endpoints sit on the **same compute** (a kernel-link prerequisite). The controller derives the bridge name once from the link id and ships one `nio_bridge` record to each endpoint — same bridge name, each side's own filters/markers and the suspend flag — the two POSTs overlapping (`asyncio.gather`, either failure rolls the peer back). The bridge has no owner: both sides race `brctl create` (EEXIST tolerated, the loser verifies via `brctl show`). The anchors already exist, born at node start (Anchor lifecycle below), so attaching only binds and enslaves: +Both endpoints sit on the **same compute** (a kernel-link prerequisite). The controller derives the bridge name once from the link id and ships one `nio_bridge` record to each endpoint — same bridge name, each side's own filters/markers and the suspend flag — the two POSTs overlapping (`asyncio.gather`, either failure rolls the peer back). The bridge has no owner: both sides race `brctl create` (EEXIST tolerated, the loser verifies via `brctl show`). When both endpoints run, the anchors already exist (born at node start — Anchor lifecycle below), so attaching only binds and enslaves: ```mermaid sequenceDiagram @@ -183,7 +183,7 @@ sequenceDiagram NA->>UA: brctl create NA->>UA: link set bridge up NA->>UA: brctl addif giA - NA->>UA: markers / capture if any + NA->>UA: capture / markers if any NA->>UA: tc netem + bpf + eBPF NA->>UA: link set giA up and POST nio_bridge to node B @@ -195,17 +195,19 @@ sequenceDiagram `iol_bridge add_nio_tap ""` is the one IOU-specific step: it binds the fabric port to the anchor and hands uBridge the TAP fd. Deletion reverses it in miniature — markers, tc reset, delif, `brctl delete`, then `iol_bridge delete_nio_tap` releases the fd while the control channel lives (Link operations below). -## Anchor lifecycle — and why the stop order is QEMU's reverse +A stopped endpoint defers its whole side of the flow — the common case, since a project is typically opened with its links already drawn. With no uBridge up, the NIO POST lands as bookkeeping only (the duplicate-port guard still applies), and the node's start replays this same attach from the recorded NIO once the anchors exist (`_networking` → `_attach_kernel_nio`), capture and filters included — which is also how a link survives its node's stop/start. + +## Anchor lifecycle — and why the fd holder must let go first * **Birth (node start)** — `_prepare_tap_datapath` probes the capability and creates one persistent TAP per Ethernet bay/unit (4 units per bay, swept for stale leftovers first, born DOWN). uBridge holds a device's fd only while a kernel link binds the port; until then the TAP has no holder. -* **Life** — the TAP is never recreated. A kernel link adds `iol_bridge add_nio_tap ` then enslaves the anchor (`_kernel_attach`); a relay link keeps `iol_bridge add_nio_udp` and never touches the anchor. Switching a port between datapaths is just an NIO swap — the IOL port frees its previous NIO (fd or socket) on attach. -* **Death (node stop)** — `_stop_ubridge` runs the cleanup **in the reverse order of QEMU's**, because here uBridge itself holds the anchor fds: `iol_bridge delete` (releases every port's TAP fd) → `tap delete` per anchor (a held TAP answers EBADFD/207) → `brctl delete` for per-link bridges still held → stop the hypervisor. Removing a single kernel link does the same in miniature: `_remove_kernel_nio` (markers, tc reset, delif, brctl delete) then `iol_bridge delete_nio_tap` — releasing the fd while the control channel lives. +* **Life** — link churn never recreates the TAP. A kernel link adds `iol_bridge add_nio_tap ` then enslaves the anchor (`_kernel_attach`); a relay link keeps `iol_bridge add_nio_udp` and never touches the anchor. Switching a port between datapaths is just an NIO swap — the IOL port frees its previous NIO (fd or socket) on attach. +* **Death (node stop)** — the fd holder must let go before the devices are deleted. Here the holder is uBridge itself (on QEMU it is the VM's own process), so the uBridge side retires first: `iol_bridge delete` (releases every port's TAP fd) → `tap delete` per anchor (a held TAP answers EBADFD/207) → `brctl delete` for per-link bridges still held → stop uBridge. The IOU process is stopped last, holding no anchor fd — the reverse of QEMU's process-first sequence. Removing a single kernel link does the same in miniature: `_remove_kernel_nio` (markers, tc reset, delif, brctl delete) then `iol_bridge delete_nio_tap` — releasing the fd while the control channel lives. ## Link operations | Operation | Kernel-datapath action | |---|---| -| Create / attach | `iol_bridge add_nio_tap` (bind port ↔ anchor, uBridge holds the fd) + `brctl create` (EEXIST tolerated) + `link set up` + `brctl addif` + markers + `tc netem set` + carrier up | +| Create / attach | `iol_bridge add_nio_tap` (bind port ↔ anchor, uBridge holds the fd) + `brctl create` (EEXIST tolerated) + `link set up` + `brctl addif` + capture/markers + `tc netem set` + carrier up | | Delete / detach | marker teardown → `tc reset ` → carrier off → `brctl delif`/`brctl delete` (last endpoint wins) → `iol_bridge delete_nio_tap` (release the fd; the device survives) | | Update (filters/markers changed) | reconcile on the anchor only — no re-binding, no re-enslaving | | Suspend | anchor admin-down (TAP-fd writes fail EIO → dropped; reads answer EIO too — the §B hardening tolerates both) + resume restores | @@ -216,7 +218,7 @@ Capture on a kernel link is `capture start_kernel ` (driven by the NIO type ## Link-local frames (LACP, LLDP, 802.1X, STP) -The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. +The bridge stands in for a cable, but the kernel's `br_handle_frame()` does not forward the IEEE 802.1D reserved range (`01:80:c2:00:00:00`-`0f`) by default — LACP, LLDP/DCBX and 802.1X would never cross a kernel link while the UDP relay carries them fine. uBridge therefore opens every port's per-port `group_fwd_mask` (`IFLA_BRPORT_GROUP_FWD_MASK`) to `0xfffd` at `brctl addif` time (Linux 4.15+, best-effort on older kernels) — delivered by the fork's `feature/bridge-tap-l2only` branch (which also carries the `iol_bridge add_nio_tap` anchor work; a build from an earlier branch point leaves ports at the kernel default). The bridge-level knob cannot do this: `BR_GROUPFWD_RESTRICTED` rejects bits 0-2, so LACP is reachable per port only. Result on kernel links: ordinary multicast, STP/RSTP, LACP, 802.1X and LLDP/DCBX all cross; **802.3x PAUSE / PFC does not** — `case 0x01` in `br_handle_frame()` drops it unconditionally and no mask can enable it. That is a documented limit, not a regression (PFC's hardware semantics are out of emulation's reach anyway); a lab that needs PAUSE frames on the wire must use a relay link. Verified live by `tests/e2e/test_docker_link_local_frames.py` (per-MAC guest-to-guest matrix, driven on the Docker datapath — the same per-link-bridge mechanism; the IOU e2e does not repeat it); the contract and kernel references live in `docs/design/ubridge-link-local-frame-forwarding-spec.md`. @@ -226,9 +228,9 @@ A uBridge without `iol_bridge add_nio_tap` (old build) keeps the node on the rel ## Verified -Unit level: anchor lifecycle and naming, kernel/relay binding on add, update and remove, the stop order (iol_bridge delete before tap delete before brctl delete before hypervisor stop), `_networking` on restart with mixed kernel/relay ports, capture and marker command shapes on both datapaths, the capability probe (cycle, old-build fallback, cleanup, caching), capability plumbing, and controller eligibility (per-compute capability, its independence from `ubridge_tap`, serial-port exclusion). +Unit level: anchor lifecycle and naming, kernel/relay binding on add, update and remove, the stop order (iol_bridge delete before tap delete before brctl delete before uBridge stops), `_networking` on restart with mixed kernel/relay ports, capture and marker command shapes on both datapaths, the capability probe (cycle, old-build fallback, cleanup, caching), capability plumbing, and controller eligibility (per-compute capability, its independence from `ubridge_tap`, serial-port exclusion). -End-to-end on a live server (isolated instance, two IOU nodes behind a fake ELF image, the test playing the IOL fabric on `/tmp/netio/` and pushing frames through it): **64/64 checks** on the kernel datapath and **62/62** on the relay (`enable_kernel_datapath = false`). Kernel highlights: +An isolated-instance validation run — a development harness that is not part of this repository: two IOU nodes behind a fake ELF image, the driver playing the IOL fabric on `/tmp/netio/` and pushing frames through it — passed **64/64 checks** on the kernel datapath and **62/62** on the relay (`enable_kernel_datapath = false`). Kernel highlights: * 8 anchors per node (2 bays × 4 units), persistent, DOWN, address-free; `link_list` reports `kernel_datapath: true` and one per-link bridge holds both anchors with `brport/state = 3`; * fabric → anchor → bridge → anchor → fabric in both directions at ~0.2 ms; `delay 100` ⇒ netem on both anchors and a measured **100.5 ms** one-way; @@ -240,7 +242,7 @@ End-to-end on a live server (isolated instance, two IOU nodes behind a fake ELF The relay run exercises the same lifecycle with the IOL UDP NIOs (no tc qdiscs, userspace `delay` at 100.7 ms, suspend via the synthetic frequency_drop, `iol_bridge start_capture`), which is also the regression net for the port-coordinate capture restore this work touched. -The repository's live e2e suite carries the same scenario with **real IOU routers** (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker `e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on the server's telnet console, driving anchors-born-with-the-node, the §E.2 idle-silence window with the guests shut, real ICMP through the port bridge, netem delay, the TAP-anchor classifier spot check (bpf match-drop; the eBPF every-nth mode at 56 % measured round-trip loss, 55.6 % expected), suspend, capture, link delete/re-create, node stop/start rewiring with a delay filter restored on the fresh anchor (tc and traffic both), the serial link staying on the relay, and a relay negative control whose real `delay`/`frequency_drop` filters ride the port's `iol_bridge add_packet_filter` list — capture and markers on the same engine (`iol_bridge start_capture` / the port's `mark` filter). +The repository's live e2e suite carries the same scenario with **real IOU routers** — the reproducible run in the tree (`tests/e2e/test_iou_kernel_datapath.py`, pytest marker `e2e`): a real L3 IOU image provides the fabric and the real IOS CLI on the server's telnet console, driving anchors-born-with-the-node, the §E.2 idle-silence window with the guests shut, real ICMP through the port bridge, netem delay, the TAP-anchor classifier spot check (bpf match-drop; the eBPF every-nth mode at 56 % measured round-trip loss, 55.6 % expected), suspend, capture, link delete/re-create, node stop/start rewiring with a delay filter restored on the fresh anchor (tc and traffic both), the serial link staying on the relay, and a relay negative control whose real `delay`/`frequency_drop` filters ride the port's `iol_bridge add_packet_filter` list — capture and markers on the same engine (`iol_bridge start_capture` / the port's `mark` filter). The uBridge side passed its own 30/30 (`tests/iol/test_tap_anchor.py` in the fork), including the DOWN-anchor resilience and kernel-bridge interop this design depends on. diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md index 703bd1164..e1f459fd9 100644 --- a/docs/features/qemu-kernel-datapath.md +++ b/docs/features/qemu-kernel-datapath.md @@ -68,7 +68,7 @@ flowchart LR Everything in the root namespace is built the same way: gns3-server creates the persistent TAPs at node start (before QEMU is launched), the per-link bridge and its memberships at link creation, and every later change (carrier, capture, markers, filters) by sending hypervisor commands to the node's uBridge process — the privileged agent that turns them into netlink/ioctl operations. That build flow is the Link creation sequence below; this diagram is the steady state that remains, where the only uBridge relationships left are the two dashed kinds. -uBridge sits off the forwarding path — the dashed edges are control and observation, never data flow. The `qdisc install` edge is uBridge configuring the kernel over netlink; the kernel then runs those qdiscs in-flight on the anchor's egress (stages of the transmit path, not packet consumers). The `frame copies` edge is the AF_PACKET tap: the kernel clones every frame the TAP sends or receives to uBridge's packet socket (the tcpdump mechanism) — read-only copies for capture and markers; the originals never leave the kernel. The only userspace on the path is the two QEMU processes pumping their own tap fds — the irreducible VM leg, one fd crossing at each end per direction. +uBridge sits off the forwarding path — the dashed edges are control and observation, never data flow. The `qdisc install` edge is uBridge configuring the kernel over netlink; the kernel then runs those qdiscs in-flight on the anchor's egress (stages of the transmit path, not packet consumers). The `frame copies` edge is the AF_PACKET tap, there only while a capture or markers is active (a kernel link leaves no uBridge endpoint on the anchor otherwise): the kernel clones every frame the TAP sends or receives to uBridge's packet socket (the tcpdump mechanism) — read-only copies for capture and markers; the originals never leave the kernel. The only userspace on the path is the two QEMU processes pumping their own tap fds — the irreducible VM leg, one fd crossing at each end per direction. Relay link between the same two QEMU nodes on the same single compute — what every link becomes when `enable_kernel_datapath` is off (mixed node types and cross-compute wiring fall back too; there the UDP hop rides the real network instead of loopback, the copy count unchanged): @@ -177,9 +177,9 @@ So every filter, capture and marker primitive carries over verbatim, and `delay ## Datapath selection -`UDPLink._kernel_datapath_eligible` needs both endpoints on one compute and both able to anchor: +`UDPLink._kernel_datapath_eligible` needs both endpoints on one compute and both able to anchor (an Ethernet-switch endpoint is the exception — it absorbs the peer's anchor into its own bridge, so the switch side needs no anchoring capability of its own; `docs/features/ethernet-switch-kernel-datapath.md`): -* Docker always can (adapters are born as veth pairs); +* Docker can (adapters are born as veth pairs), except the IOL runner container, whose unix-socket guest leg anchors on a TAP under its own capability pair (`docs/features/iol-docker-kernel-datapath.md`); * QEMU can when that compute's uBridge reports the `tap` module — asked per compute through `/capabilities` (`ubridge_tap`, probed by creating and deleting one throwaway TAP, cached by binary identity), because an old uBridge leaves QEMU on the legacy socket-netdev datapath and cannot anchor a kernel link. An unknown/failed probe keeps the link on the relay. Because the `-netdev` type is fixed when QEMU is launched, this is all-or-nothing per VM: on the TAP datapath *every* link can attach to (or detach from) a running VM, including switching a link between the two datapaths (delete + re-create, or a project reopen). @@ -206,8 +206,7 @@ sequenceDiagram CA->>UA: brctl create CA->>UA: link set bridge up CA->>UA: brctl addif gq3f2a1b9ce0p0 - CA->>UA: port fwd_mask 0xfffd - CA->>UA: markers / capture if any + CA->>UA: capture / markers if any CA->>UA: tc netem + bpf + eBPF CA->>UA: link set gq3f2a1b9ce0p0 up and QEMU node B @@ -216,9 +215,9 @@ sequenceDiagram Note over UA,UB: steady state — kernel forwards, uBridge taps ``` -Every arrow to uBridge is one `_ubridge_send` command line on the node's local hypervisor console; uBridge executes it (netlink/ioctl) and replies OK or an error synchronously before the next command is sent — those replies are omitted for brevity. uBridge never initiates anything: it is a pure executor, and the only data ever flowing back (capture/marker frame copies) rides separate AF_PACKET sockets, not this console. +Every arrow to uBridge is one `_ubridge_send` command line on the node's local hypervisor console; uBridge executes it (netlink/ioctl) and replies OK or an error synchronously before the next command is sent — those replies are omitted for brevity. The per-port `group_fwd_mask` 0xfffd is deliberately not an arrow: uBridge applies it inside `brctl addif` itself (Link-local frames below). uBridge never initiates anything: it is a pure executor, and the only data ever flowing back (capture/marker frame copies) rides separate AF_PACKET sockets, not this console. -The TAP already exists at attach time — created at node start, before QEMU launched (Adapter lifecycle below) — which is what makes the attach runtime-safe on a running VM: anchors, bridge memberships and qdiscs all change around the `-netdev tap` QEMU already holds, and nothing talks to QEMU itself outside suspend/resume (QMP `set_link`, so the guest's carrier follows the link). +The TAP already exists at attach time — created at node start, before QEMU launched (Adapter lifecycle below) — which is what makes the attach runtime-safe on a running VM: anchors, bridge memberships and qdiscs all change around the `-netdev tap` QEMU already holds, and the only QEMU traffic is carrier replication — QMP `set_link` on attach, detach and suspend/resume, gated by the node's `replicate_network_connection_state` option (default on), so the guest's carrier follows the link. A mixed Docker↔QEMU link runs the identical sequence per end — each node's uBridge receives the same commands on its own anchor, the Docker end's `gv` veth host end in place of the `gq` TAP; only the anchor name differs (the mixin is anchor-agnostic, which is the point of the shared `gns3server/compute/kernel_datapath.py`). @@ -226,13 +225,13 @@ A mixed Docker↔QEMU link runs the identical sequence per end — each node's u * **Birth (node start, before QEMU is launched)** — `_prepare_tap_datapath` probes the tap module, then `_create_taps` sweeps any leftover (a persistent TAP outlives a crash) and creates one TAP per adapter: `tap create gq{node_id[:8]}e{adapter}p{port}` → `tap set_owner ` (so the unprivileged QEMU process can open it) → `link set down` (carrier off until a link attaches). QEMU then opens it as `-netdev tap,id=gns3-{adapter},ifname=,script=no,downscript=no`. * **Life (running)** — the TAP is never created or destroyed again. Link create/delete/switch, suspend, capture and markers only change *what is attached to it*. -* **Death (node stop)** — the QEMU process is stopped **first**, then `_remove_taps` un-persists the TAPs (`tap delete`) and deletes the per-link kernel bridges the node still holds. The order matters: uBridge's `tap delete` refuses a device another process holds open ("Device or resource busy") and that best-effort delete is suppressed, so deleting the TAPs while QEMU still ran leaked every persistent TAP (process side first, then the devices — the same lesson as IOU's reverse stop order). Deleting a node never runs the link-teardown path, so those bridges would otherwise stay behind as empty orphans; a restart rebuilds them from the NIO. +* **Death (node stop)** — the QEMU process is stopped **first**, then `_remove_taps` un-persists the TAPs (`tap delete`) and deletes the per-link kernel bridges the node still holds. The order matters: uBridge's `tap delete` refuses a device another process holds open ("Device or resource busy") and that best-effort delete is suppressed, so deleting the TAPs while QEMU still ran leaked every persistent TAP (process side first, then the devices — the same lesson as IOU's stop order, where uBridge itself holds the anchors and is made to release them before `tap delete`). Stopping a node never runs the link-teardown path — links survive stops by design, and project close is the same (it stops nodes without deleting links) — so those bridges would otherwise stay behind as empty orphans; a restart rebuilds them from the NIO. Deleting a node from the project is not that case: the controller deletes its links first, so the teardown does run. ## Link operations | Operation | Kernel-datapath action | |---|---| -| Create / attach | `brctl create` (EEXIST tolerated via `brctl show`) + `link set up` + `brctl addif ` + carrier up + markers + `tc netem set` (filters) | +| Create / attach | `brctl create` (EEXIST tolerated via `brctl show`) + `link set up` + `brctl addif `, then capture and markers and filters on the anchor (`tc netem set`, bpf/eBPF drops), carrier up last | | Delete / detach | marker teardown → `tc reset ` → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The TAP survives — the adapter keeps it for the next link | | Reset (`POST /links/{id}/reset`) | delete + create (re-evaluates eligibility) | | Suspend (`PUT /links/{id}` `{"suspend": true}`) | tap admin-down (writing to a down TAP fd fails with EIO, so the link is genuinely dead) + QMP `set_link gns3-N off` so the guest notices; resume restores both, and the netem qdisc survives the flap | @@ -255,7 +254,7 @@ A uBridge build without the `tap` module (or without `CAP_NET_ADMIN`) keeps the End-to-end through a real server (isolated instance, two QEMU nodes with a stub binary, the test playing the guest by holding the TAP fds and pushing frames): **32/32 checks** on the kernel datapath and **29/29** on the relay variant (`enable_kernel_datapath = false`). Kernel run highlights: -* anchors created persistent and owned by the server user, QEMU's netdev is the TAP, no local UDP tunnel left; `link_list` reports `kernel_datapath: true` and one per-link bridge holds both taps; +* anchors created persistent and owned by the server user, QEMU's netdev is the TAP, no local UDP tunnel left; the link object reports `kernel_datapath: true` (the `POST /links` response) and one per-link bridge holds both taps; * frames cross in both directions (~0.1 ms); `delay 100` ⇒ netem on both anchors and a measured **100.3 ms** one-way; * suspend ⇒ anchor DOWN, no traffic, resume ⇒ 100.3 ms again (filter kept); * capture ⇒ a 252-byte pcap written by `capture start_kernel`; @@ -282,6 +281,6 @@ One residual is accepted and documented there: enslaving a port makes the *kerne ## Roadmap -IOU (its fabric terminator in uBridge needs a TAP-terminated port — `iol_bridge add_nio_tap`, frozen in `docs/design/ubridge-iol-tap-anchor-spec.md`; unlike QEMU, one userspace hop on the IOU leg is irreducible, the fabric is Unix sockets), Dynamips (hypervisor-created taps, enslavable as they are), and the Ethernet switch / cloud paths (their anchors are ubridge-owned by design) are still on the relay; cross-compute kernel links need VXLAN/GENEVE encapsulation, which is a separate project (it would serve Docker links the same way). +The rest of the node family has landed on the same mixin: **IOU** — see `docs/features/iou-kernel-datapath.md` (Ethernet bays anchor on persistent TAPs bound to the IOL fabric, `iol_bridge add_nio_tap` per `docs/design/ubridge-iol-tap-anchor-spec.md`; serial links stay relay, and one userspace hop on the IOU leg is irreducible — the fabric is Unix sockets); **Dynamips** — see `docs/features/dynamips-kernel-datapath.md` (the hypervisor opens uBridge-created TAPs with `nio create_tap`: the same external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay); **the Ethernet switch** — see `docs/features/ethernet-switch-kernel-datapath.md` (the switch absorbs a peer's anchor into its own kernel bridge; switch-to-switch cascades too); **the IOL runner container** — see `docs/features/iol-docker-kernel-datapath.md` (persistent TAP anchors through the generic bridge module's swappable TAP leg; the unix-socket guest leg keeps one userspace hop, like IOU's fabric). -Update: **IOU has landed** on the same mixin — see `docs/features/iou-kernel-datapath.md` (Ethernet bays anchor on persistent TAPs bound to the IOL fabric; serial links stay relay). **Dynamips has landed** too — see `docs/features/dynamips-kernel-datapath.md` (the hypervisor opens uBridge-created TAPs with `nio create_tap`: the same external-fd-holder shape QEMU uses; serial/ATM/POS ports stay relay). **The Ethernet switch has landed** — see `docs/features/ethernet-switch-kernel-datapath.md` (the switch absorbs a peer's anchor into its own kernel bridge; switch-to-switch cascades too). **The IOL runner container has landed** — see `docs/features/iol-docker-kernel-datapath.md` (persistent TAP anchors through the generic bridge module's swappable TAP leg; the unix-socket guest leg keeps one userspace hop, like IOU's fabric). +Still on the relay: the cloud paths (their anchors are ubridge-owned by design); and cross-compute kernel links need VXLAN/GENEVE encapsulation, which is a separate project (it would serve Docker links the same way). diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index 3e8665af2..5d820eab0 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -616,7 +616,7 @@ class KernelDatapathMixin: and the kernel-only ``window_drop`` type the time-window mode (``tc window_drop``: single outage, or recurring flaps with an optional per-cycle jitter). The program evaluates the modes in the - fixed order nth → quota → window. A mode is only sent when + fixed order nth → quota → window → flow. A mode is only sent when ``tc capabilities`` declares its token (``ebpf_modes``); builds predating the field keep their shipped modes (everything but window). diff --git a/gns3server/compute/nios/nio_bridge.py b/gns3server/compute/nios/nio_bridge.py index bdb00c9e5..987431446 100644 --- a/gns3server/compute/nios/nio_bridge.py +++ b/gns3server/compute/nios/nio_bridge.py @@ -21,9 +21,10 @@ Interface for kernel-datapath bridge NIOs. A NIO of this type instructs the node to enslave its adapter's veth host end into the named Linux kernel bridge instead of wiring a uBridge UDP relay. Frames then flow entirely in the kernel (veth -> bridge -> veth). Impairment -filters ride the NIO and run as one tc netem qdisc per veth host end (the -types without a netem equivalent — frequency_drop, bpf — stay relay-only); -markers and packet capture are served by uBridge's AF_PACKET modules on the +filters ride the NIO and run on the veth host end — one tc netem qdisc +(delay/loss/corrupt and the netem extensions), cls_bpf match-drop (bpf) and +the eBPF stateful classifier (frequency_drop, quota, window_drop); markers +and packet capture are served by uBridge's AF_PACKET modules on the veth host end (marker add_kernel / capture start_kernel). ``bridge`` may be ``None``: the anchor is then bridged elsewhere — an From d0b0cdbd380f767f11446b45f858d5ca23107357 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sun, 4 Oct 2026 12:38:53 +0800 Subject: [PATCH 076/101] fix(compute): a passive absorbed anchor no longer resets the switch's tc state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The peer end of an Ethernet switch fast-path link is a NIOBridge with no bridge of its own (the switch absorbed its anchor into the switch bridge). Its update reconcile still ran the impairment pass with the empty filter set the controller routes to that end — and an empty set is a plain `tc reset` on the shared anchor, silently wiping the netem/cls_bpf/eBPF the switch had just applied. Any link update did it (filters, markers, suspend, capture start/stop), and the outcome depended on the link's endpoint order: UdpLink.update() PUTs node1 then node2, so with the switch first its apply ran first and the peer's reset ran last (API green, data plane unimpaired), while with the peer first the reset ran first and the filter survived. That order dependence is also why the shipped e2e never caught it — it only created peer-first links. Ownership is now explicit: only the end that owns the anchor's tc state applies the impairment reconcile (_owns_anchor_impairments — a NIOBridge with `bridge is None` is a passive carrier, the same idiom the bridge-delete path already used). Markers, capture and the suspend carrier still apply on both ends: markers reconcile per process, so neither end can strip the other's, and the carrier converges by agreement. Red/green: two new docker unit tests fail on the unfixed tree with `tc reset ""` and pass after; the switch e2e also creates one link switch-first and asserts the netem qdisc and the RTT hold under that order (failed with the wipe, passes with the fix). Full unit suite 2110 passed; the switch e2e file passed end to end (378s). --- gns3server/compute/kernel_datapath.py | 45 ++++++++++++---- .../docker/test_docker_kernel_datapath.py | 53 +++++++++++++++++++ .../test_ethernet_switch_kernel_datapath.py | 26 ++++++++- 3 files changed, 111 insertions(+), 13 deletions(-) diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index 5d820eab0..b73f1afc3 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -135,6 +135,21 @@ class KernelDatapathMixin: # Link attach / detach # ------------------------------------------------------------------ + @staticmethod + def _owns_anchor_impairments(nio): + """ + Whether this end owns the tc state (netem, cls_bpf, eBPF) of the + anchor it was handed. A NIOBridge with no bridge of its own + (``bridge is None``) means the anchor is absorbed by an Ethernet + switch: the switch owns the anchor's bridge membership and its + impairment state — the controller routes the link's filters to the + switch end, so this NIO always carries an empty filter set, whose + reconcile is a plain ``tc reset``. Applying it here would silently + wipe the netem/bpf/eBPF the switch just put on the shared anchor. + """ + + return not (isinstance(nio, NIOBridge) and nio.bridge is None) + async def _kernel_attach(self, anchor, nio): """ Attach an anchor to the per-link kernel bridge carried by *nio* @@ -150,9 +165,11 @@ class KernelDatapathMixin: A NIO whose ``bridge`` is None means the anchor is bridged elsewhere — an Ethernet switch absorbed it into the switch's own bridge — so the membership work is skipped here and only the link - state the anchor carries (capture, markers, impairment filters) is - applied. Exactly one end of such a link owns the anchor's bridge - state; the other end is a passive carrier. + state this end owns is applied: capture and markers always, the + impairment filters only when this end owns the anchor's tc state + (see _owns_anchor_impairments). Exactly one end of such a link + owns the anchor's bridge and impairment state; the other end is a + passive carrier. """ if nio.bridge is not None: @@ -174,10 +191,13 @@ class KernelDatapathMixin: # (restored here on node restart, like the capture above); bpf # expressions become cls_bpf match-drop classifiers and # frequency_drop/quota/window_drop become the eBPF stateful - # classifier. - await self._ubridge_apply_netem(anchor, nio.filters) - await self._ubridge_apply_bpf_drops(anchor, nio.filters) - await self._ubridge_apply_ebpf_drops(anchor, nio.filters) + # classifier. A passive end owns none of this: its empty filter + # set is a tc reset, which would wipe the switch's qdisc off the + # shared anchor. + if self._owns_anchor_impairments(nio): + await self._ubridge_apply_netem(anchor, nio.filters) + await self._ubridge_apply_bpf_drops(anchor, nio.filters) + await self._ubridge_apply_ebpf_drops(anchor, nio.filters) async def _relay_attach(self, anchor, bridge_name, nio): """ @@ -226,16 +246,19 @@ class KernelDatapathMixin: """ Re-apply everything a kernel link carries on an already-attached anchor (filters and markers changed): no re-enslaving, no bridge - create. + create. A passive end (bridge is None — an anchor absorbed by an + Ethernet switch) leaves the anchor's impairment state to the + switch that owns it. """ await self._ubridge_apply_markers(anchor, nio) # The netem apply resets the interface first (the kernel merges # optional netem attrs on replace), so everything anchored on clsact # must re-apply after it: bpf drops flush + re-add, eBPF modes re-set. - await self._ubridge_apply_netem(anchor, nio.filters) - await self._ubridge_apply_bpf_drops(anchor, nio.filters) - await self._ubridge_apply_ebpf_drops(anchor, nio.filters) + if self._owns_anchor_impairments(nio): + await self._ubridge_apply_netem(anchor, nio.filters) + await self._ubridge_apply_bpf_drops(anchor, nio.filters) + await self._ubridge_apply_ebpf_drops(anchor, nio.filters) async def _remove_kernel_nio(self, nio, adapter_number, port_number=0): """ diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index f68ddb11f..f54be3dc9 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -322,6 +322,32 @@ async def test_connect_nio_kernel_eexist_still_reapplies_everything(vm): vm._ubridge_send.assert_any_call(f'tc netem set "{host_ifc}" delay 10') +@pytest.mark.asyncio +async def test_connect_nio_kernel_passive_anchor_skips_impairments(vm): + """ + A NIOBridge with no bridge of its own (bridge is None) is the absorbed + end of an Ethernet switch link: the anchor belongs to the switch, which + owns its tc state and applies the link's filters there (the controller + routes them to the switch end, so this NIO always carries {}). The + passive end must not reconcile impairments — an empty filter set is a + tc reset, which would silently wipe the switch's qdisc off the shared + anchor at every node start. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": None}) + + await vm._connect_nio(0, nio) + + tc_calls = [c.args[0] for c in vm._ubridge_send.call_args_list if str(c.args[0]).startswith("tc ")] + assert tc_calls == [], tc_calls + # The membership is the switch's as well: no per-link bridge work here. + assert not any("brctl" in c.args[0] for c in vm._ubridge_send.call_args_list) + + @pytest.mark.asyncio async def test_connect_nio_kernel_create_failure_propagates(vm): @@ -533,6 +559,33 @@ async def test_update_nio_kernel_turns_ebpf_modes_off(vm): assert "nth_drop" not in c.args[0] or "off" in c.args[0] +@pytest.mark.asyncio +async def test_update_nio_kernel_passive_anchor_skips_impairments(vm): + """ + The link-update reconcile on the absorbed (passive) end must leave the + anchor's tc state to the switch: no tc reset / netem / drop classifiers + from this end, or the reset would wipe what the switch applied moments + earlier in the same link update (switch-first links PUT this side last). + The carrier (suspend) is still driven here, and markers still reconcile + (this end may host some). + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + host_ifc, _ = vm._veth_names(0, 0) + vm._kernel_veths[(0, 0)] = host_ifc + nio = vm.manager.create_nio({"type": "nio_bridge", "bridge": None}) + vm._ethernet_adapters[0].add_nio(0, nio) + + await vm.adapter_update_nio_binding(0, nio) + + tc_calls = [c.args[0] for c in vm._ubridge_send.call_args_list if str(c.args[0]).startswith("tc ")] + assert tc_calls == [], tc_calls + # The carrier still follows the suspend flag on the shared anchor. + vm._ubridge_send.assert_any_call(f'link set "{host_ifc}" up') + + @pytest.mark.asyncio async def test_apply_ebpf_drops_requires_ebpf_capability(vm): """ diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index bbc0e00fa..6dc85fe2c 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -29,7 +29,9 @@ Real ICMP crossing the switch proves the datapath: * VLAN programming applies to absorbed anchors: different access VLANs isolate the routers (ping dies), the same VLAN bridges them again; * a ``delay 100`` filter lands as netem on the absorbed anchor (single - interface, switch-owned) and the measured RTT grows by ~100 ms; + interface, switch-owned) and the measured RTT grows by ~100 ms — on both + link endpoint orders (peer-first and switch-first; the order decides + which side's NIO update runs last and must not decide the outcome); * suspend admin-downs the anchor and kills the traffic; resume restores; * capture (hosted by the switch side) writes a pcap of ICMP records; * deleting a link detaches the anchor (which survives — the router owns @@ -128,8 +130,12 @@ def test_ethernet_switch_kernel_fast_path(): # The switch bridge exists from the moment the switch node does; a # link against a stopped router must not fail — the join defers. assert harness.bridge_members(sw_bridge) == [] + # link1 is created peer-first, link2 switch-first: the endpoint order + # decides which side's NIO update runs last, and only the switch-first + # order used to expose the passive end's tc-reset wipe (its empty-filter + # reconcile ran after the switch's apply). Both orders must work. link1 = compute.create_link(pid, (r1_id, *ETH), (switch_id, 0, 0)) - link2 = compute.create_link(pid, (r2_id, *ETH), (switch_id, 0, 1)) + link2 = compute.create_link(pid, (switch_id, 0, 1), (r2_id, *ETH)) assert link1["kernel_datapath"] is True, link1 assert link2["kernel_datapath"] is True, link2 assert harness.bridge_members(sw_bridge) == [] @@ -223,6 +229,22 @@ def test_ethernet_switch_kernel_fast_path(): fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < 50, fast + # The same single-owned impairment on link2 — created switch-first, + # the order whose peer NIO update runs LAST. The passive end's empty + # reconcile used to tc-reset the shared anchor right after the switch + # applied the netem, wiping it (API green, data plane unimpaired); + # the qdisc and the RTT must both hold under this order too. + compute.call("PUT", f"/projects/{pid}/links/{link2['link_id']}", {"filters": {"delay": [100]}}) + assert "netem" in harness.qdiscs(a2), harness.qdiscs(a2) + assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) + delayed2 = harness.wait_ping(c1, "10.1.1.2") + assert delayed2["success"] == 100, delayed2["raw"] + assert 60 <= delayed2["avg"] <= 250, (baseline, delayed2) + compute.call("PUT", f"/projects/{pid}/links/{link2['link_id']}", {"filters": {}}) + assert "netem" not in harness.qdiscs(a2) + fast2 = harness.wait_ping(c1, "10.1.1.2") + assert fast2["success"] == 100 and fast2["avg"] < 50, fast2 + # suspend: the anchor admin-downs, the link is dead; resume restores compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"suspend": True}) assert not harness.tap_up(a1) From e13e5f3804d76592b997c81288760310347eec7e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Sun, 4 Oct 2026 19:20:22 +0800 Subject: [PATCH 077/101] fix(tools): merge same-device entries before device command execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A batch carrying the same device twice lost all but the last entry's commands: the per-device command map kept only the last entry while result processing iterated every original entry, so each row reported the last entry's output under its own command list. c3c78f99a consolidated the merge into the MCP template renderer, but that only runs when a template is passed — the MCP direct-commands path and gns3-copilot's direct _run path were left uncovered. Normalize once in each tool's _run (single point covering all entry paths, restoring the c42b3a59b coverage) and keep the renderer merge as boundary normalization. Blocked-command info now also accumulates across same-device entries instead of last-winning in the per-device map. --- .../tools_v2/config_tools_nornir.py | 7 +- .../tools_v2/display_tools_nornir.py | 7 +- .../tools_v2/vpcs_tools_netmiko.py | 7 +- .../gns3_copilot/utils/device_configs.py | 71 ++++++ tests/agent/mcp/test_device_tools.py | 213 +++++++++++++++++- 5 files changed, 300 insertions(+), 5 deletions(-) create mode 100644 gns3server/agent/gns3_copilot/utils/device_configs.py diff --git a/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py b/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py index d16ef714c..485f02dbc 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py +++ b/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py @@ -47,7 +47,6 @@ from nornir.core.task import AggregatedResult, Result, Task from nornir_netmiko.tasks import netmiko_send_config from gns3server.agent.gns3_copilot.gns3_client import get_gns3_server_host - # Import custom Netmiko device types for GNS3 emulation # This registers gns3_huawei_telnet_ce and other custom device types # NOTE: Must be imported BEFORE any Nornir operations to ensure device types are registered @@ -55,6 +54,9 @@ from gns3server.agent.gns3_copilot.utils import get_device_ports_from_topology from gns3server.agent.gns3_copilot.utils.command_filter import ( filter_forbidden_commands, ) +from gns3server.agent.gns3_copilot.utils.device_configs import ( + merge_duplicate_device_configs, +) # Explicitly register custom device types to ensure they are available # This is a safety measure in case the auto-registration on import doesn't work @@ -196,6 +198,9 @@ class ExecuteMultipleDeviceConfigCommands(BaseTool): if len(device_configs_list) > 0 and "error" in device_configs_list[0]: return device_configs_list + # Merge same-device entries before anything keys by device_name + device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="config_commands") + # Filter forbidden commands and store blocked commands info device_configs_list, blocked_commands_map = self._filter_forbidden_commands_from_device_configs( device_configs_list diff --git a/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py b/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py index c7526f2c0..1569c224e 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py +++ b/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py @@ -47,7 +47,6 @@ from nornir.core.task import AggregatedResult, Result, Task from nornir_netmiko.tasks import netmiko_multiline from gns3server.agent.gns3_copilot.gns3_client import get_gns3_server_host - # Import custom Netmiko device types for GNS3 emulation # This registers gns3_huawei_telnet_ce and other custom device types # NOTE: Must be imported BEFORE any Nornir operations to ensure device types are registered @@ -55,6 +54,9 @@ from gns3server.agent.gns3_copilot.utils import get_device_ports_from_topology from gns3server.agent.gns3_copilot.utils.command_filter import ( filter_forbidden_commands, ) +from gns3server.agent.gns3_copilot.utils.device_configs import ( + merge_duplicate_device_configs, +) # Explicitly register custom device types to ensure they are available # This is a safety measure in case the auto-registration on import doesn't work @@ -199,6 +201,9 @@ class ExecuteMultipleDeviceCommands(BaseTool): if len(device_configs_list) > 0 and "error" in device_configs_list[0]: return device_configs_list + # Merge same-device entries before anything keys by device_name + device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="commands") + # Filter forbidden commands and store blocked commands info device_configs_list, blocked_commands_map = self._filter_forbidden_commands_from_device_configs( device_configs_list diff --git a/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py b/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py index 2148106e7..2444453f3 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py +++ b/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py @@ -47,11 +47,13 @@ from nornir.core.task import AggregatedResult, Result, Task from nornir_netmiko.tasks import netmiko_multiline from gns3server.agent.gns3_copilot.gns3_client import get_gns3_server_host - # Import custom Netmiko device types for GNS3 emulation # This registers gns3_vpcs_telnet and other custom device types # NOTE: Must be imported BEFORE any Nornir operations to ensure device types are registered from gns3server.agent.gns3_copilot.utils import get_device_ports_from_topology +from gns3server.agent.gns3_copilot.utils.device_configs import ( + merge_duplicate_device_configs, +) # Explicitly register VPCS device type to ensure it is available try: @@ -179,6 +181,9 @@ class VPCSCommands(BaseTool): if len(device_configs_list) > 0 and "error" in device_configs_list[0]: return device_configs_list + # Merge same-device entries before anything keys by device_name + device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="commands") + # Create a mapping of device names to their commands device_configs_map = self._configs_map(device_configs_list) diff --git a/gns3server/agent/gns3_copilot/utils/device_configs.py b/gns3server/agent/gns3_copilot/utils/device_configs.py new file mode 100644 index 000000000..e757eb089 --- /dev/null +++ b/gns3server/agent/gns3_copilot/utils/device_configs.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# GNS3-Copilot - AI-powered Network Lab Assistant for GNS3 +# +# This file is part of GNS3-Copilot project. +# +# GNS3-Copilot is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# GNS3-Copilot is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNS3-Copilot. If not, see . +# +# Copyright (C) 2025 Yue Guobin (岳国宾) +# Author: Yue Guobin (岳国宾) +# +# Project Home: https://github.com/yueguobin/gns3-copilot +# +""" +Device config entry normalization shared by the tools_v2 executors. + +Callers (LLM agents via MCP, or gns3-copilot directly) may send the same +device several times in one batch. Every downstream stage keys by +device_name, so duplicates must merge before execution, not after. +""" + +from typing import Any + + +def merge_duplicate_device_configs( + device_configs: list[dict[str, Any]], + commands_field: str, +) -> list[dict[str, Any]]: + """ + Merge entries that share a device_name, concatenating their command lists. + + Without this merge the per-device command map keeps only the last + entry's commands while result processing iterates every original entry — + duplicate devices would silently drop all but the last command batch and + every row would report that same output. + + Args: + device_configs: Device config entries; each holds device_name and a + command list under commands_field. + commands_field: Field carrying the commands ("commands" or + "config_commands" depending on the tool). + + Returns: + One entry per device_name, in first-appearance order. The first + entry's extra fields are preserved; later duplicates only contribute + their commands. + """ + merged: dict[str, dict[str, Any]] = {} + for config in device_configs: + name = config.get("device_name") + if not name: + continue + commands = list(config.get(commands_field, [])) + if name in merged: + merged[name][commands_field].extend(commands) + else: + entry = dict(config) + entry[commands_field] = commands + merged[name] = entry + return list(merged.values()) diff --git a/tests/agent/mcp/test_device_tools.py b/tests/agent/mcp/test_device_tools.py index db65412c8..e68075d3c 100644 --- a/tests/agent/mcp/test_device_tools.py +++ b/tests/agent/mcp/test_device_tools.py @@ -1,14 +1,24 @@ """ Device config tool tests with mocked topology and Nornir layers. -Covers the VPCS node-type guard and the error contract shared by -device_config_send / device_show_run / vpcs_config_set. +Covers the VPCS node-type guard, the error contract shared by +device_config_send / device_show_run / vpcs_config_set, and the +same-device entry merging that keeps duplicate entries from +overwriting each other's commands and outputs. """ import json from unittest.mock import MagicMock, patch +from gns3server.agent.gns3_copilot.utils.device_configs import ( + merge_duplicate_device_configs, +) + VPCS_MOD = "gns3server.agent.gns3_copilot.tools_v2.vpcs_tools_netmiko" +DISPLAY_MOD = "gns3server.agent.gns3_copilot.tools_v2.display_tools_nornir" +CONFIG_MOD = "gns3server.agent.gns3_copilot.tools_v2.config_tools_nornir" + +PROJECT_ID = "0c0fde25-6ead-4413-a283-ea8fd2324291" def _topology_ports(node_type): @@ -16,6 +26,26 @@ def _topology_ports(node_type): return {"PC1": {"port": 5000, "node_type": node_type}} +def _topology_two_devices(): + """Mocked topology with two console-reachable devices.""" + return { + "R1": {"port": 5000, "node_type": "iou"}, + "R2": {"port": 5001, "node_type": "iou"}, + } + + +def _aggregated_result(output_by_device): + """Real AggregatedResult: device -> MultiResult with one successful Result.""" + from nornir.core.task import AggregatedResult, MultiResult, Result + + aggregated = AggregatedResult("task") + for name, output in output_by_device.items(): + multi = MultiResult("task") + multi.append(Result(host=None, result=output, failed=False)) + aggregated[name] = multi + return aggregated + + class TestVPCSNodeTypeGuard: def test_non_vpcs_node_is_rejected(self): from gns3server.agent.gns3_copilot.tools_v2.vpcs_tools_netmiko import VPCSCommands @@ -161,3 +191,182 @@ class TestDeviceToolErrorContract: assert len(result) == 1 assert result[0]["status"] == "failed" assert "Template rendering failed" in result[0]["error"] + + +class TestDuplicateDeviceMerging: + """ + Same-device entries in one batch must merge before execution. + + Regression: the per-device command map kept only the last entry's + commands while result processing iterated every original entry, so all + rows reported the last entry's output (first reproduced with the MCP + direct-commands path, no template). + """ + + def test_show_run_merges_same_device_entries(self): + from gns3server.agent.gns3_copilot.tools_v2.display_tools_nornir import ( + ExecuteMultipleDeviceCommands, + ) + + nornir = MagicMock() + nornir.run.return_value = _aggregated_result({"R1": "merged output", "R2": "r2 output"}) + with ( + patch(f"{DISPLAY_MOD}.get_device_ports_from_topology", return_value=_topology_two_devices()), + patch.object(ExecuteMultipleDeviceCommands, "_initialize_nornir", return_value=nornir), + ): + result = ExecuteMultipleDeviceCommands()._run( + json.dumps( + { + "project_id": PROJECT_ID, + "device_configs": [ + {"device_name": "R1", "commands": ["show ip route"]}, + {"device_name": "R2", "commands": ["show version"]}, + {"device_name": "R1", "commands": ["show ip interface brief"]}, + ], + } + ) + ) + + # every command reached the execution layer, in order + assert nornir.run.call_args.kwargs["device_configs_map"] == { + "R1": ["show ip route", "show ip interface brief"], + "R2": ["show version"], + } + # one truthful row per device + assert [row["device_name"] for row in result] == ["R1", "R2"] + assert result[0]["status"] == "success" + assert result[0]["output"] == "merged output" + assert result[0]["diagnostic_commands"] == ["show ip route", "show ip interface brief"] + + def test_config_send_merges_same_device_entries(self): + from gns3server.agent.gns3_copilot.tools_v2.config_tools_nornir import ( + ExecuteMultipleDeviceConfigCommands, + ) + + nornir = MagicMock() + nornir.run.return_value = _aggregated_result({"R1": "merged output"}) + with ( + patch(f"{CONFIG_MOD}.get_device_ports_from_topology", return_value={"R1": {"port": 5000, "node_type": "iou"}}), + patch.object(ExecuteMultipleDeviceConfigCommands, "_initialize_nornir", return_value=nornir), + ): + result = ExecuteMultipleDeviceConfigCommands()._run( + json.dumps( + { + "project_id": PROJECT_ID, + "device_configs": [ + {"device_name": "R1", "config_commands": ["int lo0"]}, + {"device_name": "R1", "config_commands": ["ip address 1.1.1.1 255.255.255.255", "exit"]}, + ], + } + ) + ) + + assert nornir.run.call_args.kwargs["device_configs_map"] == { + "R1": ["int lo0", "ip address 1.1.1.1 255.255.255.255", "exit"], + } + assert len(result) == 1 + assert result[0]["status"] == "success" + assert result[0]["config_commands"] == ["int lo0", "ip address 1.1.1.1 255.255.255.255", "exit"] + + def test_vpcs_merges_same_device_entries(self): + from gns3server.agent.gns3_copilot.tools_v2.vpcs_tools_netmiko import VPCSCommands + + nornir = MagicMock() + host_result = MagicMock(failed=False) + host_result.result = "OK" + nornir.run.return_value = {"PC1": host_result} + with ( + patch(f"{VPCS_MOD}.get_device_ports_from_topology", return_value=_topology_ports("vpcs")), + patch.object(VPCSCommands, "_initialize_nornir", return_value=nornir), + ): + result = VPCSCommands()._run( + json.dumps( + { + "project_id": PROJECT_ID, + "device_configs": [ + {"device_name": "PC1", "commands": ["ip 10.0.0.1/24"]}, + {"device_name": "PC1", "commands": ["save"]}, + ], + } + ) + ) + + assert nornir.run.call_args.kwargs["device_configs_map"] == {"PC1": ["ip 10.0.0.1/24", "save"]} + assert len(result) == 1 + assert result[0]["status"] == "success" + assert result[0]["commands"] == ["ip 10.0.0.1/24", "save"] + + def test_blocked_commands_accumulate_across_duplicate_entries(self): + """Blocked-command info must not last-win between same-device entries.""" + from gns3server.agent.gns3_copilot.tools_v2.display_tools_nornir import ( + ExecuteMultipleDeviceCommands, + ) + + def fake_filter(commands): + blocked = {c: "matched forbidden pattern" for c in commands if c.startswith("debug")} + allowed = [c for c in commands if not c.startswith("debug")] + return allowed, blocked + + nornir = MagicMock() + nornir.run.return_value = _aggregated_result({"R1": "ok"}) + with ( + patch(f"{DISPLAY_MOD}.get_device_ports_from_topology", return_value={"R1": {"port": 5000, "node_type": "iou"}}), + patch.object(ExecuteMultipleDeviceCommands, "_initialize_nornir", return_value=nornir), + patch(f"{DISPLAY_MOD}.filter_forbidden_commands", side_effect=fake_filter), + ): + result = ExecuteMultipleDeviceCommands()._run( + json.dumps( + { + "project_id": PROJECT_ID, + "device_configs": [ + {"device_name": "R1", "commands": ["show version", "debug ip routing"]}, + {"device_name": "R1", "commands": ["debug ospf events", "show clock"]}, + ], + } + ) + ) + + assert nornir.run.call_args.kwargs["device_configs_map"] == {"R1": ["show version", "show clock"]} + assert result[0]["status"] == "partial_success" + assert set(result[0]["blocked_commands"]) == {"debug ip routing", "debug ospf events"} + + +class TestMergeDuplicateDeviceConfigs: + """Unit tests for the shared normalization helper.""" + + def test_merges_in_first_appearance_order(self): + merged = merge_duplicate_device_configs( + [ + {"device_name": "R1", "commands": ["a"]}, + {"device_name": "R2", "commands": ["b"]}, + {"device_name": "R1", "commands": ["c", "d"]}, + ], + commands_field="commands", + ) + assert merged == [ + {"device_name": "R1", "commands": ["a", "c", "d"]}, + {"device_name": "R2", "commands": ["b"]}, + ] + + def test_preserves_first_entry_fields_without_mutating_input(self): + original = [ + {"device_name": "R1", "commands": ["a"], "extra": 1}, + {"device_name": "R1", "commands": ["b"]}, + ] + merged = merge_duplicate_device_configs(original, commands_field="commands") + assert merged == [{"device_name": "R1", "commands": ["a", "b"], "extra": 1}] + assert original[0]["commands"] == ["a"] + + def test_entry_without_device_name_is_dropped(self): + merged = merge_duplicate_device_configs( + [{"commands": ["a"]}, {"device_name": "R1", "commands": ["b"]}], + commands_field="commands", + ) + assert merged == [{"device_name": "R1", "commands": ["b"]}] + + def test_missing_commands_field_defaults_to_empty(self): + merged = merge_duplicate_device_configs( + [{"device_name": "R1"}, {"device_name": "R1", "commands": ["b"]}], + commands_field="commands", + ) + assert merged == [{"device_name": "R1", "commands": ["b"]}] From 7d04d08dba84b5f340ae39b6584be368976034ce Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 11:54:57 +0800 Subject: [PATCH 078/101] fix: repair imports lost in the 3.1 rebase and meet the new lint baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rebase onto upstream 3.1 (mypy + ruff I001/F401 waves) left the tree short of the newly enforced baselines in three ways: * contextlib / UbridgeError imports dropped as false duplicates during conflict resolution left NameErrors in iou_vm and qemu_vm (F821); the Dynamips router grabbed the compute-level NIOTAP instead of the hypervisor one, breaking TAP anchoring (test failure). * unused imports (KERNEL_UNSUPPORTED_FILTERS, UbridgeError, List) and unsorted blocks left behind where the stack's later commits moved the last usage elsewhere — ruff I001/F401/RUF022 now enforce these. * mypy, now baseline-free: cache dict annotations in tc_probe, the locators candidate list, the resolved-params None branch in the MCP runner, and validate_bpf_syntax keeps the upstream Dict[str, Any] return (the strict Optional[str] was wrong for the "valid" bool entry). The e2e raw-frame probe resolves its compiler via shutil.which (S607) and zips with strict= (B905); the remaining subprocess call is S603 by nature and gets a per-file ignore like other test files. ruff check/format clean, mypy at upstream parity (no new errors), full unit suite 2167 passed. --- .../tools_v2/config_tools_nornir.py | 1 + .../tools_v2/display_tools_nornir.py | 1 + .../tools_v2/vpcs_tools_netmiko.py | 1 + gns3server/agent/mcp/__init__.py | 8 +++---- gns3server/agent/mcp/locators.py | 15 +++++++----- gns3server/api/routes/compute/capabilities.py | 2 +- gns3server/api/routes/compute/docker_nodes.py | 12 ++++++---- .../routes/compute/ethernet_switch_nodes.py | 18 +++++++------- .../compute/docker/docker_kernel_datapath.py | 5 ++-- gns3server/compute/docker/docker_vm.py | 3 +-- gns3server/compute/docker/iol_docker_vm.py | 1 - gns3server/compute/dynamips/nodes/router.py | 2 +- gns3server/compute/iou/iou_vm.py | 5 ++-- gns3server/compute/kernel_datapath.py | 9 ++++--- gns3server/compute/qemu/qemu_vm.py | 2 ++ gns3server/compute/ubridge/tc_probe.py | 24 +++++++++---------- gns3server/controller/udp_link.py | 1 - gns3server/schemas/__init__.py | 14 +++++++++-- gns3server/schemas/controller/computes.py | 3 ++- gns3server/utils/packet_filter_validation.py | 12 +++++----- pyproject.toml | 1 + tests/agent/mcp/test_device_tools.py | 8 +++++-- tests/agent/mcp/test_locators.py | 24 ++++++++++--------- tests/api/routes/compute/test_capabilities.py | 4 +++- .../docker/test_docker_kernel_datapath.py | 23 +++++++++--------- .../docker/test_iol_docker_kernel_datapath.py | 12 ++++------ tests/compute/docker/test_vendor_docker_vm.py | 1 + .../dynamips/test_dynamips_kernel_datapath.py | 21 +++++++--------- tests/compute/iou/test_iou_kernel_datapath.py | 11 ++++----- .../compute/qemu/test_qemu_kernel_datapath.py | 13 ++++------ tests/compute/ubridge/test_tc_probe.py | 4 ++-- tests/controller/test_compute.py | 13 +++++++++- tests/controller/test_kernel_datapath_link.py | 9 +++---- tests/e2e/harness.py | 4 +++- tests/e2e/test_docker_link_local_frames.py | 14 ++++++----- 35 files changed, 165 insertions(+), 136 deletions(-) diff --git a/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py b/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py index 485f02dbc..4bf507b25 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py +++ b/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py @@ -47,6 +47,7 @@ from nornir.core.task import AggregatedResult, Result, Task from nornir_netmiko.tasks import netmiko_send_config from gns3server.agent.gns3_copilot.gns3_client import get_gns3_server_host + # Import custom Netmiko device types for GNS3 emulation # This registers gns3_huawei_telnet_ce and other custom device types # NOTE: Must be imported BEFORE any Nornir operations to ensure device types are registered diff --git a/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py b/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py index 1569c224e..afdeac035 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py +++ b/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py @@ -47,6 +47,7 @@ from nornir.core.task import AggregatedResult, Result, Task from nornir_netmiko.tasks import netmiko_multiline from gns3server.agent.gns3_copilot.gns3_client import get_gns3_server_host + # Import custom Netmiko device types for GNS3 emulation # This registers gns3_huawei_telnet_ce and other custom device types # NOTE: Must be imported BEFORE any Nornir operations to ensure device types are registered diff --git a/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py b/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py index 2444453f3..31ed0c916 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py +++ b/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py @@ -47,6 +47,7 @@ from nornir.core.task import AggregatedResult, Result, Task from nornir_netmiko.tasks import netmiko_multiline from gns3server.agent.gns3_copilot.gns3_client import get_gns3_server_host + # Import custom Netmiko device types for GNS3 emulation # This registers gns3_vpcs_telnet and other custom device types # NOTE: Must be imported BEFORE any Nornir operations to ensure device types are registered diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index 232eb0193..00bef7e88 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -346,13 +346,13 @@ def _run_handler_sync(handler, params: dict[str, Any]) -> list[dict[str, Any]]: # Name-shaped project_id/node_id values become UUIDs before the handler # runs; a name that matches nothing fails fast with the known names. - params, resolution_error = resolve_locators(params, run) - if resolution_error is not None: + resolved_params, resolution_error = resolve_locators(params, run) + if resolved_params is None: result = resolution_error else: - result = handler(params, ctx) + result = handler(resolved_params, ctx) # backstop for UUID transcription slips: "Did you mean ...?" - result = add_did_you_mean(result, params, run) + result = add_did_you_mean(result, resolved_params, run) return [{"type": "text", "text": json.dumps(result, ensure_ascii=False, default=str)}] diff --git a/gns3server/agent/mcp/locators.py b/gns3server/agent/mcp/locators.py index 4bffb8014..727e0b208 100644 --- a/gns3server/agent/mcp/locators.py +++ b/gns3server/agent/mcp/locators.py @@ -37,9 +37,10 @@ distance 2 earns a "Did you mean ..." hint. import re from typing import Any, Callable -from .projects import list_projects_handler from gns3server.agent.gns3_copilot.gns3_client.api_handlers import get_links_handler, get_nodes_handler +from .projects import list_projects_handler + _UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") _ID_IN_TEXT_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") @@ -94,9 +95,7 @@ def resolve_locators( return params, None -def add_did_you_mean( - result: Any, params: dict[str, Any], run: Callable[[Callable, dict], Any] -) -> Any: +def add_did_you_mean(result: Any, params: dict[str, Any], run: Callable[[Callable, dict], Any]) -> Any: """ Append a "Did you mean ...?" hint to an error reply that names a non-existent UUID, when exactly one in-scope id is within edit @@ -118,7 +117,11 @@ def add_did_you_mean( hints = [] for bad in bad_ids: - close = [(cid, label) for cid, label in candidates if bad != cid and _edit_distance_at_most(bad.lower(), cid.lower(), 2)] + close = [ + (cid, label) + for cid, label in candidates + if bad != cid and _edit_distance_at_most(bad.lower(), cid.lower(), 2) + ] if len(close) == 1: cid, label = close[0] hints.append(f"Did you mean {label} ({cid})?") @@ -177,7 +180,7 @@ def _collect_candidates(params: dict[str, Any], run: Callable[[Callable, dict], just yields what was gathered so far. """ - candidates = [] + candidates: list[tuple[str, str]] = [] try: projects = _as_items(run(list_projects_handler, {}), "projects") or [] candidates.extend((p["project_id"], f"project '{p.get('name', '?')}'") for p in projects if p.get("project_id")) diff --git a/gns3server/api/routes/compute/capabilities.py b/gns3server/api/routes/compute/capabilities.py index 9e6a2d2bd..d0f969a81 100644 --- a/gns3server/api/routes/compute/capabilities.py +++ b/gns3server/api/routes/compute/capabilities.py @@ -28,8 +28,8 @@ from gns3server.compute import MODULES from gns3server.compute.ubridge.tc_probe import ( probe_bridge_tap_support, probe_iol_tap_support, - probe_tc_capabilities, probe_tap_support, + probe_tc_capabilities, ) from gns3server.utils.path import get_default_project_directory from gns3server.utils.tc_capabilities import usable_ebpf_modes diff --git a/gns3server/api/routes/compute/docker_nodes.py b/gns3server/api/routes/compute/docker_nodes.py index 5e7f1af98..a66084f5b 100644 --- a/gns3server/api/routes/compute/docker_nodes.py +++ b/gns3server/api/routes/compute/docker_nodes.py @@ -264,8 +264,10 @@ async def duplicate_docker_node( dependencies=[Depends(compute_authentication)], ) async def create_docker_node_nio( - adapter_number: int, port_number: int, nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], - node: DockerVM = Depends(dep_node) + adapter_number: int, + port_number: int, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], + node: DockerVM = Depends(dep_node), ) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Add a NIO (Network Input/Output) to the node. @@ -284,8 +286,10 @@ async def create_docker_node_nio( dependencies=[Depends(compute_authentication)], ) async def update_docker_node_nio( - adapter_number: int, port_number: int, nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], - node: DockerVM = Depends(dep_node) + adapter_number: int, + port_number: int, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], + node: DockerVM = Depends(dep_node), ) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Update a NIO (Network Input/Output) on the node. diff --git a/gns3server/api/routes/compute/ethernet_switch_nodes.py b/gns3server/api/routes/compute/ethernet_switch_nodes.py index 47b8b8604..ba10182d8 100644 --- a/gns3server/api/routes/compute/ethernet_switch_nodes.py +++ b/gns3server/api/routes/compute/ethernet_switch_nodes.py @@ -103,20 +103,20 @@ async def update_ethernet_switch( Update an Ethernet switch. """ - node_data = jsonable_encoder(node_data, exclude_unset=True) - if "name" in node_data and node.name != node_data["name"]: - node.name = node_data["name"] - if "usage" in node_data: - node.usage = node_data["usage"] - if "ports_mapping" in node_data: + data = jsonable_encoder(node_data, exclude_unset=True) + if "name" in data and node.name != data["name"]: + node.name = data["name"] + if "usage" in data: + node.usage = data["usage"] + if "ports_mapping" in data: # capture the mapping before the setter replaces it: the VLAN # reconcile diffs old against new to touch only what changed, in # place (the setter itself keeps the port-count guard) previous_mapping = [dict(port) for port in node.ports_mapping] - node.ports_mapping = node_data["ports_mapping"] + node.ports_mapping = data["ports_mapping"] await node.update_port_settings(previous_mapping) - if "console_type" in node_data: - node.console_type = node_data["console_type"] + if "console_type" in data: + node.console_type = data["console_type"] node.updated() return node.asdict() diff --git a/gns3server/compute/docker/docker_kernel_datapath.py b/gns3server/compute/docker/docker_kernel_datapath.py index f0b215035..06ce57b44 100644 --- a/gns3server/compute/docker/docker_kernel_datapath.py +++ b/gns3server/compute/docker/docker_kernel_datapath.py @@ -24,14 +24,13 @@ markers, attach/detach, adapter carrier — lives in """ import contextlib +import logging from gns3server.compute.kernel_datapath import KernelDatapathMixin from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError - -from .docker_error import DockerError from gns3server.utils.kernel_anchor import anchor_suffix, kernel_anchor_name -import logging +from .docker_error import DockerError log = logging.getLogger(__name__) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 9d123478c..3d6f611cb 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -33,14 +33,13 @@ import aiohttp import psutil from gns3server.compute.compute_error import ComputeError -from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError +from gns3server.compute.ubridge.ubridge_error import UbridgeNamespaceError from gns3server.utils import int_to_macaddress, macaddress_to_int from gns3server.utils.asyncio import monitor_process, wait_for_file_creation, wait_run_in_executor from gns3server.utils.asyncio.raw_command_server import AsyncioRawCommandServer from gns3server.utils.asyncio.ssh_server import AsyncioSSHServer from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer from gns3server.utils.hostname import is_rfc1123_hostname_valid -from gns3server.utils.packet_filter_validation import KERNEL_UNSUPPORTED_FILTERS from ..adapters.ethernet_adapter import EthernetAdapter from ..base_node import BaseNode diff --git a/gns3server/compute/docker/iol_docker_vm.py b/gns3server/compute/docker/iol_docker_vm.py index 212ef1390..a41b66e56 100644 --- a/gns3server/compute/docker/iol_docker_vm.py +++ b/gns3server/compute/docker/iol_docker_vm.py @@ -672,4 +672,3 @@ class IOLDockerVM(VendorDockerVM): if self._kernel_host_ifc(adapter_number, port_number) is not None: await KernelDatapathMixin._set_adapter_carrier(self, adapter_number, connected, port_number) - diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index f9e4e8bae..05c055b13 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -42,10 +42,10 @@ from ...base_node import BaseNode from ...error import NodeError from ...kernel_datapath import KernelDatapathMixin from ...nios.nio_bridge import NIOBridge -from ...nios.nio_tap import NIOTAP from ...ubridge.ubridge_error import UbridgeError from ..adapters.adapter import ETHERNET_ADAPTERS, ETHERNET_WICS from ..dynamips_error import DynamipsError +from ..nios.nio_tap import NIOTAP class Router(KernelDatapathMixin, BaseNode): diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index af3d82c1a..ec7aed43b 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -22,6 +22,7 @@ order to run an IOU VM. import asyncio import binascii import configparser +import contextlib import functools import glob import hashlib @@ -951,9 +952,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): raise self._kernel_error( "Bay {bay}/{unit} of IOU '{name}' has no TAP anchor to carry a kernel link " "(serial ports stay on the relay datapath, and a node whose uBridge lacks " - "iol_bridge add_nio_tap runs relay-only)".format( - bay=adapter_number, unit=port_number, name=self._name - ) + "iol_bridge add_nio_tap runs relay-only)".format(bay=adapter_number, unit=port_number, name=self._name) ) await self._ubridge_send( 'iol_bridge add_nio_tap {name} {iol_id} {bay} {unit} "{tap}"'.format( diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index b73f1afc3..09e80ec4a 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -41,17 +41,16 @@ A node type using this mixin provides: failures surface as ``DockerError`` / ``QemuError`` rather than ``NodeError``. """ -import os import contextlib +import logging +import os -from gns3server.utils.tc_capabilities import FILTER_EBPF_MODES, parse_tc_capabilities, usable_ebpf_modes -from gns3server.compute.ubridge.ubridge_error import UbridgeError from gns3server.compute.error import NodeError +from gns3server.compute.ubridge.ubridge_error import UbridgeError +from gns3server.utils.tc_capabilities import FILTER_EBPF_MODES, parse_tc_capabilities, usable_ebpf_modes from .nios.nio_bridge import NIOBridge -import logging - log = logging.getLogger(__name__) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 873986ace..410094348 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -20,6 +20,7 @@ order to run a QEMU VM. """ import asyncio +import contextlib import json import logging import math @@ -53,6 +54,7 @@ from ..kernel_datapath import KernelDatapathMixin from ..nios.nio_bridge import NIOBridge from ..nios.nio_tap import NIOTAP from ..nios.nio_udp import NIOUDP +from ..ubridge.ubridge_error import UbridgeError from .qemu_error import QemuError from .utils.qcow2 import Qcow2, Qcow2Error from .utils.ziputils import pack_zip, unpack_zip diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py index 7e5e19d1c..fdff74d6d 100644 --- a/gns3server/compute/ubridge/tc_probe.py +++ b/gns3server/compute/ubridge/tc_probe.py @@ -32,29 +32,31 @@ failure (missing binary, old build answering "Unknown command", timeout) means unknown, reported as None and cached as such. """ -import os -import uuid -import shutil import asyncio -import tempfile -import logging import contextlib +import logging +import os +import shutil +import tempfile +import uuid +from typing import Optional from gns3server.config import Config from gns3server.utils.tc_capabilities import parse_tc_capabilities + from .hypervisor import Hypervisor from .ubridge_error import UbridgeError log = logging.getLogger(__name__) # binary identity -> parsed report (None = probed and unusable) -_cache = {} +_cache: dict[tuple[str, int, int], Optional[dict[str, str]]] = {} # binary identity -> whether the tap module works (None = unknown) -_tap_cache = {} +_tap_cache: dict[tuple[str, int, int], Optional[bool]] = {} # binary identity -> whether iol_bridge can bind a port to a TAP (None = unknown) -_iol_tap_cache = {} +_iol_tap_cache: dict[tuple[str, int, int], Optional[bool]] = {} # binary identity -> whether the bridge module can release a named TAP NIO (None = unknown) -_bridge_tap_cache = {} +_bridge_tap_cache: dict[tuple[str, int, int], Optional[bool]] = {} _lock = asyncio.Lock() @@ -252,9 +254,7 @@ async def _probe_iol_tap(path, config, timeout): await hypervisor.send(f"iol_bridge create {bridge} {_IOL_PROBE_BRIDGE_ID}") try: await hypervisor.send(f"tap create {tap}") - await hypervisor.send( - f"iol_bridge add_nio_tap {bridge} {_IOL_PROBE_BRIDGE_ID - 1} 0 0 {tap}" - ) + await hypervisor.send(f"iol_bridge add_nio_tap {bridge} {_IOL_PROBE_BRIDGE_ID - 1} 0 0 {tap}") await hypervisor.send(f"iol_bridge delete_nio_tap {bridge} 0 0") finally: # iol_bridge delete releases every port NIO (the TAP fd), diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 391bb04e3..8e607f1b5 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -23,7 +23,6 @@ from gns3server.config import Config from gns3server.utils.application_id import is_iol_runner_environment from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_anchor_type, kernel_cascade_names from gns3server.utils.packet_filter_validation import ( - KERNEL_UNSUPPORTED_FILTERS, split_kernel_only_features, validate_bpf_syntax, ) diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index e66a39a4a..0a26874a5 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -101,7 +101,17 @@ from .compute.iou_nodes import IOU, IOUCreate, IOUStart, IOUUpdate from .compute.nat_nodes import NAT, NATCreate, NATUpdate # Compute schemas -from .compute.nios import AnchorNIO, TAPNIO, UDPNIO, BridgeNIO, BatchNIOCreate, BatchNIOEntry, EthernetNIO, MarkerRebuild, MarkerToggle +from .compute.nios import ( + TAPNIO, + UDPNIO, + AnchorNIO, + BatchNIOCreate, + BatchNIOEntry, + BridgeNIO, + EthernetNIO, + MarkerRebuild, + MarkerToggle, +) from .compute.qemu_nodes import Qemu, QemuCreate, QemuUpdate from .compute.virtualbox_nodes import VirtualBox, VirtualBoxCreate, VirtualBoxUpdate from .compute.vmware_nodes import VMware, VMwareCreate, VMwareUpdate @@ -159,12 +169,12 @@ __all__ = [ "ATMSwitch", "ATMSwitchCreate", "ATMSwitchUpdate", + "AnchorNIO", "ApiKeyCreate", "Appliance", "ApplianceVersion", "ApplianceVersionV8", "AutoIdlePC", - "AnchorNIO", "BatchNIOCreate", "BatchNIOEntry", "BridgeNIO", diff --git a/gns3server/schemas/controller/computes.py b/gns3server/schemas/controller/computes.py index 4924827e5..5742e4391 100644 --- a/gns3server/schemas/controller/computes.py +++ b/gns3server/schemas/controller/computes.py @@ -16,11 +16,12 @@ import uuid from enum import Enum -from typing import Any, List, Optional, Union +from typing import Any, Optional, Union from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator from .base import DateTimeModelMixin + # The capabilities payload has one definition (controller.capabilities): a # second, private copy here silently stripped every field it did not know # about from GET /computes/{id} — which is how ubridge_tap and diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index c9c3a6d96..cee97e59c 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -5,7 +5,7 @@ Packet filter parameter validation utilities. import logging import re import subprocess -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Set, Tuple log = logging.getLogger(__name__) @@ -47,7 +47,7 @@ _RATE_UNITS = { _RATE_MAX_BITS = 100 * 1000**3 -def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Optional[str]]: +def validate_bpf_syntax(bpf_expression: str) -> Dict[str, Any]: """ Validate BPF filter expression syntax using tcpdump. @@ -142,9 +142,7 @@ def _validate_int_parameter(filter_type: str, rules: dict, i: int, value: Any) - else: int_value = int(value) except (ValueError, TypeError): - raise FilterValidationError( - f"{filter_type} parameter {rules['names'][i]} must be an integer, got: {value}" - ) + raise FilterValidationError(f"{filter_type} parameter {rules['names'][i]} must be an integer, got: {value}") # Range validation min_val, max_val = rules["ranges"][i] @@ -441,7 +439,9 @@ def validate_all_filters(filters: Dict[str, List[Any]]) -> None: if "reorder" in filters and "delay" not in filters: raise FilterValidationError("reorder requires delay") if "gemodel" in filters and "packet_loss" in filters: - raise FilterValidationError("gemodel and packet_loss are mutually exclusive (both map to the netem loss keyword)") + raise FilterValidationError( + "gemodel and packet_loss are mutually exclusive (both map to the netem loss keyword)" + ) window = filters.get("window_drop") if window and len(window) >= 4 and int(window[3]) < int(window[1]): raise FilterValidationError("window_drop period must be greater than or equal to the outage length") diff --git a/pyproject.toml b/pyproject.toml index 65c67d1bd..fc2d9b861 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,6 +254,7 @@ extend-immutable-calls = [ "tests/controller/test_project.py" = ["RUF015"] "tests/controller/test_snapshot.py" = ["RUF015"] "tests/controller/test_udp_link.py" = ["B006"] +"tests/e2e/test_docker_link_local_frames.py" = ["S603"] "tests/test_server.py" = ["E722"] "tests/test_topologies.py" = ["B011"] "tests/test_utils.py" = ["F403", "F405"] diff --git a/tests/agent/mcp/test_device_tools.py b/tests/agent/mcp/test_device_tools.py index e68075d3c..3a4308053 100644 --- a/tests/agent/mcp/test_device_tools.py +++ b/tests/agent/mcp/test_device_tools.py @@ -246,7 +246,9 @@ class TestDuplicateDeviceMerging: nornir = MagicMock() nornir.run.return_value = _aggregated_result({"R1": "merged output"}) with ( - patch(f"{CONFIG_MOD}.get_device_ports_from_topology", return_value={"R1": {"port": 5000, "node_type": "iou"}}), + patch( + f"{CONFIG_MOD}.get_device_ports_from_topology", return_value={"R1": {"port": 5000, "node_type": "iou"}} + ), patch.object(ExecuteMultipleDeviceConfigCommands, "_initialize_nornir", return_value=nornir), ): result = ExecuteMultipleDeviceConfigCommands()._run( @@ -310,7 +312,9 @@ class TestDuplicateDeviceMerging: nornir = MagicMock() nornir.run.return_value = _aggregated_result({"R1": "ok"}) with ( - patch(f"{DISPLAY_MOD}.get_device_ports_from_topology", return_value={"R1": {"port": 5000, "node_type": "iou"}}), + patch( + f"{DISPLAY_MOD}.get_device_ports_from_topology", return_value={"R1": {"port": 5000, "node_type": "iou"}} + ), patch.object(ExecuteMultipleDeviceCommands, "_initialize_nornir", return_value=nornir), patch(f"{DISPLAY_MOD}.filter_forbidden_commands", side_effect=fake_filter), ): diff --git a/tests/agent/mcp/test_locators.py b/tests/agent/mcp/test_locators.py index ee4bd1cee..993000ab8 100644 --- a/tests/agent/mcp/test_locators.py +++ b/tests/agent/mcp/test_locators.py @@ -69,8 +69,12 @@ def test_uuid_params_pass_through_without_calls(): def test_project_name_resolved(): """A name-shaped project_id becomes the project's UUID.""" - run, calls = _run_factory(projects=[{"project_id": "11111111-1111-1111-1111-111111111111", "name": "other"}, - {"project_id": PROJECT_ID, "name": "my-lab"}]) + run, calls = _run_factory( + projects=[ + {"project_id": "11111111-1111-1111-1111-111111111111", "name": "other"}, + {"project_id": PROJECT_ID, "name": "my-lab"}, + ] + ) params, error = resolve_locators({"project_id": "my-lab"}, run) assert error is None assert params["project_id"] == PROJECT_ID @@ -88,8 +92,9 @@ def test_project_name_not_found_lists_known_names(): def test_node_name_resolved_within_project(): - run, calls = _run_factory(projects=[{"project_id": PROJECT_ID, "name": "my-lab"}], - nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}]) + run, calls = _run_factory( + projects=[{"project_id": PROJECT_ID, "name": "my-lab"}], nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}] + ) params, error = resolve_locators({"project_id": "my-lab", "node_id": "FRRDocker-4"}, run) assert error is None assert params["project_id"] == PROJECT_ID @@ -125,11 +130,10 @@ def test_handler_failure_during_resolution_is_reported(): def test_did_you_mean_appended_for_transcription_slip(): """The 2c9e -> 3b9e class of typo earns exactly one hint.""" - run, _calls = _run_factory(projects=[{"project_id": PROJECT_ID, "name": "my-lab"}], - nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}]) - result = add_did_you_mean( - {"error": f"Node ID {NODE_ID_TYPO} doesn't exist"}, {"project_id": PROJECT_ID}, run + run, _calls = _run_factory( + projects=[{"project_id": PROJECT_ID, "name": "my-lab"}], nodes=[{"node_id": NODE_ID, "name": "FRRDocker-4"}] ) + result = add_did_you_mean({"error": f"Node ID {NODE_ID_TYPO} doesn't exist"}, {"project_id": PROJECT_ID}, run) assert f"Did you mean node 'FRRDocker-4' ({NODE_ID})" in result["error"] # original message preserved assert NODE_ID_TYPO in result["error"] @@ -144,9 +148,7 @@ def test_no_hint_when_too_far_or_ambiguous(): two = [NODE_ID, "b78a2630-2b9e-4b43-9397-c855c8053d1a"] # both within distance 2 of the typo run2, _calls2 = _run_factory(nodes=[{"node_id": i, "name": f"n{i}"} for i in two]) - result2 = add_did_you_mean( - {"error": f"Node ID {NODE_ID_TYPO} doesn't exist"}, {"project_id": PROJECT_ID}, run2 - ) + result2 = add_did_you_mean({"error": f"Node ID {NODE_ID_TYPO} doesn't exist"}, {"project_id": PROJECT_ID}, run2) assert "Did you mean" not in result2["error"] diff --git a/tests/api/routes/compute/test_capabilities.py b/tests/api/routes/compute/test_capabilities.py index 97d7948c5..e0017cedf 100644 --- a/tests/api/routes/compute/test_capabilities.py +++ b/tests/api/routes/compute/test_capabilities.py @@ -170,7 +170,9 @@ class TestCapabilitiesRoutes: async def fake_bridge_tap_probe(): return True - monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", fake_bridge_tap_probe) + monkeypatch.setattr( + "gns3server.api.routes.compute.capabilities.probe_bridge_tap_support", fake_bridge_tap_probe + ) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_iol_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index f54be3dc9..c50c35c6a 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -23,23 +23,18 @@ fallback and cleanup. import os import uuid -from unittest.mock import MagicMock, call +from unittest.mock import MagicMock, call, patch import pytest import pytest_asyncio -from unittest.mock import patch - -from tests.utils import AsyncioMagicMock - -from gns3server.compute.docker import Docker -from gns3server.compute.docker.docker_vm import DockerVM -from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error from gns3server.compute.compute_error import ComputeError -from gns3server.compute.nios.nio_udp import NIOUDP +from gns3server.compute.docker import Docker +from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error +from gns3server.compute.docker.docker_vm import DockerVM from gns3server.compute.nios.nio_bridge import NIOBridge from gns3server.compute.ubridge.ubridge_error import UbridgeError - +from tests.utils import AsyncioMagicMock BRIDGE = "gns3a1b2c3d4e5f" @@ -1373,7 +1368,13 @@ def test_create_nio_udp_rejects_kernel_only_filters(vm): with pytest.raises(ComputeError, match="kernel-datapath"): vm.manager.create_nio( - {"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1", "filters": {"gemodel": [100, 0, 30]}} + { + "type": "nio_udp", + "lport": 4242, + "rport": 4343, + "rhost": "127.0.0.1", + "filters": {"gemodel": [100, 0, 30]}, + } ) diff --git a/tests/compute/docker/test_iol_docker_kernel_datapath.py b/tests/compute/docker/test_iol_docker_kernel_datapath.py index 8c7a4f7db..b3f692706 100644 --- a/tests/compute/docker/test_iol_docker_kernel_datapath.py +++ b/tests/compute/docker/test_iol_docker_kernel_datapath.py @@ -14,19 +14,15 @@ Image-free: everything is asserted against the uBridge command stream and the anchor naming contract. """ -import uuid +from unittest.mock import MagicMock, patch import pytest import pytest_asyncio -from unittest.mock import MagicMock, patch - -from tests.utils import AsyncioMagicMock - from gns3server.compute.docker import Docker from gns3server.compute.docker.docker_error import DockerError from gns3server.compute.docker.iol_docker_vm import IOLDockerVM - +from tests.utils import AsyncioMagicMock NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" TAP00 = "gx00010203e0p0" # the vm fixture's node id, bay 0 unit 0 @@ -296,7 +292,9 @@ async def test_remove_nio_binding_kernel_releases_the_port_leg_first(vm): commands = _commands(vm) assert f"bridge stop {PORT_BRIDGE}" in commands assert f'bridge delete_nio_tap {PORT_BRIDGE} "{TAP00}"' in commands - assert commands.index(f"bridge stop {PORT_BRIDGE}") < commands.index(f'bridge delete_nio_tap {PORT_BRIDGE} "{TAP00}"') + assert commands.index(f"bridge stop {PORT_BRIDGE}") < commands.index( + f'bridge delete_nio_tap {PORT_BRIDGE} "{TAP00}"' + ) assert f'tc reset "{TAP00}"' in commands assert f'brctl delif "{BRIDGE}" "{TAP00}"' in commands assert f'brctl delete "{BRIDGE}"' in commands diff --git a/tests/compute/docker/test_vendor_docker_vm.py b/tests/compute/docker/test_vendor_docker_vm.py index f508bcf29..b45ba8c0e 100644 --- a/tests/compute/docker/test_vendor_docker_vm.py +++ b/tests/compute/docker/test_vendor_docker_vm.py @@ -908,6 +908,7 @@ async def test_create_reparse_refreshes_env_knobs(compute_project, manager): # Kernel-datapath (NIOBridge) rejection # --------------------------------------------------------------------------- + @pytest.mark.asyncio async def test_unix_socket_nio_rejects_kernel_datapath_link(compute_project, manager): diff --git a/tests/compute/dynamips/test_dynamips_kernel_datapath.py b/tests/compute/dynamips/test_dynamips_kernel_datapath.py index 885780524..d09932eea 100644 --- a/tests/compute/dynamips/test_dynamips_kernel_datapath.py +++ b/tests/compute/dynamips/test_dynamips_kernel_datapath.py @@ -25,27 +25,24 @@ hypervisor's fds released before the TAPs are unpersisted — QEMU's order, since here it is Dynamips that holds them). """ +import contextlib import os import shutil -import contextlib +from unittest.mock import MagicMock, patch import pytest import pytest_asyncio -from unittest.mock import MagicMock, patch - -from tests.utils import AsyncioMagicMock - from gns3server.compute.dynamips import Dynamips -from gns3server.compute.dynamips.nodes.router import Router -from gns3server.compute.dynamips.dynamips_error import DynamipsError from gns3server.compute.dynamips.adapters.gt96100_fe import GT96100_FE from gns3server.compute.dynamips.adapters.pa_4t import PA_4T from gns3server.compute.dynamips.adapters.pa_fe_tx import PA_FE_TX from gns3server.compute.dynamips.adapters.wic_1enet import WIC_1ENET -from gns3server.compute.ubridge.ubridge_error import UbridgeError +from gns3server.compute.dynamips.dynamips_error import DynamipsError +from gns3server.compute.dynamips.nodes.router import Router from gns3server.compute.nios.nio_bridge import NIOBridge - +from gns3server.compute.ubridge.ubridge_error import UbridgeError +from tests.utils import AsyncioMagicMock NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" BRIDGE = "gns3a1b2c3d4e5f" @@ -237,7 +234,7 @@ async def test_prepare_tap_datapath_wires_nios_bound_while_stopped(router): hypervisor = _hypervisor_commands(router) assert any(c.startswith("nio create_tap tap-") and c.endswith(f"{TAP00}") for c in hypervisor) - assert any(c.startswith(f"vm slot_add_nio_binding \"R1\" 0 0 tap-") for c in hypervisor) + assert any(c.startswith('vm slot_add_nio_binding "R1" 0 0 tap-') for c in hypervisor) router._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') assert (0, 0) in router._tap_nios @@ -489,9 +486,7 @@ async def test_stop_ubridge_releases_hypervisor_fds_before_deleting_taps(router) router._ubridge_send.reset_mock() router._hypervisor.commands.clear() - with patch( - "gns3server.compute.base_node.BaseNode._stop_ubridge", new=AsyncioMagicMock() - ) as stop: + with patch("gns3server.compute.base_node.BaseNode._stop_ubridge", new=AsyncioMagicMock()) as stop: await router._stop_ubridge() hypervisor = _hypervisor_commands(router) diff --git a/tests/compute/iou/test_iou_kernel_datapath.py b/tests/compute/iou/test_iou_kernel_datapath.py index 40645b279..427f19502 100644 --- a/tests/compute/iou/test_iou_kernel_datapath.py +++ b/tests/compute/iou/test_iou_kernel_datapath.py @@ -22,19 +22,16 @@ it, kernel-link enslavement, the relay fallback for a uBridge without the command, and the reverse-of-QEMU stop order (uBridge holds the anchor fds). """ +from unittest.mock import MagicMock, patch + import pytest import pytest_asyncio -from unittest.mock import MagicMock, patch - -from tests.utils import AsyncioMagicMock - from gns3server.compute.iou import IOU -from gns3server.compute.iou.iou_vm import IOUVM from gns3server.compute.iou.iou_error import IOUError -from gns3server.compute.nios.nio_bridge import NIOBridge +from gns3server.compute.iou.iou_vm import IOUVM from gns3server.compute.ubridge.ubridge_error import UbridgeError - +from tests.utils import AsyncioMagicMock NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" BRIDGE = "gns3a1b2c3d4e5f" diff --git a/tests/compute/qemu/test_qemu_kernel_datapath.py b/tests/compute/qemu/test_qemu_kernel_datapath.py index ccee4796c..ba6ea96ec 100644 --- a/tests/compute/qemu/test_qemu_kernel_datapath.py +++ b/tests/compute/qemu/test_qemu_kernel_datapath.py @@ -24,21 +24,16 @@ legacy socket-netdev fallback for a uBridge without the tap module. import asyncio import os import stat -import sys +from unittest.mock import MagicMock, patch import pytest import pytest_asyncio -from unittest.mock import MagicMock, patch - -from tests.utils import asyncio_patch, AsyncioMagicMock - from gns3server.compute.qemu import Qemu -from gns3server.compute.qemu.qemu_vm import QemuVM from gns3server.compute.qemu.qemu_error import QemuError -from gns3server.compute.nios.nio_bridge import NIOBridge +from gns3server.compute.qemu.qemu_vm import QemuVM from gns3server.compute.ubridge.ubridge_error import UbridgeError - +from tests.utils import AsyncioMagicMock, asyncio_patch BRIDGE = "gns3a1b2c3d4e5f" TAP0 = "gq00010203e0p0" # the vm fixture's node id, adapter 0 @@ -272,7 +267,7 @@ async def test_update_nio_binding_reapplies_filters_on_the_anchor(vm): vm._ubridge_send.assert_any_call(f'tc reset "{TAP0}"') vm._ubridge_send.assert_any_call(f'tc netem set "{TAP0}" delay 10') - assert not any(f"brctl addif" in str(c) for c in vm._ubridge_send.call_args_list) + assert not any("brctl addif" in str(c) for c in vm._ubridge_send.call_args_list) @pytest.mark.asyncio diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py index 9408c70c0..1cdeba245 100644 --- a/tests/compute/ubridge/test_tc_probe.py +++ b/tests/compute/ubridge/test_tc_probe.py @@ -25,8 +25,8 @@ from gns3server.compute.ubridge import tc_probe from gns3server.compute.ubridge.tc_probe import ( probe_bridge_tap_support, probe_iol_tap_support, - probe_tc_capabilities, probe_tap_support, + probe_tc_capabilities, ) from gns3server.compute.ubridge.ubridge_error import UbridgeError @@ -65,7 +65,7 @@ class FakeHypervisor: if self.error: raise UbridgeError(self.error) if self.error_prefix and command.startswith(self.error_prefix): - raise UbridgeError(f"202-Unknown command") + raise UbridgeError("202-Unknown command") if command.startswith("bridge delete_nio_tap "): if FakeHypervisor.bridge_tap_error: raise UbridgeError(FakeHypervisor.bridge_tap_error) diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index 17b96b4ad..6a8f3b27b 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -381,7 +381,18 @@ async def test_json(compute): "disk_usage_percent": 0, "connected": True, "last_error": None, - "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None, "ubridge_iol_tap": None, "ubridge_bridge_tap": None}, + "capabilities": { + "version": "", + "platform": "", + "cpus": 0, + "memory": 0, + "disk_size": 0, + "node_types": [], + "ubridge_tc": None, + "ubridge_tap": None, + "ubridge_iol_tap": None, + "ubridge_bridge_tap": None, + }, } assert compute.asdict(topology_dump=True) == { "compute_id": "my_compute_id", diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 9c0ababfe..7d326f896 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -20,14 +20,15 @@ Controller-side kernel-datapath link selection: eligibility of Docker-to- Docker links, NIO emission and the filters/markers guards. """ -import pytest from unittest.mock import MagicMock, patch +import pytest + from gns3server.config import Config -from gns3server.controller.udp_link import UDPLink -from gns3server.controller.ports.ethernet_port import EthernetPort -from gns3server.controller.node import Node from gns3server.controller.controller_error import ControllerError +from gns3server.controller.node import Node +from gns3server.controller.ports.ethernet_port import EthernetPort +from gns3server.controller.udp_link import UDPLink def _node(project, compute, name, node_type="docker", status="stopped", environment=None): diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index a7f821565..9ebb0e351 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -903,7 +903,9 @@ class DockerDaemon: ) response = conn.getresponse() payload = response.read() - assert response.status == 200, f"docker cp into {container_id[:12]} -> HTTP {response.status}: {payload[:300]!r}" + assert response.status == 200, ( + f"docker cp into {container_id[:12]} -> HTTP {response.status}: {payload[:300]!r}" + ) except (OSError, http.client.HTTPException) as e: raise AssertionError(f"docker cp into {container_id[:12]} failed: {e}") from None finally: diff --git a/tests/e2e/test_docker_link_local_frames.py b/tests/e2e/test_docker_link_local_frames.py index bd4b2a394..8ac4a5404 100644 --- a/tests/e2e/test_docker_link_local_frames.py +++ b/tests/e2e/test_docker_link_local_frames.py @@ -47,6 +47,7 @@ The negative control for the old behaviour is the uBridge suite's import functools import io import os +import shutil import subprocess import tarfile import tempfile @@ -246,9 +247,12 @@ def _llprobe(): binary = os.path.join(directory, "llprobe") with open(source, "w") as f: f.write(_LLPROBE_C) + cc = shutil.which("cc") + if cc is None: + pytest.skip("cannot build the raw-frame probe (no C compiler found)") try: - subprocess.run(["cc", "-static", "-O2", "-o", binary, source], check=True, capture_output=True) - except (FileNotFoundError, subprocess.CalledProcessError) as e: + subprocess.run([cc, "-static", "-O2", "-o", binary, source], check=True, capture_output=True) + except subprocess.CalledProcessError as e: pytest.skip(f"cannot build the raw-frame probe (cc -static): {e}") return binary @@ -270,9 +274,7 @@ def _upload_probe(daemon, *container_ids): def _recv_counts(daemon, container_id, window, macs): """Run the receiver to completion and parse its "=" lines.""" - code, output = daemon.exec( - container_id, ["/tmp/llprobe", "recv", "eth0", str(window), *macs], timeout=window + 30 - ) + code, output = daemon.exec(container_id, ["/tmp/llprobe", "recv", "eth0", str(window), *macs], timeout=window + 30) assert code == 0, output return {line.split("=")[0]: int(line.split("=")[1]) for line in output.strip().splitlines() if "=" in line} @@ -316,7 +318,7 @@ def test_docker_link_local_frames(): print(".. starting containers") for node in nodes: compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start") - for node, address in zip(nodes, (R1_IP, R2_IP)): + for node, address in zip(nodes, (R1_IP, R2_IP), strict=True): harness.docker_wait_address(daemon, node["container_id"], address) a1 = harness.docker_anchor_name(n1["node_id"], *ETH) From dc5e2d90baeac62159dfcf14bd7400ef31d8c6ed Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 12:53:28 +0800 Subject: [PATCH 079/101] fix(iou): keep BridgeNIO filters across the NIO update route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream's mypy pass narrowed the filter copy to UDPNIO only (the Ethernet/TAP members of the union have no filters field). BridgeNIO — the kernel-datapath NIO — carries filters too, so on an IOU kernel link every PUT of a link's filters cleared them and applied nothing: netem never landed on the anchor (e2e test_iou_kernel_datapath caught it). Narrow to (UDPNIO, BridgeNIO): still avoids attribute access on the models without the field, keeps both filter-carrying types. --- gns3server/api/routes/compute/iou_nodes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gns3server/api/routes/compute/iou_nodes.py b/gns3server/api/routes/compute/iou_nodes.py index 4ae44a215..f0ec12299 100644 --- a/gns3server/api/routes/compute/iou_nodes.py +++ b/gns3server/api/routes/compute/iou_nodes.py @@ -225,7 +225,9 @@ async def update_iou_node_nio( nio = node.get_nio(adapter_number, port_number) nio.filters.clear() - if isinstance(nio_data, schemas.UDPNIO) and nio_data.filters: + # EthernetNIO/TAPNIO carry no filters field; UDPNIO (relay) and + # BridgeNIO (kernel datapath) both do. + if isinstance(nio_data, (schemas.UDPNIO, schemas.BridgeNIO)) and nio_data.filters: nio.filters = nio_data.filters # NIO type is a Union (Ethernet/TAP/UDP/bridge); only UDPNIO and # BridgeNIO carry markers and suspend. Suspend is what the compute turns From 1c85befbab1ba9c559a4a3036505159430826eef Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 13:39:30 +0800 Subject: [PATCH 080/101] test(e2e): take the switch anchors out of NetworkManager's reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The esw kernel fast-path scenario deletes and re-creates a link, which re-enslaves the peer anchor — and on NetworkManager hosts NM's netdev state machine can release an NM-managed TAP from its bridge some 60-70 ms after enslavement, ~50% of re-enslavements, while the tap is in NM's not-yet-assumed state. The freed port stays administratively up and no GNS3 process logs anything: the membership assert sees the join, then the ping measures a silent blackhole (this was the line-277 e2e failure). Root-caused with a server-free harness (one uBridge, one fresh tap, one bridge, an addif/delif loop): the same tap frees 20/40 while NM-managed, 0/40 after `nmcli device set managed no`, 20/40 again when re-managed. Docker's veth host ends are not NM-managed, which is why only TAP anchors ever showed it — including the "anchor lost its bridge membership after a ports_mapping update (TAP peers only)" incident from the switch's development, whose conclusion is corrected in the docs. Unmanage the anchors once the routers have booted, before the scenario re-enslaves them; document the failure signature and the host-side remedy (per-device nmcli call, or NM unmanaged-devices configuration) in the L2-anchor spec and the switch feature doc. --- docs/design/ubridge-l2-anchor-spec.md | 6 +++++ .../ethernet-switch-kernel-datapath.md | 14 +++++++--- tests/e2e/harness.py | 26 +++++++++++++++++++ .../test_ethernet_switch_kernel_datapath.py | 9 +++++++ 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/docs/design/ubridge-l2-anchor-spec.md b/docs/design/ubridge-l2-anchor-spec.md index 168386d4c..2594de893 100644 --- a/docs/design/ubridge-l2-anchor-spec.md +++ b/docs/design/ubridge-l2-anchor-spec.md @@ -86,6 +86,12 @@ acceptance test in §E.2 therefore measures after a settle window. Suppressing it (e.g. `disable_ipv6` on the bridge before enslaving, for the MLD half) would be a new, separate requirement and has not been asked for. +### NetworkManager releases managed TAPs from bridges (measured) + +On hosts where NetworkManager manages the data-plane TAPs — it claims uBridge-created tun/tap devices on many desktops (`nmcli device` shows them as `disconnected` before NM's state machine has assumed them, `connected (externally)` after) — NM can **release a tap from its bridge** some 60-70 ms after enslavement, while the port is in that not-yet-assumed state. Measured with a minimal harness (one uBridge, one fresh tap, one bridge, an `addif`/`delif` loop, no server): ~50 % of re-enslavements lose the port, the freed device stays administratively **up** with the bridge's promiscuity/allmulti cleared, and no GNS3 process sends anything — gns3-server and uBridge log nothing, and the server's attachment bookkeeping stays intact. The *first* enslavement of a newly created tap was stable in every run; the hazard is a **re-enslavement** (link delete + recreate, a port re-join after a peer restart) before NM assumes the device — on the same tap, `nmcli device set managed no` flips the failure rate from 20/40 to 0/40, and re-managing it restores 20/40. + +Consequence for an affected link: the port silently leaves the bridge (frames drop) while the API stays green — the next NIO update re-heals it, because the switch's `_kernel_update` re-checks `brif` membership, but nothing forces one. Docker's veth host ends are not NM-managed and never showed the failure; every TAP anchor family is exposed (Dynamips `gd`, QEMU `gq`, IOU `gi`, IOL `gx`, and switch-absorbed anchors). Host-side remedy: `nmcli device set managed no` (runtime, per device), or NM's `unmanaged-devices` configuration keyed on the anchor name prefixes; the e2e harness takes the anchors it re-enslaves out of NM's hands (`harness.unmanage_from_networkmanager`). + ### Goal Every host-side device uBridge creates for the data plane is **pure L2**: no diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index 322e06d43..eb1ed607a 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -113,10 +113,16 @@ on **absorbed anchors** re-enslaving an interface owned by another node proved to be where links died: live testing showed an anchor occasionally losing its bridge membership right after a `ports_mapping` update (later updates then failed `delif` with EINVAL), reproducibly with qemu, IOU and -Dynamips peers, never with Docker's veths. Replaying the exact command -sequence by hand against uBridge (held TAPs, no server involved) did not -reproduce it, and the in-place reconcile removes the operation entirely — -the failure mode no longer exists by construction. +Dynamips peers, never with Docker's veths. That failure was root-caused +later: it is NetworkManager releasing NM-managed TAPs from their bridge +tens of milliseconds after (re)enslavement — veths are not NM-managed, +which is exactly why only TAP peers showed it (the mechanics and the +`nmcli device set managed no` remedy are in the L2-anchor spec). The +in-place reconcile removes this particular re-enslavement, but link delete ++ recreate and peer restarts still re-enslave the anchor, so the hazard +remains on NM hosts wherever NM manages the taps; the kernel e2e scenarios +take the anchors out of NM's hands +(`harness.unmanage_from_networkmanager`) before they exercise those paths. ## Link-local frames (LACP, LLDP, 802.1X, STP) diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 9ebb0e351..852367709 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -1053,6 +1053,32 @@ def bridge_members(bridge): return sorted(os.listdir(f"/sys/class/net/{bridge}/brif")) +def unmanage_from_networkmanager(*names): + """Best-effort ``nmcli device set managed no`` for the given + interfaces. + + NetworkManager (desktop hosts) manages the TAP devices uBridge creates. + While a tap is in NM's not-yet-assumed state, NM's netdev-event state + machine can *release* it from its bridge some 60-70 ms after + enslavement (measured: ~50% of re-enslavements on an NM host): the port + is left administratively up, out of the bridge, with no server-side log + or error — a silent blackhole. Scenarios that detach and re-enslave an + anchor (link delete + recreate, node restart) must call this once their + nodes have booted, or they are nondeterministically flaky on such + hosts. Silently a no-op where nmcli is unavailable. + """ + nmcli = shutil.which("nmcli") + if not nmcli: + return + for name in names: + # S603: resolved binary, fixed argv, no shell + subprocess.run( # noqa: S603 + [nmcli, "device", "set", name, "managed", "no"], + capture_output=True, + check=False, + ) + + # --------------------------------------------------------------------------- # L2-anchor spec assertions (docs/design/ubridge-l2-anchor-spec.md §E) # --------------------------------------------------------------------------- diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index 6dc85fe2c..7cd91f48f 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -146,6 +146,15 @@ def test_ethernet_switch_kernel_fast_path(): c1 = _boot(server, pid, r1_id) c2 = _boot(server, pid, r2_id) + # Keep NetworkManager's hands off the anchors: on NM hosts its + # netdev state machine can release a freshly re-enslaved tap from + # the bridge ~60 ms after the join (silently — the port stays up, + # the server keeps believing it is attached), which the link + # delete/recreate below triggers nondeterministically. The first + # join at node start is unaffected; this covers everything after + # it. + harness.unmanage_from_networkmanager(a1, a2) + # Deferred joins completed by the node-start re-push: the anchors # ARE the switch ports now. assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) == sorted([a1, a2]), timeout=15), ( From 3018d3aecce987582746b156f3b191b2e2ee17ea Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 22:15:55 +0800 Subject: [PATCH 081/101] docs(bugs): record the NetworkManager TAP-release root cause, repro and fix A kernel-datapath link could die silently after a delete/recreate: on NetworkManager hosts, NM's state machine releases an NM-managed TAP from its bridge 58-87 ms after (re)enslavement (~50% while the tap has not been assumed yet; the port stays up, no GNS3 process logs anything). Root-caused with a server-free minimal harness (one uBridge, one tap, one bridge, an addif/delif loop) and a single-variable NM A/B; the NetworkManager side is explained by three of its commits (assumption recheck on external enslavement, master removal during activation, external changes not touching assumed slaves). Records: forensic signatures (ifindex unchanged, flags 0x300 cleared, heal-on-next-update), the reproducer, the verified host fix (unmanaged-devices conf.d snippet: 19-20/40 -> 0/40, and the e2e scenario green with test-side workarounds disabled), scope and history (2016 issue #440 veth incident; the 2026-10-01 ethernet-switch incident previously misattributed). Cross-linked from the L2-anchor spec and the switch feature doc. --- docs/bugs/networkmanager-tap-release.md | 90 +++++++++++++++++++ docs/design/ubridge-l2-anchor-spec.md | 2 +- .../ethernet-switch-kernel-datapath.md | 3 + 3 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 docs/bugs/networkmanager-tap-release.md diff --git a/docs/bugs/networkmanager-tap-release.md b/docs/bugs/networkmanager-tap-release.md new file mode 100644 index 000000000..cf845d72c --- /dev/null +++ b/docs/bugs/networkmanager-tap-release.md @@ -0,0 +1,90 @@ + + +> This documentation is organized by AI with reference to actual code. AI can make mistakes — please verify against the source code when in doubt. + +# NetworkManager Releases Enslaved TAP Anchors from Their Bridge (Silent Link Death) + +## Bug Report + +**Date**: 2026-10-05 (root-caused; the same signature first hit on 2026-10-01 during the Ethernet-switch kernel-datapath development) +**Severity**: High on affected hosts (one link dies silently — API and UI stay green; ~50 % per re-enslavement while NetworkManager has not yet assumed the TAP; environment-dependent and intermittent) +**Status**: **Root-caused; host-side fix verified.** The NetworkManager declaration (`unmanaged-devices`) takes the reproducing loop from 19-20/40 to 0/40 and lets the e2e scenario pass with all test-side workarounds disabled. Test-side best-effort hardening is shipped (`harness.unmanage_from_networkmanager`); a product-side guard remains an open decision. +**Component**: Host environment — NetworkManager × uBridge-created TAP anchors. No GNS3 code path performs the detaching operation: gns3-server and uBridge log nothing and their bookkeeping stays intact. + +## Symptoms + +On a kernel-datapath link, after a **re-enslavement of the TAP anchor** — a link delete + recreate, a port re-join after a peer restart — the link may die: ping 0 %, routing protocols never form — while every API/UI surface reports health (`kernel_datapath: true`, no error, no log). The node, the emulated router and the anchor device itself keep running; only the bridge membership is gone. + +Self-heal exists but nothing forces it: any subsequent NIO update re-checks `/sys/class/net//brif/` and re-attaches (`EthernetSwitch.update_nio`'s "not joined" branch, and the equivalent attach path per node type), which is why the state heals on the next filter change, suspend toggle or peer restart. The *first* enslavement of a freshly created TAP is stable in every observation; the hazard is re-enslavement before NetworkManager has assumed the device (a window of minutes), at ~50 % per attempt. + +## Forensic signatures (what the corpse looks like) + +- `/sys/class/net//brif/` missing, `/sys/class/net//brport/` gone — the port was released, not the device destroyed: **ifindex unchanged**, `ip -d link show` reports no master. +- Flags drop `IFF_PROMISC|IFF_ALLMULTI` (0x300, set by the kernel bridge on enslavement): as seen with a 50 Hz brif sampler, `fl 0x1303 → 0x1003` while administrative UP is kept. +- The release lands **58-87 ms after the join** — after the attach API has already returned success, so `wait_until`-style checks see the join and the next traffic sample sees the blackhole. +- No `brctl delif` (or any netlink write) from any GNS3 process; server-side registrations (`_kernel_ports`, NIO state) are untouched; uBridge logs stay clean. +- `nmcli device status` shows the anchor as managed by NetworkManager: `disconnected` before it assumes the device, `connected (externally)` after. + +## Root Cause (NetworkManager side, source-level) + +On hosts where NetworkManager manages the anchors — it claims uBridge-created tun/tap devices by default (`nmcli device` then shows them managed) — NM's device state machine resolves the fresh enslavement in its own favour while the device is in its pre-assumption window: + +1. An **external enslavement** triggers an assumption recheck — NetworkManager commit `a6ceb382e9` ("device: connect slave assumption recheck on external enslavement") exists precisely for `ip link set master ` performed by someone else. +2. The recheck activates (assumes) the device, and activation enforces that **a connection which does not declare a master must not keep its kernel master**: the slave is force-released, reason `NM_DEVICE_STATE_REASON_CONNECTION_ASSUMED` — NetworkManager commit `ec12912908` ("device: enforce the absence of a master during activation"). This is the detaching operation; it runs several asynchronous state-machine steps after the enslavement, which is the 60-80 ms latency. +3. Once the device is assumed (`connected (externally)`), later external enslavements take the opposite path: **external changes must not touch the device** — NetworkManager commit `3127fb0d17` ("device: don't let external changes cause a release of the slave", `configure = FALSE`, "let the device be"). The model is updated; the kernel is left alone. This is why assumed taps were stable across 14 re-enslavements in the field, and why the hazard window ends at assumption. + +Both processes' worlds are mutually exclusive here: NM requires "no master during activation", the datapath requires "enslaved to the switch/per-link bridge" — there is no configuration in which both hold. The sanctioned resolution is a jurisdiction declaration (mark the devices unmanaged), not a race. + +Docker veth host ends are not NM-managed (and the veth naming/type heuristics historically keep NM away — see History), so only the TAP anchor families are affected: `gd` (Dynamips), `gq` (QEMU), `gi` (IOU), `gx` (IOL runner containers), and switch-absorbed anchors. + +## Reproduction (minimal, gns3-server not involved) + +Self-contained harness: one uBridge (`ubridge -U `), one fresh TAP (`tap create`), one kernel bridge (`brctl create` + `vlanfiltering on`), then loop { `link set up`, `brctl addif
`, 200 ms of ~1 kHz membership sampling, `brctl delif`, `link set down` }. Measured on an affected host (same code, five baseline runs, then single-variable A/B on one tap): + +| Condition | Releases / 40 re-enslavements | +|---|---| +| NM-managed (default) | 19, 20, 19, 20, 20 | +| `nmcli device set managed no` | **0** | +| re-managed (`nmcli device set managed yes`) | 20 | + +e2e-level reproduction: `tests/e2e/test_ethernet_switch_kernel_datapath.py::test_ethernet_switch_kernel_fast_path` fails at its link delete/recreate step (`wait_ping` after the anchor re-join) roughly half of fresh runs on an NM host — it failed twice consecutively on 2026-10-05 during the 3.1 rebase validation before the fix. + +## Fix + +Three layers, of which the first is the verified root fix: + +1. **Host declaration** — `/etc/NetworkManager/conf.d/99-gns3-datapath.conf`: + + ```ini + [keyfile] + unmanaged-devices=interface-name:gd*;interface-name:gq*;interface-name:gi*;interface-name:gx*;interface-name:gv*;interface-name:gns3* + ``` + + then `systemctl reload NetworkManager`. Devices matching are unmanaged from birth (also future ones and after re-creations); hosts without NM treat the file as inert. Verified on the affected host: minimal loop **0/40**, and the e2e scenario passes with the test-side workaround disabled (pure host config). This belongs in installers/documentation, not in the server (no root; and the server must not reconfigure the host's NM). + +2. **Test-side best effort** — `harness.unmanage_from_networkmanager(*taps)` runs `nmcli device set managed no` where nmcli exists (no-op otherwise); kernel-datapath scenarios that re-enslave anchors call it after their nodes have booted. This keeps the e2e deterministic on un-declared hosts. + +3. **Product-side guard** — open decision: a bounded post-attach re-check (~250 ms) that re-attaches once and warns (naming NetworkManager), a warn-only variant, or nothing (documentation only). Until decided, an undeclared host relies on the existing heal-on-next-update path. + +## Scope + +- Affected: Linux hosts where NetworkManager manages the TAPs — desktop-native `gns3server` deployments (the most common GNS3 form factor for Linux users). Every TAP anchor family, the switch fast path and per-link kernel links alike. +- Not affected: the GNS3 VM (its Ubuntu-server stack is netplan/systemd-networkd, no NM — verified against the gns3-vm repository), hosts without NM, Docker veth anchors, relay links, cross-compute links (relay by default). +- Blast radius of one occurrence: exactly one link, silently dead; no incorrect data, but it can mislead an experiment's conclusions (e.g. "this protocol never converges here") and it mimics the project's own "API green, data plane dead" bug class — during the rebase validation it was triaged as a regression until the timing signature (join, then +60 ms release with no command) identified it as environmental. + +## History + +- **2026-10-01**: the same signature first hit during the Ethernet-switch development, described then as "an anchor occasionally losing its bridge membership right after a `ports_mapping` update"; reproduced with QEMU, IOU and Dynamips peers (all TAPs) and never with Docker veths; the then-unknown cause was worked around by making VLAN reconfiguration in-place (no re-enslavement) and the failure mode was assumed gone "by construction". See `docs/features/ethernet-switch-kernel-datapath.md`. +- **2016**: the same NM-vs-GNS3 jurisdiction class hit Docker veths — NM configured them like plain NICs and a container DHCP server hijacked the host's default route (issue #440, closed). Upstream worked around it by renaming `gns3-veth…` to `veth-gns3-…`, the commit message reading "the name create different behavior with network manager" (`59c1e125d`). NM's management heuristics even depend on interface names — the rename moved the veths into NM's ignored namespace. +- **2026-10-05**: root-caused with the minimal harness and the single-variable A/B; host fix applied and verified; test-side hardening shipped; full write-up (this document). + +## Related + +- NetworkManager commits: [`a6ceb382e9`](https://codetest.hyprland.org/fdo-mirrors/NetworkManager/commit/a6ceb382e90a9ebc278e380a3b204ad8c2c543c4), [`ec12912908`](http://git.baserock.org/cgit/delta/NetworkManager.git/commit/src/devices/nm-device.c?id=ec12912908d739459ff3913917e9937500345c2a), [`3127fb0d17`](https://codetest.hyprland.org/fdo-mirrors/NetworkManager/commit/3127fb0d17bff0b250218c7bf82b4335b5290825). +- GNS3 issue [#440](https://github.com/GNS3/gns3-server/issues/440) and commit `59c1e125d` (veth rename). +- `docs/design/ubridge-l2-anchor-spec.md` — "NetworkManager releases managed TAPs from bridges" subsection (short form). +- `docs/features/ethernet-switch-kernel-datapath.md` — the corrected account of the 2026-10-01 incident. +- `tests/e2e/harness.py` — `unmanage_from_networkmanager`. diff --git a/docs/design/ubridge-l2-anchor-spec.md b/docs/design/ubridge-l2-anchor-spec.md index 2594de893..2cc566053 100644 --- a/docs/design/ubridge-l2-anchor-spec.md +++ b/docs/design/ubridge-l2-anchor-spec.md @@ -90,7 +90,7 @@ be a new, separate requirement and has not been asked for. On hosts where NetworkManager manages the data-plane TAPs — it claims uBridge-created tun/tap devices on many desktops (`nmcli device` shows them as `disconnected` before NM's state machine has assumed them, `connected (externally)` after) — NM can **release a tap from its bridge** some 60-70 ms after enslavement, while the port is in that not-yet-assumed state. Measured with a minimal harness (one uBridge, one fresh tap, one bridge, an `addif`/`delif` loop, no server): ~50 % of re-enslavements lose the port, the freed device stays administratively **up** with the bridge's promiscuity/allmulti cleared, and no GNS3 process sends anything — gns3-server and uBridge log nothing, and the server's attachment bookkeeping stays intact. The *first* enslavement of a newly created tap was stable in every run; the hazard is a **re-enslavement** (link delete + recreate, a port re-join after a peer restart) before NM assumes the device — on the same tap, `nmcli device set managed no` flips the failure rate from 20/40 to 0/40, and re-managing it restores 20/40. -Consequence for an affected link: the port silently leaves the bridge (frames drop) while the API stays green — the next NIO update re-heals it, because the switch's `_kernel_update` re-checks `brif` membership, but nothing forces one. Docker's veth host ends are not NM-managed and never showed the failure; every TAP anchor family is exposed (Dynamips `gd`, QEMU `gq`, IOU `gi`, IOL `gx`, and switch-absorbed anchors). Host-side remedy: `nmcli device set managed no` (runtime, per device), or NM's `unmanaged-devices` configuration keyed on the anchor name prefixes; the e2e harness takes the anchors it re-enslaves out of NM's hands (`harness.unmanage_from_networkmanager`). +Consequence for an affected link: the port silently leaves the bridge (frames drop) while the API stays green — the next NIO update re-heals it, because the switch's `_kernel_update` re-checks `brif` membership, but nothing forces one. Docker's veth host ends are not NM-managed and never showed the failure; every TAP anchor family is exposed (Dynamips `gd`, QEMU `gq`, IOU `gi`, IOL `gx`, and switch-absorbed anchors). Host-side remedy: `nmcli device set managed no` (runtime, per device), or NM's `unmanaged-devices` configuration keyed on the anchor name prefixes; the e2e harness takes the anchors it re-enslaves out of NM's hands (`harness.unmanage_from_networkmanager`). Full root-cause record with the minimal reproducer, the NetworkManager source-level mechanism and the verified fix: `docs/bugs/networkmanager-tap-release.md`. ### Goal diff --git a/docs/features/ethernet-switch-kernel-datapath.md b/docs/features/ethernet-switch-kernel-datapath.md index eb1ed607a..d6bbbc76c 100644 --- a/docs/features/ethernet-switch-kernel-datapath.md +++ b/docs/features/ethernet-switch-kernel-datapath.md @@ -123,6 +123,9 @@ in-place reconcile removes this particular re-enslavement, but link delete remains on NM hosts wherever NM manages the taps; the kernel e2e scenarios take the anchors out of NM's hands (`harness.unmanage_from_networkmanager`) before they exercise those paths. +The full record — forensic signatures, the minimal server-free reproducer, +the NetworkManager source-level mechanism and the verified host fix — is in +`docs/bugs/networkmanager-tap-release.md`. ## Link-local frames (LACP, LLDP, 802.1X, STP) From 6c73bb15f052b8b65ef12f7f882aaacb304afb68 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 22:19:17 +0800 Subject: [PATCH 082/101] docs(bugs): embed the runnable TAP-release reproducer and the measurement method The reproduction section described the harness only in prose; the actual driver scripts were throwaway lab files. Embed the distilled, runnable reproducer (drives uBridge's control socket through the repo's own client, kept tap for the single-variable A/B), the exact measurement semantics (40 re-enslavements, ~1 kHz / 200 ms membership sampling, a release = the brif entry vanishing, latency print) and the flip protocol, so a reader can reproduce the 19-20/40 vs 0/40 numbers on an affected host. Also point the host-fix layer at the reproducer for re-verification. --- docs/bugs/networkmanager-tap-release.md | 108 +++++++++++++++++++++++- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/docs/bugs/networkmanager-tap-release.md b/docs/bugs/networkmanager-tap-release.md index cf845d72c..473e1160f 100644 --- a/docs/bugs/networkmanager-tap-release.md +++ b/docs/bugs/networkmanager-tap-release.md @@ -42,13 +42,113 @@ Docker veth host ends are not NM-managed (and the veth naming/type heuristics hi ## Reproduction (minimal, gns3-server not involved) -Self-contained harness: one uBridge (`ubridge -U `), one fresh TAP (`tap create`), one kernel bridge (`brctl create` + `vlanfiltering on`), then loop { `link set up`, `brctl addif
`, 200 ms of ~1 kHz membership sampling, `brctl delif`, `link set down` }. Measured on an affected host (same code, five baseline runs, then single-variable A/B on one tap): +A self-contained reproducer — one uBridge, one TAP, one kernel bridge, an enslavement loop — needs none of the server. It drives uBridge's control socket through the repository's own client, so no build step and no extra dependencies beyond the venv: + +```python +#!/usr/bin/env python3 +"""Minimal reproducer: NetworkManager releases an enslaved TAP from its bridge. + +Run from the gns3-server repo root on an affected host (Linux + NetworkManager +managing the TAPs; uBridge installed with cap_net_admin): + + venv/bin/python nm_tap_release_repro.py # 40 re-enslavements + nmcli device set gdlab-nm managed no # the A/B flip (same tap!) + venv/bin/python nm_tap_release_repro.py # -> 0 releases + +The tap and bridge are kept between runs (same device = same NM state). +Clean up afterwards: + + venv/bin/python nm_tap_release_repro.py --cleanup +""" +import asyncio +import os +import shutil +import subprocess +import sys +import time + +sys.path.insert(0, os.getcwd()) # run from the repo root for gns3server imports +from gns3server.compute.ubridge.ubridge_hypervisor import UBridgeHypervisor + +TAP = "gdlab-nm" +BRIDGE = "gns3lab-nm" +SOCK = "/tmp/nm_tap_release_repro.sock" +ITERATIONS = 40 +SAMPLE_MS = 200 # membership sampled at ~1 kHz for this long after each addif + + +def in_bridge(): + return os.path.exists(f"/sys/class/net/{BRIDGE}/brif/{TAP}") + + +async def main(): + ubridge = shutil.which("ubridge") + if not ubridge: + sys.exit("ubridge not found in PATH") + if os.path.exists(SOCK): + os.unlink(SOCK) + proc = subprocess.Popen( + [ubridge, "-U", SOCK], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd="/tmp" + ) + try: + h = UBridgeHypervisor(socket_path=SOCK) + await h.connect() + + async def send(cmd): + try: + await h.send(cmd) + except Exception: + pass # ENOENT during setup/teardown is expected + + if "--cleanup" in sys.argv: + await send(f'tap delete "{TAP}"') + await send(f'brctl delete "{BRIDGE}"') + return + await send(f'tap create "{TAP}"') # no-op if the tap already exists + await send(f'brctl create "{BRIDGE}"') + await send(f'link set "{BRIDGE}" up') + await send(f'brctl vlanfiltering "{BRIDGE}" on') + + releases, latencies = 0, [] + for _ in range(ITERATIONS): + await send(f'link set "{TAP}" up') + await send(f'brctl addif "{BRIDGE}" "{TAP}"') + t0 = time.time() + for _ in range(SAMPLE_MS): + await asyncio.sleep(0.001) + if not in_bridge(): + releases += 1 + latencies.append(round((time.time() - t0) * 1000, 1)) + break + await send(f'brctl delif "{BRIDGE}" "{TAP}"') + await send(f'link set "{TAP}" down') + await asyncio.sleep(0.005) + + print(f"releases: {releases}/{ITERATIONS} re-enslavements", flush=True) + if latencies: + print(f"release latency after join (ms): {latencies[:10]}") + print(f"expect ~19-20/40 while NetworkManager manages {TAP}, " + f"0/40 after `nmcli device set {TAP} managed no`") + finally: + proc.kill() + + +asyncio.run(main()) +``` + +How the numbers are produced: + +- 40 iterations; each is one **re-enslavement**: `link set up`, `brctl addif`, then membership sampled at ~1 kHz for 200 ms, then `brctl delif` + `link set down`. +- A **release** is counted when `/sys/class/net//brif/` disappears inside that 200 ms window; the script prints the release latency after the join (the NM release lands at 58-87 ms). +- The tap is kept between runs, so the A/B flip is single-variable on the *same* device; a fresh device would reset NM's assumption state (the first enslavement of a fresh tap is always stable). + +Measured on an affected host (same code, five baseline runs, then the A/B between runs): | Condition | Releases / 40 re-enslavements | |---|---| | NM-managed (default) | 19, 20, 19, 20, 20 | -| `nmcli device set managed no` | **0** | -| re-managed (`nmcli device set managed yes`) | 20 | +| `nmcli device set managed no`, same tap | **0** | +| re-managed (`nmcli device set managed yes`), same tap | 20 | e2e-level reproduction: `tests/e2e/test_ethernet_switch_kernel_datapath.py::test_ethernet_switch_kernel_fast_path` fails at its link delete/recreate step (`wait_ping` after the anchor re-join) roughly half of fresh runs on an NM host — it failed twice consecutively on 2026-10-05 during the 3.1 rebase validation before the fix. @@ -63,7 +163,7 @@ Three layers, of which the first is the verified root fix: unmanaged-devices=interface-name:gd*;interface-name:gq*;interface-name:gi*;interface-name:gx*;interface-name:gv*;interface-name:gns3* ``` - then `systemctl reload NetworkManager`. Devices matching are unmanaged from birth (also future ones and after re-creations); hosts without NM treat the file as inert. Verified on the affected host: minimal loop **0/40**, and the e2e scenario passes with the test-side workaround disabled (pure host config). This belongs in installers/documentation, not in the server (no root; and the server must not reconfigure the host's NM). + then `systemctl reload NetworkManager`. Devices matching are unmanaged from birth (also future ones and after re-creations); hosts without NM treat the file as inert. Verified on the affected host: minimal loop **0/40**, and the e2e scenario passes with the test-side workaround disabled (pure host config); re-verify with the reproducer below — same tap, expect 0/40 after the reload. This belongs in installers/documentation, not in the server (no root; and the server must not reconfigure the host's NM). 2. **Test-side best effort** — `harness.unmanage_from_networkmanager(*taps)` runs `nmcli device set managed no` where nmcli exists (no-op otherwise); kernel-datapath scenarios that re-enslave anchors call it after their nodes have booted. This keeps the e2e deterministic on un-declared hosts. From 20b7968e33ae0a667dc9018c3343ce670ca6e8fa Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 22:30:29 +0800 Subject: [PATCH 083/101] docs(bugs): note how to re-observe the baseline on an already-declared host --- docs/bugs/networkmanager-tap-release.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/bugs/networkmanager-tap-release.md b/docs/bugs/networkmanager-tap-release.md index 473e1160f..885821ce3 100644 --- a/docs/bugs/networkmanager-tap-release.md +++ b/docs/bugs/networkmanager-tap-release.md @@ -141,6 +141,7 @@ How the numbers are produced: - 40 iterations; each is one **re-enslavement**: `link set up`, `brctl addif`, then membership sampled at ~1 kHz for 200 ms, then `brctl delif` + `link set down`. - A **release** is counted when `/sys/class/net//brif/` disappears inside that 200 ms window; the script prints the release latency after the join (the NM release lands at 58-87 ms). - The tap is kept between runs, so the A/B flip is single-variable on the *same* device; a fresh device would reset NM's assumption state (the first enslavement of a fresh tap is always stable). +- The numbers below were measured on a host **without** the declaration from *Fix*. On a host that already carries it the tap is unmanaged from birth and the loop prints `0/40` by design; to re-observe the baseline there, temporarily remove the conf.d file (`sudo rm /etc/NetworkManager/conf.d/99-gns3-datapath.conf && sudo systemctl reload NetworkManager`) or run on an un-declared host. Measured on an affected host (same code, five baseline runs, then the A/B between runs): From 28e9b82a5f88ceba3131378666c40e43c4449a42 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 22:32:48 +0800 Subject: [PATCH 084/101] docs(bugs): managed-yes cannot re-arm the baseline once the conf matches --- docs/bugs/networkmanager-tap-release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/bugs/networkmanager-tap-release.md b/docs/bugs/networkmanager-tap-release.md index 885821ce3..e5c149b17 100644 --- a/docs/bugs/networkmanager-tap-release.md +++ b/docs/bugs/networkmanager-tap-release.md @@ -141,7 +141,7 @@ How the numbers are produced: - 40 iterations; each is one **re-enslavement**: `link set up`, `brctl addif`, then membership sampled at ~1 kHz for 200 ms, then `brctl delif` + `link set down`. - A **release** is counted when `/sys/class/net//brif/` disappears inside that 200 ms window; the script prints the release latency after the join (the NM release lands at 58-87 ms). - The tap is kept between runs, so the A/B flip is single-variable on the *same* device; a fresh device would reset NM's assumption state (the first enslavement of a fresh tap is always stable). -- The numbers below were measured on a host **without** the declaration from *Fix*. On a host that already carries it the tap is unmanaged from birth and the loop prints `0/40` by design; to re-observe the baseline there, temporarily remove the conf.d file (`sudo rm /etc/NetworkManager/conf.d/99-gns3-datapath.conf && sudo systemctl reload NetworkManager`) or run on an un-declared host. +- The numbers below were measured on a host **without** the declaration from *Fix*. On a host that already carries it the tap is unmanaged from birth and the loop prints `0/40` by design; to re-observe the baseline there, temporarily remove the conf.d file (`sudo rm /etc/NetworkManager/conf.d/99-gns3-datapath.conf && sudo systemctl reload NetworkManager`) or run on an un-declared host. A runtime `nmcli device set managed yes` is *not* enough — the configuration match wins (the device stays `unmanaged` and the loop still prints 0/40; measured). Measured on an affected host (same code, five baseline runs, then the A/B between runs): From 15fbef93fb3d84dcca2638e386e5413422575306 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 22:51:17 +0800 Subject: [PATCH 085/101] test(e2e): unmanage the TAP anchors in the remaining kernel scenarios Same guard the ethernet-switch scenario already carries: NetworkManager can release an NM-managed TAP from its bridge shortly after (re-)enslavement (docs/bugs/networkmanager-tap-release.md). Applied to the Dynamips, QEMU, IOU, IOL-runner and switch-cascade scenarios after their nodes have booted (no-op where nmcli/NetworkManager is absent). This keeps those tests deterministic on un-declared hosts. --- tests/e2e/test_dynamips_kernel_datapath.py | 5 +++++ tests/e2e/test_ethernet_switch_cascade.py | 5 +++++ tests/e2e/test_iol_docker_kernel_datapath.py | 5 +++++ tests/e2e/test_iou_kernel_datapath.py | 5 +++++ tests/e2e/test_qemu_kernel_datapath.py | 6 ++++++ 5 files changed, 26 insertions(+) diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py index 64e61e331..c4fc774c6 100644 --- a/tests/e2e/test_dynamips_kernel_datapath.py +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -111,6 +111,11 @@ def test_dynamips_kernel_datapath(): c1 = _boot(server, pid, r1_id) c2 = _boot(server, pid, r2_id) + # NetworkManager can release an NM-managed TAP from its bridge after + # (re-)enslavement (docs/bugs/networkmanager-tap-release.md); take the + # anchors out of its reach for the rest of the run. + harness.unmanage_from_networkmanager(t1, t2) + # Kernel objects: anchors exist and the per-link bridge has exactly # the two of them enslaved. assert harness.tap_exists(t1) and harness.tap_exists(t2), (t1, t2) diff --git a/tests/e2e/test_ethernet_switch_cascade.py b/tests/e2e/test_ethernet_switch_cascade.py index 6e082d175..1bc02c5b8 100644 --- a/tests/e2e/test_ethernet_switch_cascade.py +++ b/tests/e2e/test_ethernet_switch_cascade.py @@ -188,6 +188,11 @@ def test_ethernet_switch_cascade_kernel_datapath(): c1 = _boot(server, pid, r1_id) c2 = _boot(server, pid, r2_id) + # NetworkManager can release an NM-managed TAP from its bridge after + # (re-)enslavement (docs/bugs/networkmanager-tap-release.md); take the + # anchors out of its reach for the rest of the run. + harness.unmanage_from_networkmanager(a1_10, a1_20, a2_10, a2_20) + # The deferred host joins completed via the node-start re-push: each # switch bridges its two router anchors plus its cascade end. assert harness.wait_until(lambda: harness.bridge_members(br1) == sorted([a1_10, a1_20, e0]), timeout=15), ( diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py index addc125e6..a8fa56a63 100644 --- a/tests/e2e/test_iol_docker_kernel_datapath.py +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -121,6 +121,11 @@ def test_iol_docker_kernel_datapath(): c1 = _boot(server, pid, n1_id) c2 = _boot(server, pid, n2_id) + # NetworkManager can release an NM-managed TAP from its bridge after + # (re-)enslavement (docs/bugs/networkmanager-tap-release.md); take the + # anchors out of its reach for the rest of the run. + harness.unmanage_from_networkmanager(a1, a2) + # Kernel objects: anchors exist (born with the node, before the link # was ever wired) and the per-link bridge has exactly the two of them. assert harness.tap_exists(a1) and harness.tap_exists(a2), (a1, a2) diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index 23f2f1c18..a857bcd40 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -135,6 +135,11 @@ def test_iou_kernel_datapath(): c1 = _boot(server, pid, n1_id) c2 = _boot(server, pid, n2_id) + # NetworkManager can release an NM-managed TAP from its bridge after + # (re-)enslavement (docs/bugs/networkmanager-tap-release.md); take the + # anchors out of its reach for the rest of the run. + harness.unmanage_from_networkmanager(a1, a2) + # Kernel objects: the anchors are one persistent TAP per Ethernet # bay/unit, born with the node (all eight — the IOU port model), # and the per-link bridge has exactly the two addressed ports. diff --git a/tests/e2e/test_qemu_kernel_datapath.py b/tests/e2e/test_qemu_kernel_datapath.py index 721895c53..419d2c4ac 100644 --- a/tests/e2e/test_qemu_kernel_datapath.py +++ b/tests/e2e/test_qemu_kernel_datapath.py @@ -127,6 +127,12 @@ def test_qemu_kernel_datapath(): c1 = _boot(server, pid, n1_id) c2 = _boot(server, pid, n2_id) + # NetworkManager can release an NM-managed TAP from its bridge after + # (re-)enslavement (docs/bugs/networkmanager-tap-release.md); take the + # anchors out of its reach for the rest of the run (the restart below + # re-attaches them under fresh devices). + harness.unmanage_from_networkmanager(a1, a2, harness.qemu_anchor_name(n1_id, 1)) + # Kernel objects: one persistent TAP per adapter (both adapters of # router 1, the addressed one of router 2), and the per-link bridge # has exactly the two addressed ports. From f21d7be99c5d3562708c24b6a326653867d0198d Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 22:51:17 +0800 Subject: [PATCH 086/101] test(e2e): derive busybox ping loss from replies when the summary is lost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exec stream can truncate after busybox's 4294967 ms RTT timer-wrap artifact, dropping the summary line — docker_ping then reported a phantom 100 % loss and made the corrupt-filter scenario fail spuriously. Factor the parsing into _busybox_ping_parse: the summary stays authoritative when present; otherwise the loss is derived from the seq= replies actually seen (no replies at all still reports 100 %). --- tests/e2e/harness.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 852367709..3b8bbda1b 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -983,6 +983,25 @@ def docker_wait_address(daemon, container_id, address, timeout=30): _BUSYBOX_PING_LOSS_RE = re.compile(r"(\d+)% packet loss") _BUSYBOX_PING_RTT_RE = re.compile(r"round-trip min/avg/max = [\d.]+/([\d.]+)/[\d.]+ ms") +_BUSYBOX_PING_REPLY_RE = re.compile(r"\bseq=\d+") + + +def _busybox_ping_parse(output, count): + """(loss percent, avg rtt or None) from busybox ping output. + + The summary line can be lost when the exec stream truncates (observed + after busybox's 4294967 ms RTT timer-wrap artifact truncates the + tail): with replies visible but no summary, derive the loss from the + replies actually seen instead of reporting a phantom 100 %. + """ + loss = _BUSYBOX_PING_LOSS_RE.search(output) + if loss is not None: + loss_pct = int(loss.group(1)) + else: + replies = len(_BUSYBOX_PING_REPLY_RE.findall(output)) + loss_pct = round(100 * max(0, count - replies) / count) if replies else 100 + rtt = _BUSYBOX_PING_RTT_RE.search(output) + return loss_pct, (float(rtt.group(1)) if rtt else None) def docker_ping(daemon, container_id, target, count=3, timeout=1, interval=0.2, size=None): @@ -1003,11 +1022,10 @@ def docker_ping(daemon, container_id, target, count=3, timeout=1, interval=0.2, cmd += ["-s", str(size)] cmd.append(target) code, output = daemon.exec(container_id, cmd, timeout=cap + 10) - loss = _BUSYBOX_PING_LOSS_RE.search(output) - rtt = _BUSYBOX_PING_RTT_RE.search(output) + loss_pct, avg_rtt = _busybox_ping_parse(output, count) return { - "loss": int(loss.group(1)) if loss else 100, - "avg": float(rtt.group(1)) if rtt else None, + "loss": loss_pct, + "avg": avg_rtt, "raw": output, "exit": code, } From f688e654e1e3908cf4e35096eb8092720ebba6ae Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 5 Oct 2026 23:41:10 +0800 Subject: [PATCH 087/101] test(e2e): widen the fast-path RTT bound to 100 ms via harness.FAST_RTT_MS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'impairment cleared' asserts used avg < 50 on three-packet IOS pings; the unimpaired baseline is 20-45 ms with 4 ms quantization, and host scheduling jitter pushed averages to 54/56 in two full-suite runs (esw, dynamips) — a margin, not an impairment (the qdisc asserts pass, success is 100 %). The contrasted delayed regime is >= 150 ms, so a 100 ms bound still separates the two cleanly. Docker's sub-millisecond veth asserts keep their 50 ms bound (three orders of magnitude of headroom, never observed to flake). --- tests/e2e/harness.py | 9 +++++++++ tests/e2e/test_dynamips_kernel_datapath.py | 2 +- tests/e2e/test_ethernet_switch_cascade.py | 2 +- tests/e2e/test_ethernet_switch_kernel_datapath.py | 4 ++-- tests/e2e/test_iol_docker_kernel_datapath.py | 2 +- tests/e2e/test_iou_kernel_datapath.py | 6 +++--- tests/e2e/test_qemu_kernel_datapath.py | 2 +- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 3b8bbda1b..217fe2dbc 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -727,6 +727,15 @@ def configure_ios(console, hostname, eth_ip=None, serial_ip=None, clock=False, e console.run("end") +# "No impairment applied" RTT bound for the fast-path asserts of the +# kernel-datapath scenarios: an unimpaired link measures ~20-45 ms on the +# IOS-based nodes (4 ms-quantized RTT; host scheduling jitter has pushed +# three-packet averages to 56 measured), while the delayed regime those +# asserts contrast against is >= 150 ms. 100 separates the two with +# headroom for a blip. +FAST_RTT_MS = 100 + + def ping(console, target, repeat=3, timeout=1): """Run an IOS ping; returns {success, avg, raw} (success in percent).""" out = console.run(f"ping {target} repeat {repeat} timeout {timeout}", timeout=repeat * timeout + 20) diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py index c4fc774c6..5d170eb98 100644 --- a/tests/e2e/test_dynamips_kernel_datapath.py +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -167,7 +167,7 @@ def test_dynamips_kernel_datapath(): compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(t1), harness.qdiscs(t1) fast = harness.wait_ping(c1, "10.1.1.2") - assert fast["success"] == 100 and fast["avg"] < 50, fast + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast # suspend: anchor admin-down, traffic dead; resume restores it compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) diff --git a/tests/e2e/test_ethernet_switch_cascade.py b/tests/e2e/test_ethernet_switch_cascade.py index 1bc02c5b8..c474a6124 100644 --- a/tests/e2e/test_ethernet_switch_cascade.py +++ b/tests/e2e/test_ethernet_switch_cascade.py @@ -269,7 +269,7 @@ def test_ethernet_switch_cascade_kernel_datapath(): compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"filters": {}}) assert "netem" not in harness.qdiscs(e0) and "netem" not in harness.qdiscs(e1) fast = harness.wait_ping(c1, IP10 % 2) - assert fast["success"] == 100 and fast["avg"] < 50, fast + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast # suspend: both ends admin-down, both VLANs die; resume restores. compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"suspend": True}) diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index 7cd91f48f..d42cd3a93 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -236,7 +236,7 @@ def test_ethernet_switch_kernel_fast_path(): compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") - assert fast["success"] == 100 and fast["avg"] < 50, fast + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast # The same single-owned impairment on link2 — created switch-first, # the order whose peer NIO update runs LAST. The passive end's empty @@ -252,7 +252,7 @@ def test_ethernet_switch_kernel_fast_path(): compute.call("PUT", f"/projects/{pid}/links/{link2['link_id']}", {"filters": {}}) assert "netem" not in harness.qdiscs(a2) fast2 = harness.wait_ping(c1, "10.1.1.2") - assert fast2["success"] == 100 and fast2["avg"] < 50, fast2 + assert fast2["success"] == 100 and fast2["avg"] < harness.FAST_RTT_MS, fast2 # suspend: the anchor admin-downs, the link is dead; resume restores compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"suspend": True}) diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py index a8fa56a63..9100aaf9b 100644 --- a/tests/e2e/test_iol_docker_kernel_datapath.py +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -179,7 +179,7 @@ def test_iol_docker_kernel_datapath(): compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") - assert fast["success"] == 100 and fast["avg"] < 50, fast + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast # Classifier spot check on the container's TAP anchor (the Docker # suite runs the full matrix on veth host ends): cls_bpf match-drop diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index a857bcd40..da4a22ddc 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -196,7 +196,7 @@ def test_iou_kernel_datapath(): compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") - assert fast["success"] == 100 and fast["avg"] < 50, fast + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast # Classifier spot check on a TAP anchor (the Docker suite runs the # full matrix on veth host ends): cls_bpf match-drop and the eBPF @@ -300,7 +300,7 @@ def test_iou_kernel_datapath(): assert survivor["avg"] >= 150, (baseline, survivor) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) cleared = harness.ping(c1, "10.1.1.2", repeat=3) - assert cleared["success"] == 100 and cleared["avg"] < 50, (baseline, cleared) + assert cleared["success"] == 100 and cleared["avg"] < harness.FAST_RTT_MS, (baseline, cleared) c1.close() c2.close() @@ -380,7 +380,7 @@ def test_iou_relay_control(): assert delayed["avg"] >= 150, (relay_ping, delayed) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) fast = harness.ping(c1, "10.1.1.2", repeat=3) - assert fast["success"] == 100 and fast["avg"] < 50, (relay_ping, fast) + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, (relay_ping, fast) # frequency_drop shares one counter between the port's two # directions: on an alternating ping stream one whole direction dies diff --git a/tests/e2e/test_qemu_kernel_datapath.py b/tests/e2e/test_qemu_kernel_datapath.py index 419d2c4ac..607b4387d 100644 --- a/tests/e2e/test_qemu_kernel_datapath.py +++ b/tests/e2e/test_qemu_kernel_datapath.py @@ -187,7 +187,7 @@ def test_qemu_kernel_datapath(): compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") - assert fast["success"] == 100 and fast["avg"] < 50, fast + assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast # suspend: anchor admin-down — the e1000 loses carrier, the bridge # port is disabled and nothing crosses; resume restores From 5841138cefeab42ac682e63c09c525606141a0a4 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 6 Oct 2026 00:13:06 +0800 Subject: [PATCH 088/101] =?UTF-8?q?docs(e2e):=20complete=20the=20README=20?= =?UTF-8?q?=E2=80=94=20requirements,=20image=20matrix,=20host=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds what running the suite on a real machine actually requires (uBridge with caps, tc for the filter assertions, a Docker daemon, cc -static for the raw-frame probe), the full image-selection matrix (preferred filenames, fallback patterns, per-folder layout, the alpine digest ladder; missing images skip, never false-fail), the NetworkManager host hazard and its record, timing/load sensitivity notes, the complete GNS3_E2E_* environment-variable list — and the link-local scenario, which the scenario list was missing. --- tests/e2e/README.md | 171 ++++++++++++++++---------------------------- 1 file changed, 61 insertions(+), 110 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 0ec9a509a..ef5a1ef82 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -5,12 +5,7 @@ See LICENSE file for licensing information. # Live end-to-end tests -Tests under this directory run against a **real** server: real uBridge, real -kernel interfaces (privileged), real emulator binaries and real images. They -verify what unit tests cannot — that a datapath actually forwards traffic, -that kernel bridges actually enslave anchors, that tc impairments actually -shape packets. They are marked `e2e` and **excluded from normal test runs** -(see `tests/pytest.ini`); run them explicitly: +Tests under this directory run against a **real** server: real uBridge, real kernel interfaces (privileged), real emulator binaries and real images. They verify what unit tests cannot — that a datapath actually forwards traffic, that kernel bridges actually enslave anchors, that tc impairments actually shape packets. They are marked `e2e` and **excluded from normal test runs** (see `tests/pytest.ini`) — and from CI, which runs the unit suite only. Run them explicitly, on a real machine: ```bash pytest -m e2e tests/e2e -s # -s shows the progress prints @@ -19,116 +14,72 @@ pytest -m e2e tests/e2e -k dynamips # one scenario Two run modes, chosen per test through `harness.live_server(kernel=...)`: -* **Target an existing server** — set `GNS3_E2E_URL` (e.g. - `http://127.0.0.1:3080/v3`, plus `GNS3_E2E_USER`/`GNS3_E2E_PASSWORD`, - default admin/admin). The server must already be configured for the - datapath under test (kernel links need `enable_kernel_datapath = True`). -* **Isolated instance** (the default) — the harness starts its own server - process on a free port with its own config directory (the controller DB - and every path follow the config), so nothing on the machine is touched. - It needs `ubridge`, `dynamips` and an images directory - (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally — the IOU - scenario takes its L3 image from that directory's `IOU` subfolder, the - QEMU scenario its L3 IOSv qcow2 from the `QEMU` subfolder; the Docker - scenarios additionally need the Docker daemon the server talks to. +* **Target an existing server** — set `GNS3_E2E_URL` (e.g. `http://127.0.0.1:3080/v3`, plus `GNS3_E2E_USER`/`GNS3_E2E_PASSWORD`, default admin/admin). The server must already be configured for the datapath under test (kernel links need `enable_kernel_datapath = True`). +* **Isolated instance** (the default) — the harness starts its own server process on a free port with its own config directory (the controller DB and every path follow the config), so nothing on the machine is touched besides the kernel devices the scenarios themselves create. It needs `ubridge`, `dynamips` and an images directory (`GNS3_E2E_IMAGES`, default `~/GNS3/images`) available locally — the IOU scenario takes its L3 image from that directory's `IOU` subfolder, the QEMU scenario its L3 IOSv qcow2 from the `QEMU` subfolder; the Docker scenarios additionally need the Docker daemon the server talks to. -Either way each test creates its own project and deletes it again. Set -`GNS3_E2E_KEEP=1` to keep the project (and a started instance) on failure -and print where to look at it. +Either way each test creates its own project and deletes it again. Set `GNS3_E2E_KEEP=1` to keep the project (and a started instance) on failure and print where to look at it. -The dynamips scenarios give their routers an idle-PC: detected once per -image through the controller's `auto_idlepc` (a throwaway project and a -real CPU-usage measurement — the GUI's idle-PC finder path) and cached in -`~/.cache/gns3-e2e/idlepc.json` keyed by the image checksum. Without one, -each c7200 burns a **full CPU core** for the whole scenario — the e2e -creates raw nodes, so no template supplies a value. Set `GNS3_E2E_IDLEPC` -to skip the detection and use the given value (detection failing is never -fatal: the routers simply run without one). +## Host requirements -`harness.py` holds the shared pieces — REST client, server lifecycle, the -IOS console driver, the Docker guest driver (a tiny Docker Engine API -client over the daemon socket, plus the digest-pinned image helper), a -WebSocket notification collector (a sync facade over `websockets`' sync -client for asserting on live notification streams), host-side kernel -inspections (bridge membership, tap admin state, tc qdiscs, the L2-anchor -spec §E assertions) — so a new node type adds only its own scenario module. +Beyond a Linux kernel with the datapath features (bridges, veth/TAP devices, tc netem/clsact/cls_bpf, the eBPF stateful classifier — each probed where it matters, and the affected assertions skip on kernels or uBridge builds without them): + +* **uBridge** with `cap_net_admin,cap_net_raw,cap_bpf` on `PATH` — the normal server install. +* **`tc` (iproute2)** for the filter scenarios: their qdisc and classifier assertions read it from `PATH`. Without it those assertions cannot run. +* **A Docker daemon** for the Docker scenarios (the harness's own guest driver talks to the same daemon as the server; socket override: `GNS3_E2E_DOCKER_SOCKET`). +* **A C compiler that can link statically** (`cc -static`) for the raw-frame probe of the link-local scenario; without it that probe skips. +* **Emulator images** — see the matrix below. + +## Test images + +Every scenario picks its image from the images directory (`GNS3_E2E_IMAGES`, default `~/GNS3/images`; a missing directory skips everything image-based). The first preferred name wins when present; otherwise the fallback pattern scans the directory. A missing image **skips** the scenario — never a false failure. + +| Scenario | Where | Preferred file(s) | Fallback | +|---|---|---|---| +| Dynamips (c7200) | images dir root | — | any file name starting with `c7200` | +| IOU | `IOU/` | `x86_64_crb_linux-adventerprisek9-ms.iol`, then `l3-adventerprisek9-ms-17.15.01.bin` | any name containing `adventerprisek9` and **not** `l2` | +| QEMU (IOSv) | `QEMU/` | `vios-adventerprisek9-m.spa.159-3.m12.qcow2` | any `vios` `.qcow2` without `l2` | +| IOL runner containers | Docker daemon | — | any locally loaded `iol-xe/iol-xe:*` image (any tag) | +| Docker scenarios | Docker daemon | `alpine:3` pinned at digest `sha256:28bd5fe8b56d…` | none — cached, else pulled, else skip (below) | + +Notes on the rows: + +* The Dynamips image must be a bootable c7200 IOS that supports the `PA-2FE-TX` module; the idle-PC story is described below. +* The IOU scenario needs an **L3** image — the `l2` exclusion is deliberate (the scenarios exercise the classic 2×4 Ethernet port model; L2 IOU images have their own issues). The isolated instance has the IOU license check disabled, so no `iourc` is needed. +* The IOL runner scenario stages the repository's `gns3server/configs/iol-xe-base.txt` as the base config so Cisco's iol-runner boots past its setup dialog. +* The preferred names are the exact builds the suites were live-verified against (Cisco-licensed artifacts — they cannot ship with the repo, hence the fallbacks). A fallback-selected build is tolerated but is an unverified combination. + +The dynamips scenarios give their routers an idle-PC: detected once per image through the controller's `auto_idlepc` (a throwaway project and a real CPU-usage measurement — the GUI's idle-PC finder path) and cached in `~/.cache/gns3-e2e/idlepc.json` keyed by the image checksum. Without one, each c7200 burns a **full CPU core** for the whole scenario — the e2e creates raw nodes, so no template supplies a value. Set `GNS3_E2E_IDLEPC` to skip the detection and use the given value (detection failing is never fatal: the routers simply run without one). + +`harness.py` holds the shared pieces — REST client, server lifecycle, the IOS console driver, the Docker guest driver (a tiny Docker Engine API client over the daemon socket, plus the digest-pinned image helper), a WebSocket notification collector (a sync facade over `websockets`' sync client for asserting on live notification streams), host-side kernel inspections (bridge membership, tap admin state, tc qdiscs, the L2-anchor spec §E assertions) — so a new node type adds only its own scenario module. Current scenarios: -* `test_dynamips_kernel_datapath.py` — two real c7200 routers: kernel link - (anchors, per-link bridge, netem delay, suspend, capture, delete/re-create, - stop/start) plus a serial link staying on the relay, and a negative - control on an isolated relay-configured instance. -* `test_ethernet_switch_kernel_datapath.py` — a real Ethernet switch - (kernel bridge) between two c7200 routers: deferred joins on boot, VLAN - isolation, absorbed-anchor filters/suspend/capture, VPCS peers staying on - the relay, and a relay negative control. -* `test_ethernet_switch_cascade.py` — two Ethernet switches cascaded through - the link-owned veth pair (`gs0/1`, each end enslaved into its own - switch's bridge): a dot1q trunk cascade carrying two VLANs at once across - two c7200 routers per switch, the symmetric access/trunk mode flip on the - cascade ends, two-sided netem (one end per direction), suspend, link - delete/re-create (the pair dies with the link, the replacement mints new - names), zero residue. -* `test_iol_docker_kernel_datapath.py` — two real iol-xe containers (Cisco - CML iol-runner images): kernel link through the port bridge's swappable - TAP leg (anchors at start, netem delay, the TAP-anchor classifier spot - check for bpf match-drop and the eBPF every-nth mode, suspend, capture, - link delete/re-create swap, node stop/start rewiring) and a relay - negative control. -* `test_iou_kernel_datapath.py` — two real IOU routers (a real IOU image - from the images directory, the real IOS CLI on the server's telnet - console): kernel link through the [IOL fabric ↔ TAP] port bridge (same - lifecycle as the iol-xe containers) including the TAP-anchor classifier - spot check and a filter surviving a node restart, a serial link staying - on the relay (serial bays never anchor), and a relay negative control - whose filters ride the IOU-specific `iol_bridge add_packet_filter` engine - (delay and frequency_drop measured through real traffic, capture and - markers on `iol_bridge start_capture` and the port's `mark` filter). -* `test_qemu_kernel_datapath.py` — two real IOSv routers (a real qcow2 - image from the images directory's `QEMU` subfolder, linked clone, the - real IOS CLI on the server's telnet console): the anchor TAP is QEMU's - own netdev (no port bridge in between) — the full kernel lifecycle plus - a relay negative control where the anchor carries as the relay's - AF_PACKET endpoint. IOSv boots a real IOS: this is the slowest scenario - pair (~6½ min kernel, ~5 min relay). -* `test_docker_kernel_datapath.py` — two real Alpine containers (the plain - veth datapath): the per-link kernel bridge enslaves exactly the two veth - host ends, real ICMP crosses, the filter matrix runs one type at a time - (netem core and extensions, cls_bpf match-drop, the eBPF classifier — - each measured through real traffic), plus a relay negative control whose - own filters (delay, frequency_drop, bpf) really shape the wire as uBridge - userspace bridge filters and whose capture and markers ride the relay - engine (`bridge start_capture` / the bridge `mark` filter), and a - relay→kernel reopen upgrade (the server's datapath choice is flipped - across a restart). -* `test_marker_kernel_datapath.py` — two real Alpine containers on a - kernel-only link with traffic-insight markers attached to a veth anchor - (`marker add_kernel`): `marker.match` signals collected off the dedicated - marker WebSocket with their full identity (filter/tag/link_id/node_id/len - and the tx/rx direction set of a ping), the main project channel asserted - free of matches, BPF and direction enforcement, host pcap exactness, - pause/resume silence counting against a still-firing sibling, tag replay - (timeline, display filter, link narrowing, window, the 409 gate) and - marker delete taking pcap plus signals with it. +* `test_dynamips_kernel_datapath.py` — two real c7200 routers: kernel link (anchors, per-link bridge, netem delay, suspend, capture, delete/re-create, stop/start) plus a serial link staying on the relay, and a negative control on an isolated relay-configured instance. +* `test_ethernet_switch_kernel_datapath.py` — a real Ethernet switch (kernel bridge) between two c7200 routers: deferred joins on boot, VLAN isolation, absorbed-anchor filters/suspend/capture, VPCS peers staying on the relay, and a relay negative control. +* `test_ethernet_switch_cascade.py` — two Ethernet switches cascaded through the link-owned veth pair (`gs0/1`, each end enslaved into its own switch's bridge): a dot1q trunk cascade carrying two VLANs at once across two c7200 routers per switch, the symmetric access/trunk mode flip on the cascade ends, two-sided netem (one end per direction), suspend, link delete/re-create (the pair dies with the link, the replacement mints new names), zero residue. +* `test_iol_docker_kernel_datapath.py` — two real iol-xe containers (Cisco CML iol-runner images): kernel link through the port bridge's swappable TAP leg (anchors at start, netem delay, the TAP-anchor classifier spot check for bpf match-drop and the eBPF every-nth mode, suspend, capture, link delete/re-create swap, node stop/start rewiring) and a relay negative control. +* `test_iou_kernel_datapath.py` — two real IOU routers (a real IOU image from the images directory, the real IOS CLI on the server's telnet console): kernel link through the [IOL fabric ↔ TAP] port bridge (same lifecycle as the iol-xe containers) including the TAP-anchor classifier spot check and a filter surviving a node restart, a serial link staying on the relay (serial bays never anchor), and a relay negative control whose filters ride the IOU-specific `iol_bridge add_packet_filter` engine (delay and frequency_drop measured through real traffic, capture and markers on `iol_bridge start_capture` and the port's `mark` filter). +* `test_qemu_kernel_datapath.py` — two real IOSv routers (a real qcow2 image from the images directory's `QEMU` subfolder, linked clone, the real IOS CLI on the server's telnet console): the anchor TAP is QEMU's own netdev (no port bridge in between) — the full kernel lifecycle plus a relay negative control where the anchor carries as the relay's AF_PACKET endpoint. IOSv boots a real IOS: this is the slowest scenario pair (~6½ min kernel, ~5 min relay). +* `test_docker_kernel_datapath.py` — two real Alpine containers (the plain veth datapath): the per-link kernel bridge enslaves exactly the two veth host ends, real ICMP crosses, the filter matrix runs one type at a time (netem core and extensions, cls_bpf match-drop, the eBPF classifier — each measured through real traffic), plus a relay negative control whose own filters (delay, frequency_drop, bpf) really shape the wire as uBridge userspace bridge filters and whose capture and markers ride the relay engine (`bridge start_capture` / the bridge `mark` filter), and a relay→kernel reopen upgrade (the server's datapath choice is flipped across a restart). +* `test_docker_link_local_frames.py` — two real Alpine containers on a kernel link, with a static raw-Ethernet probe built at run time and docker-cp'd in: guest 1 injects frames per destination MAC while guest 2 counts what arrived. Asserts the per-port `brport/group_fwd_mask` reads 0xfffd (applied by `brctl addif`) and the bridge-level mask stays 0; the live matrix: ordinary multicast, STP/RSTP, LACP, EAPOL and LLDP/DCBX cross, MAC PAUSE does not (a documented kernel hard-drop — PFC labs need a relay link); a container stop/start re-applies the mask (the reconnect story). Needs `cc -static` on the host. +* `test_marker_kernel_datapath.py` — two real Alpine containers on a kernel-only link with traffic-insight markers attached to a veth anchor (`marker add_kernel`): `marker.match` signals collected off the dedicated marker WebSocket with their full identity (filter/tag/link_id/node_id/len and the tx/rx direction set of a ping), the main project channel asserted free of matches, BPF and direction enforcement, host pcap exactness, pause/resume silence counting against a still-firing sibling, tag replay (timeline, display filter, link narrowing, window, the 409 gate) and marker delete taking pcap plus signals with it. -The Docker scenarios run a digest-pinned `alpine:3`, so every developer -tests the same bytes: a cached copy in the local Docker daemon costs no -network, a cache miss pulls through the server's own image-pull route, and -with neither cache nor registry the scenario skips (`GNS3_E2E_DOCKER_IMAGE` -overrides the reference). Their guests are addressed the way a GNS3 user -does it — the node's persistent `/etc/network/interfaces`, which the -container's `init.sh` applies at boot — and driven through the Docker -daemon socket the server itself uses. +The Docker scenarios run a digest-pinned `alpine:3`, so every developer tests the same bytes: a cached copy in the local Docker daemon costs no network, a cache miss pulls through the server's own image-pull route, and with neither cache nor registry the scenario skips (`GNS3_E2E_DOCKER_IMAGE` overrides the reference). Their guests are addressed the way a GNS3 user does it — the node's persistent `/etc/network/interfaces`, which the container's `init.sh` applies at boot — and driven through the Docker daemon socket the server itself uses. -Every kernel scenario also asserts the L2-anchor spec's server-side -guarantees (`docs/design/ubridge-l2-anchor-spec.md` §E): anchors and bridges -carry no L3 identity (`assert_pure_l2`), joined ports are FORWARDING -(`assert_forwarding`), and an idle link with the guests silenced stays -silent for 5 s (`assert_idle_silence`) — the guests are the IOS consoles' -interfaces shut, the Docker guests' links down, or all four cascade routers -silenced, whichever the scenario runs. These skip on a uBridge without -`link l2only` (probed once by `harness.l2only_supported()`). +Every kernel scenario also asserts the L2-anchor spec's server-side guarantees (`docs/design/ubridge-l2-anchor-spec.md` §E): anchors and bridges carry no L3 identity (`assert_pure_l2`), joined ports are FORWARDING (`assert_forwarding`), and an idle link with the guests silenced stays silent for 5 s (`assert_idle_silence`) — the guests are the IOS consoles' interfaces shut, the Docker guests' links down, or all four cascade routers silenced, whichever the scenario runs. These skip on a uBridge without `link l2only` (probed once by `harness.l2only_supported()`). -Set `GNS3_E2E_DEBUG=1` to run the isolated instance with debug logging (the -uBridge command stream lands in its `server.log`). +## Host environment notes + +* **NetworkManager**: on hosts where NM manages the TAP anchors (many desktop Linux installs), NM can release a freshly (re-)enslaved anchor from its bridge ~60 ms after the join — silently, leaving the API green and the link dead until the next NIO update. The harness takes the anchors out of NM's reach where scenarios re-enslave them (`harness.unmanage_from_networkmanager` — a no-op where `nmcli` is absent), and the permanent host-side fix (an `unmanaged-devices` declaration) plus the full root-cause record live in `docs/bugs/networkmanager-tap-release.md`. A host with NM but without `nmcli` can still see the rare spontaneous release on the first runs. +* **Timing and load**: the traffic assertions of the IOS-based scenarios are three-packet averages over real emulated routers; the RTT bounds are deliberately generous (`harness.FAST_RTT_MS`) but a heavily loaded host can still push a sample over — run the suite on a machine with a few cores to spare. Boot waits (`_boot` timeouts) assume a responsive host too. +* **CI**: these tests do not run in CI (the testing workflow runs the unit suite, with the `e2e` marker deselected). They are a developer/validator tool for real machines. + +## Environment variables + +* `GNS3_E2E_URL`, `GNS3_E2E_USER`, `GNS3_E2E_PASSWORD` — target an existing server instead of starting an isolated instance (default user/password: admin/admin). +* `GNS3_E2E_KEEP=1` — keep the project (and an isolated instance) on failure and print where to look at it. +* `GNS3_E2E_IMAGES` — the images directory (default `~/GNS3/images`). +* `GNS3_E2E_DOCKER_IMAGE` — override the pinned Docker image entirely (the operator then owns its content). +* `GNS3_E2E_DOCKER_SOCKET` — the Docker daemon socket the harness's guest driver talks to (default `/var/run/docker.sock` — the same daemon the server uses). +* `GNS3_E2E_IDLEPC` — skip idle-PC detection for the Dynamips scenarios and use the given value. +* `GNS3_E2E_DEBUG=1` — run the isolated instance with debug logging (the uBridge command stream lands in its `server.log`). From 87ca2138e3395cb63c6952d56a09e74a4af43746 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 6 Oct 2026 00:28:06 +0800 Subject: [PATCH 089/101] chore(lint): apply the 3.1 f-string wave to the kernel-datapath code The rebase onto 65c858d45 brought upstream's UP032 enforcement; the conflict resolution kept this branch's semantics on the overlapping lines, so their .format() calls needed the conversion afterwards (ruff check --fix, 41 sites) plus the formatter's rewrap (7 files). Replicates the CI recipe locally: pip check, ruff check/format, mypy 2.3.1 (clean, 434 files), unit suite 2216 passed. --- gns3server/compute/base_node.py | 2 +- .../compute/builtin/nodes/ethernet_switch.py | 26 +++------ .../compute/docker/docker_kernel_datapath.py | 13 +---- gns3server/compute/docker/docker_vm.py | 18 ++----- gns3server/compute/docker/iol_docker_vm.py | 4 +- gns3server/compute/dynamips/nodes/router.py | 54 ++++--------------- gns3server/compute/iou/iou_vm.py | 48 ++++------------- gns3server/compute/kernel_datapath.py | 24 +++------ gns3server/compute/qemu/qemu_vm.py | 18 ++----- 9 files changed, 49 insertions(+), 158 deletions(-) diff --git a/gns3server/compute/base_node.py b/gns3server/compute/base_node.py index 56076e83c..7ff62ae07 100644 --- a/gns3server/compute/base_node.py +++ b/gns3server/compute/base_node.py @@ -1226,7 +1226,7 @@ class BaseNode: # mark [tag ] [pcap ] — tag/pcap keyword pairs, any order. self._validate_marker_name(name) - cmd = 'bridge add_packet_filter {bridge} {name} mark "{bpf}"'.format(bridge=bridge_name, name=name, bpf=bpf) + cmd = f'bridge add_packet_filter {bridge_name} {name} mark "{bpf}"' if tag is not None: cmd += f" tag {tag}" # Per-link attribution (contract §3.2): when one ubridge bridge serves diff --git a/gns3server/compute/builtin/nodes/ethernet_switch.py b/gns3server/compute/builtin/nodes/ethernet_switch.py index 2b9dab4d0..e508649c3 100644 --- a/gns3server/compute/builtin/nodes/ethernet_switch.py +++ b/gns3server/compute/builtin/nodes/ethernet_switch.py @@ -488,19 +488,15 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): await self._ubridge_send(f'link l2only "{anchor}" on') except UbridgeError as e: log.warning( - 'Ethernet switch "{name}" [{id}]: could not harden cascade end "{anchor}" L2-only: {error}'.format( - name=self._name, id=self._id, anchor=anchor, error=e - ) + f'Ethernet switch "{self._name}" [{self._id}]: could not harden cascade end "{anchor}" L2-only: {e}' ) # Register before _kernel_attach: the marker reconcile recognises # kernel anchors through _kernel_anchors(). self._kernel_ports[port_number] = anchor if not os.path.exists(f"/sys/class/net/{anchor}"): log.info( - 'Ethernet switch "{name}" [{id}]: anchor {anchor} for port {port} does not exist yet ' - "(peer not started); join deferred".format( - name=self._name, id=self._id, anchor=anchor, port=port_number - ) + f'Ethernet switch "{self._name}" [{self._id}]: anchor {anchor} for port {port_number} does not exist yet ' + "(peer not started); join deferred" ) return try: @@ -737,8 +733,8 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): # a wired port always has previous settings; without them a # diff is impossible and re-enslaving would be guessing log.warning( - 'Ethernet switch "{name}" [{id}]: no previous settings for wired port {port}; ' - "VLAN reconfiguration skipped".format(name=self._name, id=self._id, port=port_number) + f'Ethernet switch "{self._name}" [{self._id}]: no previous settings for wired port {port_number}; ' + "VLAN reconfiguration skipped" ) continue await self._reconfigure_port_vlan(iface, old_settings, new_settings) @@ -769,11 +765,7 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): else: ubridge_bridge = self._ubridge_bridge_name(port_number) await self._ubridge_send(f'bridge start_capture {ubridge_bridge} "{output_file}"') - log.debug( - 'Ethernet switch "{name}" [{id}]: starting packet capture on port {port}'.format( - name=self.name, id=self.id, port=port_number - ) - ) + log.debug(f'Ethernet switch "{self.name}" [{self.id}]: starting packet capture on port {port_number}') async def stop_capture(self, port_number): """ @@ -792,8 +784,4 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): else: ubridge_bridge = self._ubridge_bridge_name(port_number) await self._ubridge_send(f"bridge stop_capture {ubridge_bridge}") - log.debug( - 'Ethernet switch "{name}" [{id}]: stopping packet capture on port {port}'.format( - name=self.name, id=self.id, port=port_number - ) - ) + log.debug(f'Ethernet switch "{self.name}" [{self.id}]: stopping packet capture on port {port_number}') diff --git a/gns3server/compute/docker/docker_kernel_datapath.py b/gns3server/compute/docker/docker_kernel_datapath.py index 06ce57b44..8c8c62b9a 100644 --- a/gns3server/compute/docker/docker_kernel_datapath.py +++ b/gns3server/compute/docker/docker_kernel_datapath.py @@ -101,11 +101,7 @@ class DockerKernelDatapathMixin(KernelDatapathMixin): try: adapter = self._ethernet_adapters[adapter_number] except IndexError: - raise DockerError( - "Adapter {adapter_number} doesn't exist on Docker container '{name}'".format( - name=self.name, adapter_number=adapter_number - ) - ) + raise DockerError(f"Adapter {adapter_number} doesn't exist on Docker container '{self.name}'") host_ifc, guest_ifc = self._veth_names(adapter_number, port_number) # The host end lives in the root namespace and survives container @@ -144,12 +140,7 @@ class DockerKernelDatapathMixin(KernelDatapathMixin): adapter.host_ifc = host_ifc self._kernel_veths[(adapter_number, port_number)] = host_ifc log.debug( - "Created veth adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {namespace}".format( - adapter_number=adapter_number, - port_number=port_number, - mac_address=mac_address, - namespace=self._namespace, - ) + f"Created veth adapter {adapter_number} port {port_number} with MAC address {mac_address} in namespace {self._namespace}" ) async def _remove_kernel_veths(self): diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 3d6f611cb..4cb34d85e 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1815,11 +1815,9 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): # rejected earlier): the kernel path needs an anchor — the # veth host end, or the persistent TAP of an IOL runner. raise DockerError( - "Adapter {adapter_number} port {port_number} of container '{name}' has no host-side " + f"Adapter {adapter_number} port {port_number} of container '{self.name}' has no host-side " "anchor interface (it was started before the unified-veth datapath); " - "restart the node to attach a kernel link to it".format( - adapter_number=adapter_number, port_number=port_number, name=self.name - ) + "restart the node to attach a kernel link to it" ) # The mechanics (bridge create/addif, capture, markers, filters) # live in KernelDatapathMixin, shared with every other node type @@ -1845,11 +1843,7 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): # NIO sets it back down). await self._ubridge_send(f'link set "{host_ifc}" up') await self._ubridge_send(f'bridge add_nio_ethernet {bridge_name} "{host_ifc}"') - await self._ubridge_send( - "bridge add_nio_udp {bridge_name} {lport} {rhost} {rport}".format( - bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport - ) - ) + await self._ubridge_send(f"bridge add_nio_udp {bridge_name} {nio.lport} {nio.rhost} {nio.rport}") if nio.capturing: await self._ubridge_send(f'bridge start_capture {bridge_name} "{nio.pcap_output_file}"') await self._ubridge_send(f"bridge start {bridge_name}") @@ -1934,11 +1928,7 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): if self.status == "started": await self._set_adapter_carrier(adapter_number, False, port_number) await self._ubridge_send(f"bridge stop {bridge_name}") - await self._ubridge_send( - "bridge remove_nio_udp {bridge_name} {lport} {rhost} {rport}".format( - bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport - ) - ) + await self._ubridge_send(f"bridge remove_nio_udp {bridge_name} {nio.lport} {nio.rhost} {nio.rport}") adapter.remove_nio(port_number) diff --git a/gns3server/compute/docker/iol_docker_vm.py b/gns3server/compute/docker/iol_docker_vm.py index a41b66e56..e6d2bf70a 100644 --- a/gns3server/compute/docker/iol_docker_vm.py +++ b/gns3server/compute/docker/iol_docker_vm.py @@ -573,10 +573,10 @@ class IOLDockerVM(VendorDockerVM): anchor = self._kernel_host_ifc(adapter_number, port_number) if anchor is None: raise DockerError( - "Bay {bay}/{unit} of IOL container '{name}' has no TAP anchor to carry a kernel " + f"Bay {adapter_number}/{port_number} of IOL container '{self._name}' has no TAP anchor to carry a kernel " "link (this compute's uBridge lacks the tap module or bridge delete_nio_tap, " "or the node was started before the kernel-datapath support); restart the " - "node after upgrading uBridge".format(bay=adapter_number, unit=port_number, name=self._name) + "node after upgrading uBridge" ) await self._ensure_unix_port_bridge(adapter_number, port_number) await self._ensure_anchor(adapter_number, port_number) diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index 05c055b13..f763d7dd5 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -340,11 +340,7 @@ class Router(KernelDatapathMixin, BaseNode): # the node was stopped wire themselves as IOS comes up. await self._prepare_tap_datapath() - await self._hypervisor.send( - 'vm set_config "{name}" "{startup}" "{private}"'.format( - name=self._name, startup=startup_config_path, private=private_config_path - ) - ) + await self._hypervisor.send(f'vm set_config "{self._name}" "{startup_config_path}" "{private_config_path}"') await self._hypervisor.send(f'vm start "{self._name}"') self.status = "started" log.debug(f'router "{self._name}" [{self._id}] has been started') @@ -1318,11 +1314,7 @@ class Router(KernelDatapathMixin, BaseNode): # Refusing also keeps the kernel wiring's own idempotence check from # silently skipping the attach while rebinding the bookkeeping. if adapter.get_nio(port_number) is not None: - raise DynamipsError( - "Port {port_number} on slot {slot_number} of router '{name}' already has a link".format( - name=self._name, slot_number=slot_number, port_number=port_number - ) - ) + raise DynamipsError(f"Port {port_number} on slot {slot_number} of router '{self._name}' already has a link") if isinstance(nio, NIOBridge): # Kernel datapath: the port's anchor TAP is opened in the @@ -1332,11 +1324,7 @@ class Router(KernelDatapathMixin, BaseNode): # the port unbound instead of half-wired. await self._attach_kernel_nio(slot_number, port_number, nio) adapter.add_nio(port_number, nio) - log.debug( - 'Router "{name}" [{id}]: {nio} bound to port {slot_number}/{port_number}'.format( - name=self._name, id=self._id, nio=nio, slot_number=slot_number, port_number=port_number - ) - ) + log.debug(f'Router "{self._name}" [{self._id}]: {nio} bound to port {slot_number}/{port_number}') return try: @@ -1401,19 +1389,11 @@ class Router(KernelDatapathMixin, BaseNode): if isinstance(nio, NIOBridge): await self._remove_kernel_nio_binding(slot_number, port_number, nio) adapter.remove_nio(port_number) - log.debug( - 'Router "{name}" [{id}]: {nio} removed from port {slot_number}/{port_number}'.format( - name=self._name, id=self._id, nio=nio, slot_number=slot_number, port_number=port_number - ) - ) + log.debug(f'Router "{self._name}" [{self._id}]: {nio} removed from port {slot_number}/{port_number}') return nio await self.slot_disable_nio(slot_number, port_number) - await self._hypervisor.send( - 'vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format( - name=self._name, slot_number=slot_number, port_number=port_number - ) - ) + await self._hypervisor.send(f'vm slot_remove_nio_binding "{self._name}" {slot_number} {port_number}') await nio.close() adapter.remove_nio(port_number) @@ -1512,9 +1492,7 @@ class Router(KernelDatapathMixin, BaseNode): if anchor is not None: await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') log.debug( - 'Router "{name}" [{id}]: starting packet capture on port {slot_number}/{port_number}'.format( - name=self._name, id=self._id, slot_number=slot_number, port_number=port_number - ) + f'Router "{self._name}" [{self._id}]: starting packet capture on port {slot_number}/{port_number}' ) return @@ -1726,19 +1704,15 @@ class Router(KernelDatapathMixin, BaseNode): if self.status != "started": return raise self._kernel_error( - "Port {slot}/{port} of router '{name}' runs on the relay datapath " - "(this uBridge cannot create persistent TAPs) and cannot carry a kernel link".format( - slot=slot_number, port=port_number, name=self._name - ) + f"Port {slot_number}/{port_number} of router '{self._name}' runs on the relay datapath " + "(this uBridge cannot create persistent TAPs) and cannot carry a kernel link" ) tap_nio = NIOTAP(self._hypervisor, anchor) await tap_nio.create() try: await self._hypervisor.send( - 'vm slot_add_nio_binding "{name}" {slot_number} {port_number} {nio}'.format( - name=self._name, slot_number=slot_number, port_number=port_number, nio=tap_nio.name - ) + f'vm slot_add_nio_binding "{self._name}" {slot_number} {port_number} {tap_nio.name}' ) except DynamipsError: with contextlib.suppress(DynamipsError): @@ -1764,11 +1738,7 @@ class Router(KernelDatapathMixin, BaseNode): with contextlib.suppress(DynamipsError, OSError): await self.slot_disable_nio(slot_number, port_number) with contextlib.suppress(DynamipsError, OSError): - await self._hypervisor.send( - 'vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format( - name=self._name, slot_number=slot_number, port_number=port_number - ) - ) + await self._hypervisor.send(f'vm slot_remove_nio_binding "{self._name}" {slot_number} {port_number}') with contextlib.suppress(DynamipsError, OSError): await tap_nio.delete() @@ -1788,9 +1758,7 @@ class Router(KernelDatapathMixin, BaseNode): for (slot_number, port_number), tap_nio in list(self._tap_nios.items()): with contextlib.suppress(DynamipsError, OSError): await self._hypervisor.send( - 'vm slot_remove_nio_binding "{name}" {slot_number} {port_number}'.format( - name=self._name, slot_number=slot_number, port_number=port_number - ) + f'vm slot_remove_nio_binding "{self._name}" {slot_number} {port_number}' ) with contextlib.suppress(DynamipsError, OSError): await tap_nio.delete() diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index ec7aed43b..d3cd01dfc 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -950,18 +950,12 @@ class IOUVM(KernelDatapathMixin, BaseNode): anchor = self._kernel_host_ifc(adapter_number, port_number) if anchor is None: raise self._kernel_error( - "Bay {bay}/{unit} of IOU '{name}' has no TAP anchor to carry a kernel link " + f"Bay {adapter_number}/{port_number} of IOU '{self._name}' has no TAP anchor to carry a kernel link " "(serial ports stay on the relay datapath, and a node whose uBridge lacks " - "iol_bridge add_nio_tap runs relay-only)".format(bay=adapter_number, unit=port_number, name=self._name) + "iol_bridge add_nio_tap runs relay-only)" ) await self._ubridge_send( - 'iol_bridge add_nio_tap {name} {iol_id} {bay} {unit} "{tap}"'.format( - name=self._iol_bridge_name(), - iol_id=self.application_id, - bay=adapter_number, - unit=port_number, - tap=anchor, - ) + f'iol_bridge add_nio_tap {self._iol_bridge_name()} {self.application_id} {adapter_number} {port_number} "{anchor}"' ) await self._kernel_attach(anchor, nio) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) @@ -1348,11 +1342,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): # a port carries at most one NIO, and overwriting the one a link # still owns orphans that link's teardown (its host state leaks). if adapter.get_nio(port_number) is not None: - raise IOUError( - "Port {port_number} on adapter {adapter_number} of IOU '{name}' already has a link".format( - name=self._name, adapter_number=adapter_number, port_number=port_number - ) - ) + raise IOUError(f"Port {port_number} on adapter {adapter_number} of IOU '{self._name}' already has a link") # Wire before bookkeeping: an attach that fails mid-way leaves the # port unbound instead of half-wired. @@ -1361,15 +1351,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): await self._attach_kernel_nio(adapter_number, port_number, nio) else: await self._ubridge_send( - "iol_bridge add_nio_udp {name} {iol_id} {bay} {unit} {lport} {rhost} {rport}".format( - name=self._iol_bridge_name(), - iol_id=self.application_id, - bay=adapter_number, - unit=port_number, - lport=nio.lport, - rhost=nio.rhost, - rport=nio.rport, - ) + f"iol_bridge add_nio_udp {self._iol_bridge_name()} {self.application_id} {adapter_number} {port_number} {nio.lport} {nio.rhost} {nio.rport}" ) location = self._iol_location(adapter_number, port_number) await self._ubridge_apply_filters(location, nio.filters) @@ -1442,7 +1424,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): return self._validate_marker_name(name) # iol_bridge add_packet_filter {br} {bay} {unit} {name} mark "{bpf}" [tag {id}] pcap "{path}" - cmd = 'iol_bridge add_packet_filter {loc} {name} mark "{bpf}"'.format(loc=location, name=name, bpf=bpf) + cmd = f'iol_bridge add_packet_filter {location} {name} mark "{bpf}"' if tag is not None: cmd += f" tag {tag}" if link_id: @@ -1452,7 +1434,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): linktype = self._marker_linktype(data_link_type) if linktype is not None: cmd += f" linktype {linktype}" - cmd += ' pcap "{path}"'.format(path=pcap_path) + cmd += f' pcap "{pcap_path}"' await self._ubridge_send(cmd) async def _ubridge_enable_marker_filter(self, location, name, state): @@ -1518,15 +1500,11 @@ class IOUVM(KernelDatapathMixin, BaseNode): # persistent device itself survives for the next link. with contextlib.suppress(UbridgeError): await self._ubridge_send( - "iol_bridge delete_nio_tap {name} {bay} {unit}".format( - name=self._iol_bridge_name(), bay=adapter_number, unit=port_number - ) + f"iol_bridge delete_nio_tap {self._iol_bridge_name()} {adapter_number} {port_number}" ) else: await self._ubridge_send( - "iol_bridge delete_nio_udp {name} {bay} {unit}".format( - name=self._iol_bridge_name(), bay=adapter_number, unit=port_number - ) + f"iol_bridge delete_nio_udp {self._iol_bridge_name()} {adapter_number} {port_number}" ) return nio @@ -1852,9 +1830,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): anchor = self._kernel_host_ifc(adapter_number, port_number) if anchor is None: raise self._kernel_error( - "Bay {bay}/{unit} of IOU '{name}' has no TAP anchor to capture on".format( - bay=adapter_number, unit=port_number, name=self._name - ) + f"Bay {adapter_number}/{port_number} of IOU '{self._name}' has no TAP anchor to capture on" ) await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') else: @@ -1886,7 +1862,5 @@ class IOUVM(KernelDatapathMixin, BaseNode): await self._ubridge_send("capture stop_kernel") else: await self._ubridge_send( - "iol_bridge stop_capture {name} {bay} {unit}".format( - name=self._iol_bridge_name(), bay=adapter_number, unit=port_number - ) + f"iol_bridge stop_capture {self._iol_bridge_name()} {adapter_number} {port_number}" ) diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index 09e80ec4a..4a23c2b51 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -214,17 +214,9 @@ class KernelDatapathMixin: # afterwards (a suspended NIO sets it back down). await self._ubridge_send(f'link set "{anchor}" up') await self._ubridge_send(f'bridge add_nio_ethernet {bridge_name} "{anchor}"') - await self._ubridge_send( - "bridge add_nio_udp {bridge_name} {lport} {rhost} {rport}".format( - bridge_name=bridge_name, lport=nio.lport, rhost=nio.rhost, rport=nio.rport - ) - ) + await self._ubridge_send(f"bridge add_nio_udp {bridge_name} {nio.lport} {nio.rhost} {nio.rport}") if nio.capturing: - await self._ubridge_send( - 'bridge start_capture {bridge_name} "{pcap_file}"'.format( - bridge_name=bridge_name, pcap_file=nio.pcap_output_file - ) - ) + await self._ubridge_send(f'bridge start_capture {bridge_name} "{nio.pcap_output_file}"') await self._ubridge_send(f"bridge start {bridge_name}") await self._ubridge_apply_filters(bridge_name, nio.filters) await self._ubridge_apply_markers(bridge_name, nio) @@ -328,7 +320,7 @@ class KernelDatapathMixin: if self._kernel_marker_anchor(bridge_name): self._validate_marker_name(name) - cmd = 'marker add_kernel {name} {ifc} "{bpf}"'.format(name=name, ifc=bridge_name, bpf=bpf) + cmd = f'marker add_kernel {name} {bridge_name} "{bpf}"' if tag is not None: cmd += f" tag {tag}" if link_id: @@ -338,7 +330,7 @@ class KernelDatapathMixin: linktype = self._marker_linktype(data_link_type) if linktype is not None: cmd += f" linktype {linktype}" - cmd += ' pcap "{path}"'.format(path=pcap_path) + cmd += f' pcap "{pcap_path}"' await self._ubridge_send(cmd) return await super()._ubridge_add_marker_filter( @@ -614,9 +606,7 @@ class KernelDatapathMixin: await self._ubridge_send(f'tc bpf_drop flush "{host_ifc}"') for offset, line in enumerate(lines): try: - await self._ubridge_send( - 'tc bpf_drop add "{ifc}" {prio} "{expr}"'.format(ifc=host_ifc, prio=10 + offset, expr=line) - ) + await self._ubridge_send(f'tc bpf_drop add "{host_ifc}" {10 + offset} "{line}"') except UbridgeError as e: if "Cannot compile filter" not in str(e): raise @@ -710,9 +700,7 @@ class KernelDatapathMixin: elif "nth" in modes: await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') if quota: - await self._ubridge_send( - 'tc quota_drop "{ifc}" {bytes} {pct}'.format(ifc=host_ifc, bytes=int(quota[0]), pct=int(quota[1])) - ) + await self._ubridge_send(f'tc quota_drop "{host_ifc}" {int(quota[0])} {int(quota[1])}') elif "quota" in modes: await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') if window: diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 410094348..db842f781 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1635,10 +1635,8 @@ class QemuVM(KernelDatapathMixin, BaseNode): if not self._tap_datapath: if isinstance(nio, NIOBridge): raise QemuError( - "Adapter {adapter_number} of QEMU VM '{name}' runs on the legacy relay datapath " - "(this uBridge cannot create persistent TAPs) and cannot carry a kernel link".format( - adapter_number=adapter_number, name=self._name - ) + f"Adapter {adapter_number} of QEMU VM '{self._name}' runs on the legacy relay datapath " + "(this uBridge cannot create persistent TAPs) and cannot carry a kernel link" ) await self.add_ubridge_udp_connection(bridge_name, self._local_udp_tunnels[adapter_number][1], nio) return @@ -1646,10 +1644,8 @@ class QemuVM(KernelDatapathMixin, BaseNode): anchor = self._kernel_host_ifc(adapter_number, port_number) if anchor is None: raise QemuError( - "Adapter {adapter_number} port {port_number} of QEMU VM '{name}' has no TAP interface; " - "restart the node to attach a link to it".format( - adapter_number=adapter_number, port_number=port_number, name=self._name - ) + f"Adapter {adapter_number} port {port_number} of QEMU VM '{self._name}' has no TAP interface; " + "restart the node to attach a link to it" ) if isinstance(nio, NIOBridge): await self._kernel_attach(anchor, nio) @@ -1673,11 +1669,7 @@ class QemuVM(KernelDatapathMixin, BaseNode): try: await self._connect_nio(adapter_number, nio) except (IndexError, KeyError): - raise QemuError( - 'Adapter {adapter_number} does not exist on QEMU VM "{name}"'.format( - name=self._name, adapter_number=adapter_number - ) - ) + raise QemuError(f'Adapter {adapter_number} does not exist on QEMU VM "{self._name}"') await self._set_adapter_carrier(adapter_number, not nio.suspend) if self._replicate_network_connection_state: await self._control_vm(f"set_link gns3-{adapter_number} on") From c3c025ab2ac2032742452118341c00d2ece83183 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 6 Oct 2026 19:55:21 +0800 Subject: [PATCH 090/101] fix: resolve the code-review findings on the kernel datapath Compute: - Track the anchor that owns uBridge's single AF_PACKET capture slot: a second kernel port's capture is refused with a clear error (its flag rolled back), a failed start no longer wedges the port, and only the owning port's stop issues the process-wide `capture stop_kernel` (docker/qemu/iou/dynamips/switch, via the mixin's slot helpers). - Unwind failed starts instead of leaking uBridge and the anchor TAPs: QEMU's stop body runs under try/finally, QEMU and IOU start the datapath inside the launch try, Dynamips undoes a half-attached port (the retry used to be short-circuited into a silent black hole), and Docker deletes the veth pair when the namespace move fails. - Restore the relay filter guard on IOU (kernel-only filters raise NodeError, an uncompilable BPF line degrades to a warning), validate the eBPF/quota/window/bpf filter shapes before touching the bridge (short or non-numeric values surfaced as 500s after partial state), and refuse a second NIO on an occupied Docker/QEMU port (the guard IOU and Dynamips already had). - Probe fixes: `_probe_bridge_tap` survives a failing uBridge start, and the probe caches key on ctime too, so a chmod/setcap remedy re-probes. Controller: - Parse GNS3_UNIX_SOCKET_NIO like the compute does (line-anchored, truthy value) and mirror the compute's class selection in kernel_anchor_type. - Read partial capability reports defensively; make BridgeNIO.bridge required-but-nullable so an omitted key is a 422 rather than a KeyError. - Strip the packet_loss correlation slot even when it is zero: the relay filter takes exactly one argument and rejected `packet_loss 10 0`. Ethernet switch: skip ports whose absorbed anchor does not exist yet (the join is deferred by design) and roll the ports_mapping back when the VLAN reconcile fails, so a retry still diffs against the old mapping. Agent: reject device config entries without a device_name instead of silently dropping them (a malformed batch used to execute partially while reporting success). Also split the netem builder to clear C901 and drop the two per-file ruff ignores the PR had added, keeping the remove-only policy. --- docs/features/docker-kernel-datapath.md | 2 +- .../tools_v2/config_tools_nornir.py | 6 +- .../tools_v2/display_tools_nornir.py | 6 +- .../tools_v2/vpcs_tools_netmiko.py | 6 +- .../gns3_copilot/utils/device_configs.py | 21 +- .../routes/compute/ethernet_switch_nodes.py | 11 +- gns3server/compute/base_node.py | 51 ++- .../compute/builtin/nodes/ethernet_switch.py | 28 +- .../compute/docker/docker_kernel_datapath.py | 7 +- gns3server/compute/docker/docker_vm.py | 32 +- gns3server/compute/dynamips/nodes/router.py | 37 ++- gns3server/compute/iou/iou_vm.py | 182 ++++++----- gns3server/compute/kernel_datapath.py | 299 ++++++++++++++---- gns3server/compute/qemu/qemu_vm.py | 161 ++++++---- gns3server/compute/ubridge/tc_probe.py | 25 +- gns3server/controller/link.py | 10 +- gns3server/controller/udp_link.py | 12 +- gns3server/schemas/compute/nios.py | 5 +- gns3server/utils/kernel_anchor.py | 18 +- gns3server/utils/packet_filter_validation.py | 7 +- pyproject.toml | 2 - 21 files changed, 654 insertions(+), 274 deletions(-) diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index 8e36cd1ce..d2f7232e1 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -243,7 +243,7 @@ The veth pairs already exist at attach time — every adapter is born as a veth ## Capture -Kernel links capture via uBridge's AF_PACKET module bound to the veth host end: `capture start_kernel "" [dlt]` / `capture stop_kernel`. Single capture per uBridge process (second concurrent → EALREADY). Start/stop key on the **NIO type**, not on veth presence — a relay NIO riding a veth captures at its relay bridge. +Kernel links capture via uBridge's AF_PACKET module bound to the veth host end: `capture start_kernel "" [dlt]` / `capture stop_kernel`. Single capture per uBridge process (second concurrent → EALREADY): the server tracks which anchor owns the slot, refuses a second port's capture with a clear error instead of leaving it wedged, and only the owning port's stop issues the process-wide `capture stop_kernel`. Start/stop key on the **NIO type**, not on veth presence — a relay NIO riding a veth captures at its relay bridge. ## Markers diff --git a/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py b/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py index 4bf507b25..5d2506a65 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py +++ b/gns3server/agent/gns3_copilot/tools_v2/config_tools_nornir.py @@ -200,7 +200,11 @@ class ExecuteMultipleDeviceConfigCommands(BaseTool): return device_configs_list # Merge same-device entries before anything keys by device_name - device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="config_commands") + try: + device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="config_commands") + except ValueError as e: + logger.error("Invalid device_configs input: %s", e) + return [{"status": "failed", "error": str(e)}] # Filter forbidden commands and store blocked commands info device_configs_list, blocked_commands_map = self._filter_forbidden_commands_from_device_configs( diff --git a/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py b/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py index afdeac035..bbb92b0c5 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py +++ b/gns3server/agent/gns3_copilot/tools_v2/display_tools_nornir.py @@ -203,7 +203,11 @@ class ExecuteMultipleDeviceCommands(BaseTool): return device_configs_list # Merge same-device entries before anything keys by device_name - device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="commands") + try: + device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="commands") + except ValueError as e: + logger.error("Invalid device_configs input: %s", e) + return [{"status": "failed", "error": str(e)}] # Filter forbidden commands and store blocked commands info device_configs_list, blocked_commands_map = self._filter_forbidden_commands_from_device_configs( diff --git a/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py b/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py index 31ed0c916..ed33642ef 100644 --- a/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py +++ b/gns3server/agent/gns3_copilot/tools_v2/vpcs_tools_netmiko.py @@ -183,7 +183,11 @@ class VPCSCommands(BaseTool): return device_configs_list # Merge same-device entries before anything keys by device_name - device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="commands") + try: + device_configs_list = merge_duplicate_device_configs(device_configs_list, commands_field="commands") + except ValueError as e: + logger.error("Invalid device_configs input: %s", e) + return [{"status": "failed", "error": str(e)}] # Create a mapping of device names to their commands device_configs_map = self._configs_map(device_configs_list) diff --git a/gns3server/agent/gns3_copilot/utils/device_configs.py b/gns3server/agent/gns3_copilot/utils/device_configs.py index e757eb089..f5ff4575f 100644 --- a/gns3server/agent/gns3_copilot/utils/device_configs.py +++ b/gns3server/agent/gns3_copilot/utils/device_configs.py @@ -34,7 +34,7 @@ from typing import Any def merge_duplicate_device_configs( - device_configs: list[dict[str, Any]], + device_configs: list[Any], commands_field: str, ) -> list[dict[str, Any]]: """ @@ -46,8 +46,9 @@ def merge_duplicate_device_configs( every row would report that same output. Args: - device_configs: Device config entries; each holds device_name and a - command list under commands_field. + device_configs: Device config entries (parsed tool input, validated + here); each holds device_name and a command list under + commands_field. commands_field: Field carrying the commands ("commands" or "config_commands" depending on the tool). @@ -55,12 +56,18 @@ def merge_duplicate_device_configs( One entry per device_name, in first-appearance order. The first entry's extra fields are preserved; later duplicates only contribute their commands. + + Raises: + ValueError: An entry is not an object or has no usable device_name. + Such an entry cannot be addressed; dropping it silently would + let a malformed batch execute partially while every returned + row reports success. """ merged: dict[str, dict[str, Any]] = {} - for config in device_configs: - name = config.get("device_name") - if not name: - continue + for index, config in enumerate(device_configs): + if not isinstance(config, dict) or not config.get("device_name"): + raise ValueError(f"device_configs[{index}] must be an object with a non-empty 'device_name' field") + name = config["device_name"] commands = list(config.get(commands_field, [])) if name in merged: merged[name][commands_field].extend(commands) diff --git a/gns3server/api/routes/compute/ethernet_switch_nodes.py b/gns3server/api/routes/compute/ethernet_switch_nodes.py index ba10182d8..82546f118 100644 --- a/gns3server/api/routes/compute/ethernet_switch_nodes.py +++ b/gns3server/api/routes/compute/ethernet_switch_nodes.py @@ -114,7 +114,16 @@ async def update_ethernet_switch( # place (the setter itself keeps the port-count guard) previous_mapping = [dict(port) for port in node.ports_mapping] node.ports_mapping = data["ports_mapping"] - await node.update_port_settings(previous_mapping) + try: + await node.update_port_settings(previous_mapping) + except Exception: + # Nothing, or only part, of the new mapping was applied — keep + # the old one so the failed change stays visible in GET and a + # retry still diffs against it. A kept-new mapping would turn + # the retry into a no-op that reports success with the VLANs + # never applied. + node.ports_mapping = previous_mapping + raise if "console_type" in data: node.console_type = data["console_type"] node.updated() diff --git a/gns3server/compute/base_node.py b/gns3server/compute/base_node.py index 7ff62ae07..6a85a97c7 100644 --- a/gns3server/compute/base_node.py +++ b/gns3server/compute/base_node.py @@ -1065,6 +1065,10 @@ class BaseNode: # than skipping them as "already installed". self._marker_filter_bridges.clear() self._marker_specs.clear() + # ...and so is the kernel capture, uBridge's single AF_PACKET slot + # (meaningful for the kernel-datapath node types; see + # KernelDatapathMixin._reserve_kernel_capture). + self._kernel_capture_ifc = None async def add_ubridge_udp_connection(self, bridge_name, source_nio, destination_nio): """ @@ -1116,9 +1120,18 @@ class BaseNode: :param filters: Array of filter dictionary """ - # The netem-extension filters (rate, reorder, gemodel…) only exist on - # the kernel datapath. The controller keeps them off relay links; - # this is the second guard for direct compute API use. + self._guard_kernel_only_filters(filters) + await self._ubridge_send("bridge reset_packet_filters " + bridge_name) + for packet_filter in self._build_filter_list(filters): + await self._ubridge_add_packet_filter(f"bridge add_packet_filter {bridge_name} {packet_filter}") + + def _guard_kernel_only_filters(self, filters): + """ + The netem-extension filters (rate, reorder, gemodel…) only exist on + the kernel datapath. The controller keeps them off relay links; this + is the second guard for direct compute API use. + """ + from gns3server.utils.packet_filter_validation import kernel_only_features kernel_only = kernel_only_features(filters) @@ -1127,19 +1140,25 @@ class BaseNode: "Packet filter(s) {} only run on a kernel-datapath link (tc netem on the " "veth host end); the uBridge relay has no equivalent".format(", ".join(sorted(kernel_only))) ) - await self._ubridge_send("bridge reset_packet_filters " + bridge_name) - for packet_filter in self._build_filter_list(filters): - cmd = f"bridge add_packet_filter {bridge_name} {packet_filter}" - try: - await self._ubridge_send(cmd) - except UbridgeError as e: - match = re.search(r"Cannot compile filter '(.*)': syntax error", str(e)) - if match: - message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {match.group(1)}" - log.warning(message) - self.project.emit("log.warning", {"message": message}) - else: - raise + + async def _ubridge_add_packet_filter(self, cmd): + """ + Send one relay filter command, degrading a BPF expression that no + longer compiles on this uBridge's libpcap to a warning instead of + breaking the link (the controller already validated the syntax with + tcpdump at create/update time). + """ + + try: + await self._ubridge_send(cmd) + except UbridgeError as e: + match = re.search(r"Cannot compile filter '(.*)': syntax error", str(e)) + if match: + message = f"Warning: ignoring BPF packet filter '{self.name}' due to syntax error: {match.group(1)}" + log.warning(message) + self.project.emit("log.warning", {"message": message}) + else: + raise def _build_filter_list(self, filters): """ diff --git a/gns3server/compute/builtin/nodes/ethernet_switch.py b/gns3server/compute/builtin/nodes/ethernet_switch.py index e508649c3..6c11a2238 100644 --- a/gns3server/compute/builtin/nodes/ethernet_switch.py +++ b/gns3server/compute/builtin/nodes/ethernet_switch.py @@ -723,6 +723,13 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): previous = {port["port_number"]: port for port in (previous_mapping or [])} wired = dict(self._tap_by_port) for port_number, anchor in self._kernel_ports.items(): + if not os.path.exists(f"/sys/class/net/{anchor}"): + # A kernel port whose peer is stopped keeps its registration + # but has no interface yet (the join is deferred by design): + # vlan commands would fail with ENODEV and abort the whole + # reconcile. Its new settings apply when the peer starts — + # the deferred join runs _apply_port_vlan then. + continue wired[port_number] = anchor for port_number, iface in wired.items(): new_settings = self._port_settings(port_number) @@ -761,7 +768,16 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): anchor = self._kernel_ports.get(port_number) if anchor is not None: - await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + # One capture per uBridge process — see + # _reserve_kernel_capture; a failed start rolls the port's + # flag back so it cannot later stop the winner's capture. + try: + self._reserve_kernel_capture(anchor) + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + except Exception: + self._release_kernel_capture(anchor) + nio.stop_packet_capture() + raise else: ubridge_bridge = self._ubridge_bridge_name(port_number) await self._ubridge_send(f'bridge start_capture {ubridge_bridge} "{output_file}"') @@ -779,8 +795,14 @@ class EthernetSwitch(KernelDatapathMixin, BaseNode): return nio.stop_packet_capture() if self._ubridge_hypervisor and self._ubridge_hypervisor.is_running(): - if port_number in self._kernel_ports: - await self._ubridge_send("capture stop_kernel") + anchor = self._kernel_ports.get(port_number) + if anchor is not None: + # Process-wide and argument-less: only the port owning the + # slot may stop it (a second port's stop would kill this + # capture while its own flag still says capturing). + if self._kernel_capture_owned_by(anchor): + await self._ubridge_send("capture stop_kernel") + self._release_kernel_capture(anchor) else: ubridge_bridge = self._ubridge_bridge_name(port_number) await self._ubridge_send(f"bridge stop_capture {ubridge_bridge}") diff --git a/gns3server/compute/docker/docker_kernel_datapath.py b/gns3server/compute/docker/docker_kernel_datapath.py index 8c8c62b9a..a6300b45d 100644 --- a/gns3server/compute/docker/docker_kernel_datapath.py +++ b/gns3server/compute/docker/docker_kernel_datapath.py @@ -128,11 +128,12 @@ class DockerKernelDatapathMixin(KernelDatapathMixin): await self._ubridge_send(f"docker move_to_ns {guest_ifc} {self._namespace} {ifname}") except UbridgeError as e: raise UbridgeNamespaceError(e) - except UbridgeNamespaceError: - raise except Exception: # Don't leave a half-created pair behind (the next start would - # fail on create_veth EEXIST until the stale sweep runs). + # fail on create_veth EEXIST until the stale sweep runs). The + # namespace failure cleans up the same way — the bare raise + # preserves its type for the caller's UbridgeNamespaceError + # handling (docker_vm.start converts it into a failed start). with contextlib.suppress(UbridgeError): await self._ubridge_send(f'docker delete_veth "{host_ifc}"') raise diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 4cb34d85e..c5cbbc19c 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -1869,6 +1869,14 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): f"Port {port_number} doesn't exist on adapter {adapter_number} of Docker container '{self.name}'" ) + # The compute-side backstop of the controller's duplicate-port guard: + # a port carries at most one NIO, and overwriting the one a link + # still owns orphans that link's teardown (its host state leaks). + if adapter.get_nio(port_number) is not None: + raise DockerError( + f"Port {port_number} on adapter {adapter_number} of Docker container '{self.name}' already has a link" + ) + if self.status == "started" and self.ubridge: await self._connect_nio(adapter_number, nio, port_number) await self._set_adapter_carrier(adapter_number, not nio.suspend, port_number) @@ -1999,11 +2007,18 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): if isinstance(nio, NIOBridge): # Kernel datapath: no relay bridge exists — capture via uBridge's # AF_PACKET module bound to the anchor (a veth host end for the - # standard container, a persistent TAP for an IOL runner). - # Single capture per uBridge process: a concurrent second one - # returns EALREADY. + # standard container, a persistent TAP for an IOL runner). One + # capture per uBridge process: the slot is claimed per anchor, + # and a failed start rolls the port's capturing flag back so the + # port cannot later stop the winner's capture. host_ifc = self._kernel_host_ifc(adapter_number, port_number) - await self._ubridge_send(f'capture start_kernel {host_ifc} "{output_file}"') + try: + self._reserve_kernel_capture(host_ifc) + await self._ubridge_send(f'capture start_kernel {host_ifc} "{output_file}"') + except Exception: + self._release_kernel_capture(host_ifc) + nio.stop_packet_capture() + raise return # Relay datapath (even though the adapter interface is a veth, the # traffic flows through the uBridge bridge — key on the NIO type, @@ -2023,8 +2038,13 @@ class DockerVM(DockerKernelDatapathMixin, BaseNode): raise DockerError("Cannot stop the packet capture: uBridge is not running") nio = self._ethernet_adapters[adapter_number].get_nio(port_number) if isinstance(nio, NIOBridge): - # Idempotent on the uBridge side (no active capture is OK). - await self._ubridge_send("capture stop_kernel") + # Idempotent on the uBridge side (no active capture is OK), but + # process-wide and argument-less: only the port owning the slot + # may stop it, or a second port's stop would kill this capture. + host_ifc = self._kernel_host_ifc(adapter_number, port_number) + if self._kernel_capture_owned_by(host_ifc): + await self._ubridge_send("capture stop_kernel") + self._release_kernel_capture(host_ifc) return bridge_name = self._bridge_name(adapter_number, port_number) await self._ubridge_send(f"bridge stop_capture {bridge_name}") diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index f763d7dd5..de0063d43 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -1486,11 +1486,23 @@ class Router(KernelDatapathMixin, BaseNode): if isinstance(nio, NIOBridge): # Kernel link: capture on the port's TAP anchor (AF_PACKET), # not in the Dynamips NIO (this link has no Dynamips NIO). + if nio.capturing: + raise DynamipsError(f"Packet capture is already activated on port {slot_number}/{port_number}") nio.start_packet_capture(output_file, data_link_type) if self.ubridge: anchor = self._kernel_host_ifc(slot_number, port_number) if anchor is not None: - await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + # One capture per uBridge process — see + # _reserve_kernel_capture; a failed start rolls the + # port's flag back so it cannot later stop the winner's + # capture. + try: + self._reserve_kernel_capture(anchor) + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + except Exception: + self._release_kernel_capture(anchor) + nio.stop_packet_capture() + raise log.debug( f'Router "{self._name}" [{self._id}]: starting packet capture on port {slot_number}/{port_number}' ) @@ -1526,8 +1538,12 @@ class Router(KernelDatapathMixin, BaseNode): if isinstance(nio, NIOBridge): nio.stop_packet_capture() - if self.ubridge and self._kernel_host_ifc(slot_number, port_number) is not None: + anchor = self._kernel_host_ifc(slot_number, port_number) + if self.ubridge and anchor is not None and self._kernel_capture_owned_by(anchor): + # Process-wide and argument-less: only the port owning the + # slot may stop it. await self._ubridge_send("capture stop_kernel") + self._release_kernel_capture(anchor) return await nio.stop_packet_capture() @@ -1720,8 +1736,21 @@ class Router(KernelDatapathMixin, BaseNode): raise await self.slot_enable_nio(slot_number, port_number) self._tap_nios[(slot_number, port_number)] = tap_nio - await self._kernel_attach(anchor, nio) - await self._set_adapter_carrier(slot_number, not nio.suspend, port_number) + try: + await self._kernel_attach(anchor, nio) + await self._set_adapter_carrier(slot_number, not nio.suspend, port_number) + except Exception: + # Undo the half-attach: while the _tap_nios entry exists the port + # guard above short-circuits every retry, and a link delete reads + # the still-unbound adapter as "nothing to do" — the port would + # stay a silent black hole until the router restarts. The undo + # tolerates the partial state (its marker/bridge steps are + # best-effort) and takes the hypervisor binding and the TAP with + # it, so a retry starts from scratch. It is best-effort here as + # well: the original failure is what must surface. + with contextlib.suppress(Exception): + await self._remove_kernel_nio_binding(slot_number, port_number, nio) + raise async def _remove_kernel_nio_binding(self, slot_number, port_number, nio): """ diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index d3cd01dfc..aad00f0bc 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -686,74 +686,87 @@ class IOUVM(KernelDatapathMixin, BaseNode): raise IOUError(f"The iourc path '{iourc_path}' is not a regular file") await self._check_iou_license() - await self._start_ubridge() - await self._prepare_tap_datapath() - self._create_netmap_config() - if self.use_default_iou_values: - # make sure we have the default nvram amount to correctly push the configs - await self.update_default_iou_values() - self._push_configs_to_nvram() - - # check if there is enough RAM to run - self.check_available_ram(self.ram) - - self._nvram_watcher = FileWatcher(self._nvram_file(), self._nvram_changed, delay=2) - - # created an environment variable pointing to the iourc file. - env = os.environ.copy() - if "IOURC" not in os.environ and iourc_path: - env["IOURC"] = iourc_path - - # create a symbolic link to the image to avoid IOU error "failed code signing checks" - # on newer images, see https://github.com/GNS3/gns3-server/issues/1484 + # The try starts before the uBridge spawn: a raise anywhere from + # here to the launch (tap datapath, netmap config, symlink, + # command build) must take the datapath down again — a node that + # reports a failed start must not keep a uBridge process and its + # anchor TAPs alive, since nothing else ever reclaims them. try: - iou_image_path = os.path.basename(self.path) - if len(iou_image_path) > 63: - # IOU file basename length must be <= 63 chars - iou_file_name, iou_file_ext = os.path.splitext(iou_image_path) - iou_image_path = iou_file_name[: 63 - len(iou_file_ext)] + iou_file_ext - symlink = os.path.join(self.working_dir, iou_image_path) - if os.path.islink(symlink): - os.unlink(symlink) - os.symlink(self.path, symlink) - except OSError as e: - raise IOUError(f"Could not create symbolic link: {e}") + await self._start_ubridge() + await self._prepare_tap_datapath() + self._create_netmap_config() + if self.use_default_iou_values: + # make sure we have the default nvram amount to correctly push the configs + await self.update_default_iou_values() + self._push_configs_to_nvram() - command = await self._build_command() - # Only start the responder when the capability probe actually - # enabled IOU's L1 protocol on the command line. - if "-l" in command: - await self._start_l1_keepalive_responder() - try: - if self._loader: - log.debug(f"Starting IOU: {command} with loader {self._loader}") - else: - log.debug(f"Starting IOU: {command}") - self.command_line = " ".join(command) - self._iou_process = await asyncio.create_subprocess_exec( - *self._loader, - *command, - stdout=asyncio.subprocess.PIPE, - stdin=asyncio.subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=self.working_dir, - env=env, - ) - log.debug(f"IOU instance {self._id} started PID={self._iou_process.pid}") - self._started = True - self.status = "started" - callback = functools.partial(self._termination_callback, "IOU") - gns3server.utils.asyncio.monitor_process(self._iou_process, callback) - except FileNotFoundError as e: - self._stop_l1_keepalive_responder() - raise IOUError( - f"Could not start IOU: {e}: 32-bit binary support is probably not installed, it is recommended to use a 64-bit image instead" - ) - except (OSError, subprocess.SubprocessError) as e: - self._stop_l1_keepalive_responder() - iou_stdout = self.read_iou_stdout() - log.error(f"Could not start IOU {self._path}: {e}\n{iou_stdout}") - raise IOUError(f"Could not start IOU {self._path}: {e}\n{iou_stdout}") + # check if there is enough RAM to run + self.check_available_ram(self.ram) + + self._nvram_watcher = FileWatcher(self._nvram_file(), self._nvram_changed, delay=2) + + # created an environment variable pointing to the iourc file. + env = os.environ.copy() + if "IOURC" not in os.environ and iourc_path: + env["IOURC"] = iourc_path + + # create a symbolic link to the image to avoid IOU error "failed code signing checks" + # on newer images, see https://github.com/GNS3/gns3-server/issues/1484 + try: + iou_image_path = os.path.basename(self.path) + if len(iou_image_path) > 63: + # IOU file basename length must be <= 63 chars + iou_file_name, iou_file_ext = os.path.splitext(iou_image_path) + iou_image_path = iou_file_name[: 63 - len(iou_file_ext)] + iou_file_ext + symlink = os.path.join(self.working_dir, iou_image_path) + if os.path.islink(symlink): + os.unlink(symlink) + os.symlink(self.path, symlink) + except OSError as e: + raise IOUError(f"Could not create symbolic link: {e}") + + command = await self._build_command() + # Only start the responder when the capability probe actually + # enabled IOU's L1 protocol on the command line. + if "-l" in command: + await self._start_l1_keepalive_responder() + try: + if self._loader: + log.debug(f"Starting IOU: {command} with loader {self._loader}") + else: + log.debug(f"Starting IOU: {command}") + self.command_line = " ".join(command) + self._iou_process = await asyncio.create_subprocess_exec( + *self._loader, + *command, + stdout=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=self.working_dir, + env=env, + ) + log.debug(f"IOU instance {self._id} started PID={self._iou_process.pid}") + self._started = True + self.status = "started" + callback = functools.partial(self._termination_callback, "IOU") + gns3server.utils.asyncio.monitor_process(self._iou_process, callback) + except FileNotFoundError as e: + self._stop_l1_keepalive_responder() + raise IOUError( + f"Could not start IOU: {e}: 32-bit binary support is probably not installed, it is recommended to use a 64-bit image instead" + ) + except (OSError, subprocess.SubprocessError) as e: + self._stop_l1_keepalive_responder() + iou_stdout = self.read_iou_stdout() + log.error(f"Could not start IOU {self._path}: {e}\n{iou_stdout}") + raise IOUError(f"Could not start IOU {self._path}: {e}\n{iou_stdout}") + except Exception: + # Re-raise unchanged after the teardown so callers keep the + # original error contract. The L1 responder needs no stop + # here: _start_l1_keepalive_responder cleans up after itself + # and everything after it runs under handlers that stop it. + await self._stop_ubridge() + raise await self.start_console() @@ -1394,10 +1407,14 @@ class IOUVM(KernelDatapathMixin, BaseNode): :param filters: Array of filter dictionnary """ + # The base relay path's guards apply here too: kernel-only filters + # must be a clear NodeError (a direct compute API update would + # otherwise reach iol_bridge and fail there), and a BPF line that no + # longer compiles degrades to a warning instead of a hard error. + self._guard_kernel_only_filters(filters) await self._ubridge_send("iol_bridge reset_packet_filters " + location) for filter in self._build_filter_list(filters): - cmd = f"iol_bridge add_packet_filter {location} {filter}" - await self._ubridge_send(cmd) + await self._ubridge_add_packet_filter(f"iol_bridge add_packet_filter {location} {filter}") async def _ubridge_add_marker_filter( self, location, name, bpf, pcap_path, tag=None, link_id=None, direction=None, data_link_type=None @@ -1826,13 +1843,22 @@ class IOUVM(KernelDatapathMixin, BaseNode): if self.ubridge: if isinstance(nio, NIOBridge): - # Kernel link: capture on the port's TAP anchor (AF_PACKET). + # Kernel link: capture on the port's TAP anchor (AF_PACKET, + # one capture per uBridge process — see _reserve_kernel_capture; + # a failed start rolls the port's flag back so the port cannot + # later stop the winner's capture). anchor = self._kernel_host_ifc(adapter_number, port_number) - if anchor is None: - raise self._kernel_error( - f"Bay {adapter_number}/{port_number} of IOU '{self._name}' has no TAP anchor to capture on" - ) - await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + try: + if anchor is None: + raise self._kernel_error( + f"Bay {adapter_number}/{port_number} of IOU '{self._name}' has no TAP anchor to capture on" + ) + self._reserve_kernel_capture(anchor) + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + except Exception: + self._release_kernel_capture(anchor) + nio.stop_packet_capture() + raise else: await self._ubridge_send( 'iol_bridge start_capture {name} {bay} {unit} "{output_file}" {data_link_type}'.format( @@ -1859,7 +1885,13 @@ class IOUVM(KernelDatapathMixin, BaseNode): log.debug(f'IOU "{self._name}" [{self._id}]: stopping packet capture on {adapter_number}/{port_number}') if self.ubridge: if isinstance(nio, NIOBridge): - await self._ubridge_send("capture stop_kernel") + # Process-wide and argument-less: only the port owning the + # slot may stop it (a second port's stop would kill this + # capture while its own flag still says capturing). + anchor = self._kernel_host_ifc(adapter_number, port_number) + if self._kernel_capture_owned_by(anchor): + await self._ubridge_send("capture stop_kernel") + self._release_kernel_capture(anchor) else: await self._ubridge_send( f"iol_bridge stop_capture {self._iol_bridge_name()} {adapter_number} {port_number}" diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index 4a23c2b51..fd5fced43 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -149,6 +149,52 @@ class KernelDatapathMixin: return not (isinstance(nio, NIOBridge) and nio.bridge is None) + # The anchor whose kernel capture currently holds the uBridge process's + # single AF_PACKET capture slot (None = free; see _reserve_kernel_capture). + # Class-level default so every node type gets it without touching its + # __init__; BaseNode._stop_ubridge resets it when uBridge goes away. + _kernel_capture_ifc = None + + def _reserve_kernel_capture(self, anchor): + """ + Claim uBridge's single kernel-capture slot for *anchor*. + + uBridge serves one AF_PACKET capture per process (a static slot: a + second ``capture start_kernel`` answers EALREADY) while the server + models captures per port — a second kernel port of the same node is + refused here with a clear error. Without the claim the loser's port + would stay marked as capturing, and its stop would issue the + process-wide, argument-less ``capture stop_kernel``, killing the + winner's capture while its own flag still says capturing. + """ + + if self._kernel_capture_ifc is not None and self._kernel_capture_ifc != anchor: + raise self._kernel_error( + f"Cannot start the packet capture: this node's uBridge already captures " + f"{self._kernel_capture_ifc} (one kernel capture per node); stop that capture first" + ) + self._kernel_capture_ifc = anchor + + def _release_kernel_capture(self, anchor): + """ + Release the slot if *anchor* holds it: after a stop, or after a + failed start whose port must not stay marked as capturing. + """ + + if self._kernel_capture_ifc == anchor: + self._kernel_capture_ifc = None + + def _kernel_capture_owned_by(self, anchor): + """ + Whether *anchor* may issue the process-wide ``capture stop_kernel``: + only the port holding the slot — or a node whose slot is empty (a + capture predating the tracker), where stopping is the historical + best-effort behaviour. A second port must never reach it: the + command takes no interface and would stop the owner's capture. + """ + + return self._kernel_capture_ifc is None or self._kernel_capture_ifc == anchor + async def _kernel_attach(self, anchor, nio): """ Attach an anchor to the per-link kernel bridge carried by *nio* @@ -181,7 +227,10 @@ class KernelDatapathMixin: await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"') if nio.capturing: # Restore a capture that was active before a node restart - # (mirrors the relay path's start_capture). + # (mirrors the relay path's start_capture). The slot is claimed + # like a fresh start: a second kernel port's replay is refused + # with a clear error instead of an EALREADY deep in the wiring. + self._reserve_kernel_capture(anchor) await self._ubridge_send(f'capture start_kernel {anchor} "{nio.pcap_output_file}"') # Markers carried by the NIO attach to the anchor (AF_PACKET taps) — # the anchor is the interface, not a relay bridge. @@ -404,40 +453,34 @@ class KernelDatapathMixin: # ------------------------------------------------------------------ @staticmethod - def _netem_command_parts(filters): + def _values_of(filters, key): + """ + One filter's values as a list, tolerating the legacy bare-value + shape ({"packet_loss": 10}). """ - Build the ``tc netem set`` keyword sequence (in the frozen grammar - order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, - limit, distribution, seed) plus the set of capability tokens the - sequence needs beyond the original netem surface. Original-surface - keywords (delay, jitter, loss, corrupt, dup) need no probe; the - extension tokens (rate, reorder, gemodel, dist, seed, limit) are - checked against ``tc capabilities`` by the caller. ``correl`` has no - token of its own in the capabilities list, so it gates on "rate" as - the netem-extension build marker. - :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) + values = filters.get(key) + if isinstance(values, (list, tuple)): + return list(values) + return [values] if values else [] + + @staticmethod + def _netem_loss_parts(filters): + """ + The loss keyword: gemodel when set, else packet_loss — mutually + exclusive, both mapping to netem's single loss argument. The gemodel + token is returned for the caller's capability check; a non-zero + packet-loss correlation gates on "rate" as the extension build + marker. :returns: (keyword sequence, required extension tokens) """ - def values_of(key): - # tolerate the legacy bare-value shape ({"packet_loss": 10}) - values = filters.get(key) - if isinstance(values, (list, tuple)): - return list(values) - return [values] if values else [] - + values_of = KernelDatapathMixin._values_of parts = [] ext_tokens = set() - delay = values_of("delay") - if delay: - parts.append(f"delay {int(delay[0])}") - if len(delay) > 1 and int(delay[1]): - parts.append(f"jitter {int(delay[1])}") - - gemodel = values_of("gemodel") + gemodel = values_of(filters, "gemodel") if gemodel: segment = f"loss gemodel {int(gemodel[0])}" if len(gemodel) > 1: @@ -446,26 +489,71 @@ class KernelDatapathMixin: segment += f" {int(gemodel[2])}" parts.append(segment) ext_tokens.add("gemodel") - else: - loss = values_of("packet_loss") - if loss and int(loss[0]): - parts.append(f"loss {int(loss[0])}") - if len(loss) > 1 and int(loss[1]): - parts.append(f"correl {int(loss[1])}") - ext_tokens.add("rate") # correl: extension build marker + return parts, ext_tokens - duplicate = values_of("duplicate") + loss = values_of(filters, "packet_loss") + if loss and int(loss[0]): + parts.append(f"loss {int(loss[0])}") + if len(loss) > 1 and int(loss[1]): + parts.append(f"correl {int(loss[1])}") + ext_tokens.add("rate") # correl: extension build marker + return parts, ext_tokens + + @staticmethod + def _netem_surface_parts(filters): + """ + The original netem surface in its frozen grammar order — delay/ + jitter, loss (or gemodel), dup (+correl), corrupt — plus the + capability tokens it needs. Original-surface keywords need no + probe; ``correl`` has no token of its own in the capabilities + list, so it gates on "rate" as the netem-extension build marker. + + :returns: (keyword sequence, required extension tokens) + """ + + values_of = KernelDatapathMixin._values_of + parts = [] + ext_tokens = set() + + delay = values_of(filters, "delay") + if delay: + parts.append(f"delay {int(delay[0])}") + if len(delay) > 1 and int(delay[1]): + parts.append(f"jitter {int(delay[1])}") + + loss_parts, loss_tokens = KernelDatapathMixin._netem_loss_parts(filters) + parts.extend(loss_parts) + ext_tokens |= loss_tokens + + duplicate = values_of(filters, "duplicate") if duplicate and int(duplicate[0]): parts.append(f"dup {int(duplicate[0])}") if len(duplicate) > 1 and int(duplicate[1]): parts.append(f"correl {int(duplicate[1])}") ext_tokens.add("rate") # correl: extension build marker - corrupt = values_of("corrupt") + corrupt = values_of(filters, "corrupt") if corrupt and int(corrupt[0]): parts.append(f"corrupt {int(corrupt[0])}") - reorder = values_of("reorder") + return parts, ext_tokens + + @staticmethod + def _netem_extension_parts(filters): + """ + The netem extensions in their frozen grammar order — reorder, rate, + limit, distribution, seed — plus each keyword's capability token. + The tokens (rate, reorder, dist, seed, limit) are checked against + ``tc capabilities`` by the caller. + + :returns: (keyword sequence, required extension tokens) + """ + + values_of = KernelDatapathMixin._values_of + parts = [] + ext_tokens = set() + + reorder = values_of(filters, "reorder") if reorder: segment = f"reorder {int(reorder[0])}" if len(reorder) > 1 and int(reorder[1]): @@ -475,23 +563,24 @@ class KernelDatapathMixin: parts.append(segment) ext_tokens.add("reorder") - rate = values_of("rate") + rate = values_of(filters, "rate") if rate and str(rate[0]).strip(): parts.append(f"rate {str(rate[0]).strip()}") ext_tokens.add("rate") - limit = values_of("limit") + limit = values_of(filters, "limit") if limit and int(limit[0]): parts.append(f"limit {int(limit[0])}") ext_tokens.add("limit") + delay = values_of(filters, "delay") if delay and len(delay) > 2 and str(delay[2]).strip().lower() not in ("", "uniform"): # "uniform" is the kernel default — emitting nothing keeps the # command compatible with the original netem surface. parts.append(f"distribution {str(delay[2]).strip().lower()}") ext_tokens.add("dist") - seed = values_of("seed") + seed = values_of(filters, "seed") if seed and int(seed[0]): # 0 is the "disabled" convention everywhere else — treat it so # here too (the controller's inactive-filter pass drops it). @@ -500,6 +589,25 @@ class KernelDatapathMixin: return parts, ext_tokens + @staticmethod + def _netem_command_parts(filters): + """ + Build the ``tc netem set`` keyword sequence (in the frozen grammar + order: delay/jitter, loss|gemodel, dup, corrupt, reorder, rate, + limit, distribution, seed) plus the set of capability tokens the + sequence needs beyond the original netem surface. + + :param filters: NIO filters dictionary ({"delay": [ms, jitter, dist], ...}) + + :returns: (keyword sequence, required extension tokens) + """ + + parts, ext_tokens = KernelDatapathMixin._netem_surface_parts(filters) + extension_parts, extension_tokens = KernelDatapathMixin._netem_extension_parts(filters) + parts.extend(extension_parts) + ext_tokens |= extension_tokens + return parts, ext_tokens + async def _ubridge_apply_netem(self, host_ifc, filters): """ Translate the NIO's impairment filters into one tc netem qdisc on the @@ -523,7 +631,12 @@ class KernelDatapathMixin: :param filters: NIO filters dictionary ({"delay": [ms, jitter], ...}) """ - parts, ext_tokens = self._netem_command_parts(filters) + try: + parts, ext_tokens = self._netem_command_parts(filters) + except (TypeError, ValueError) as e: + # int() on a stray value from an unvalidated direct-API update + # (e.g. {"delay": ["abc"]}) would otherwise surface as a 500. + raise self._kernel_error(f"Malformed packet filter values on a kernel-datapath link: {e}") if parts and ext_tokens: # Extension keywords need the netem-extension uBridge. Plain # delay/loss/corrupt/dup never probes — an old uBridge serves @@ -587,7 +700,13 @@ class KernelDatapathMixin: """ bpf = filters.get("bpf") - lines = [line.strip() for line in (bpf[0].split("\n") if bpf else []) if line.strip()] + # Accept the bare-value shape ({"bpf": "icmp"}): a str is the + # expression itself — indexing it would install its first character. + if isinstance(bpf, str): + bpf = [bpf] + elif not isinstance(bpf, (list, tuple)): + bpf = [] + lines = [line.strip() for line in (str(bpf[0]).split("\n") if bpf else []) if line.strip()] if not lines and self._ubridge_tc_caps is None: # Nothing to add and nothing was ever installed (no tc module # probed yet) — skip touching an old uBridge entirely. @@ -653,16 +772,32 @@ class KernelDatapathMixin: :param filters: NIO filters dictionary """ - def values_of(key): - values = filters.get(key) - if isinstance(values, (list, tuple)): - return list(values) - return [values] if values else [] + def numeric_values(key, minimum, maximum): + """ + The filter's values as ints, refusing a malformed shape here: the + direct compute API is unvalidated (the controller's validation + does not run for it), and indexing a short list or int()-ing a + stray string further down would surface as a bare IndexError or + ValueError 500 after the bridge/netem state was already applied. + """ - frequency = values_of("frequency_drop") - quota = values_of("quota") - window = values_of("window_drop") - requested = {filter_type: values_of(filter_type) for filter_type in FILTER_EBPF_MODES} + values = self._values_of(filters, key) + if not values: + return [] + try: + numbers = [int(value) for value in values] + except (TypeError, ValueError): + raise self._kernel_error(f"Packet filter '{key}' needs integer values, got {values!r}") + if not minimum <= len(numbers) <= maximum: + raise self._kernel_error( + f"Packet filter '{key}' needs between {minimum} and {maximum} integer values, got {len(numbers)}" + ) + return numbers + + frequency = numeric_values("frequency_drop", 1, 1) + quota = numeric_values("quota", 2, 2) + window = numeric_values("window_drop", 3, 5) + requested = {filter_type: self._values_of(filters, filter_type) for filter_type in FILTER_EBPF_MODES} if not any(requested.values()) and self._ubridge_tc_caps is None: # Nothing to set and nothing was ever installed (no probe yet) — # skip touching an old uBridge entirely. @@ -672,41 +807,67 @@ class KernelDatapathMixin: # (ebpf_modes); builds predating the field keep their shipped modes. modes = usable_ebpf_modes(caps) if not any(requested.values()): - # Only modes this build declares can ever have been installed — - # sending "off" for an undeclared mode would hit an unknown - # command; "off" is idempotent in uBridge. - for filter_type, mode in FILTER_EBPF_MODES.items(): - if mode in modes: - await self._ubridge_send(f'tc {mode}_drop "{host_ifc}" off') + await self._ebpf_turn_off_modes(host_ifc, modes) return unsupported = sorted(f for f, values in requested.items() if values and FILTER_EBPF_MODES[f] not in modes) if unsupported: - if not modes: - raise self._kernel_error( - "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " - "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " - "the uBridge binary); upgrade uBridge on this compute or keep the link on the " - "relay datapath".format(", ".join(unsupported)) - ) + self._reject_unsupported_ebpf_modes(unsupported, modes) + await self._ebpf_apply_modes(host_ifc, frequency, quota, window, modes) + + async def _ebpf_turn_off_modes(self, host_ifc, modes): + """ + Turn off every mode this build declares. Only declared modes can + ever have been installed — sending "off" for an undeclared mode + would hit an unknown command; "off" is idempotent in uBridge. + """ + + for mode in FILTER_EBPF_MODES.values(): + if mode in modes: + await self._ubridge_send(f'tc {mode}_drop "{host_ifc}" off') + + def _reject_unsupported_ebpf_modes(self, unsupported, modes): + """ + Refuse filters whose mode this uBridge cannot run: a clear error + naming the missing capability (no eBPF at all vs. an older build) + beats a late uBridge failure. + """ + + if not modes: raise self._kernel_error( - "Packet filter(s) {} on a kernel-datapath link need a newer uBridge (tc capabilities " - "reports ebpf_modes={}); upgrade uBridge on this compute or keep the link on the " - "relay datapath".format(", ".join(unsupported), ",".join(modes)) + "Packet filter(s) {} on a kernel-datapath link need a uBridge with eBPF support " + "(tc capabilities reports no ebpf; setcap cap_bpf,cap_net_admin,cap_net_raw=ep on " + "the uBridge binary); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(unsupported)) ) + raise self._kernel_error( + "Packet filter(s) {} on a kernel-datapath link need a newer uBridge (tc capabilities " + "reports ebpf_modes={}); upgrade uBridge on this compute or keep the link on the " + "relay datapath".format(", ".join(unsupported), ",".join(modes)) + ) + + async def _ebpf_apply_modes(self, host_ifc, frequency, quota, window, modes): + """ + Set the requested modes (and clear the absent ones) in the + classifier's fixed order nth → quota → window. A mode left absent + is turned off explicitly (idempotent) so a removed filter stops + dropping; for window_drop this means the schedule restarts on every + apply (the first window opens Start-ms from the moment of the call). + """ + if frequency: # relay semantics: -1 = drop everything, N = every Nth packet - nth = 1 if int(frequency[0]) == -1 else int(frequency[0]) + nth = 1 if frequency[0] == -1 else frequency[0] await self._ubridge_send(f'tc nth_drop "{host_ifc}" {nth}') elif "nth" in modes: await self._ubridge_send(f'tc nth_drop "{host_ifc}" off') if quota: - await self._ubridge_send(f'tc quota_drop "{host_ifc}" {int(quota[0])} {int(quota[1])}') + await self._ubridge_send(f'tc quota_drop "{host_ifc}" {quota[0]} {quota[1]}') elif "quota" in modes: await self._ubridge_send(f'tc quota_drop "{host_ifc}" off') if window: # [start, outage, chance, period?, jitter?] — start is relative # to this apply (the schedule restarts on every reconcile). - params = " ".join(str(int(v)) for v in window) + params = " ".join(str(v) for v in window) await self._ubridge_send(f'tc window_drop "{host_ifc}" {params}') elif "window" in modes: await self._ubridge_send(f'tc window_drop "{host_ifc}" off') diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index db842f781..37485fc53 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1132,13 +1132,20 @@ class QemuVM(KernelDatapathMixin, BaseNode): # created — and before QEMU opens them. Unlike a container, whose # veths are created after it starts, the TAP has to pre-exist the # process that reads it. + # + # The try starts here rather than at the launch: every step up to + # the launch can fail (a broken qemu_path in _build_command, a + # tap create refused), and each of those failures must take the + # datapath down again — a node that reports stopped must not keep + # a uBridge process and root-namespace TAPs alive, since nothing + # else ever reclaims them. self._ubridge_tc_caps = None - await self._start_ubridge() - await self._prepare_tap_datapath() - - command = await self._build_command() - command_string = " ".join(shlex.quote(s) for s in command) try: + await self._start_ubridge() + await self._prepare_tap_datapath() + + command = await self._build_command() + command_string = " ".join(shlex.quote(s) for s in command) log.debug(f"Starting QEMU with: {command_string}") self._stdout_file = os.path.join(self.working_dir, "qemu.log") log.debug(f"logging to {self._stdout_file}") @@ -1159,6 +1166,12 @@ class QemuVM(KernelDatapathMixin, BaseNode): # the launch: don't leave either behind on a failed start. await self._stop_ubridge() raise QemuError(f"Could not start QEMU {self.qemu_path}: {e}\n{stdout}") + except Exception: + # The same teardown for a failure raised as anything else + # (QemuError from _build_command, UbridgeError from the tap + # setup) — re-raised unchanged so callers keep their contract. + await self._stop_ubridge() + raise await self._set_process_priority() if self._cpu_throttling: @@ -1238,60 +1251,63 @@ class QemuVM(KernelDatapathMixin, BaseNode): async with self._execute_lock: # stop the QEMU process self._hw_virtualization = False - if self.is_running(): - log.debug(f'Stopping QEMU VM "{self._name}" PID={self._process.pid}') - try: - if self.on_close == "save_vm_state": - await self._control_vm("stop") - await self._control_vm("savevm GNS3_SAVED_STATE") - wait_for_savevm = 120 - while wait_for_savevm: - await asyncio.sleep(1) - status = await self._saved_state_option() - wait_for_savevm -= 1 - if status != []: - break + try: + if self.is_running(): + log.debug(f'Stopping QEMU VM "{self._name}" PID={self._process.pid}') + try: + if self.on_close == "save_vm_state": + await self._control_vm("stop") + await self._control_vm("savevm GNS3_SAVED_STATE") + wait_for_savevm = 120 + while wait_for_savevm: + await asyncio.sleep(1) + status = await self._saved_state_option() + wait_for_savevm -= 1 + if status != []: + break - if self.on_close == "shutdown_signal": - await self._control_vm("system_powerdown") - await gns3server.utils.asyncio.wait_for_process_termination(self._process, timeout=120) - else: - self._process.terminate() - await gns3server.utils.asyncio.wait_for_process_termination(self._process, timeout=3) - except ProcessLookupError: - pass - except asyncio.TimeoutError: - if self._process: - try: - self._process.kill() - except ProcessLookupError: - pass - if self._process.returncode is None: - log.warning(f'QEMU VM "{self._name}" PID={self._process.pid} is still running') - self._process = None - self._stop_cpulimit() - self._stop_swtpm() - if self.on_close != "save_vm_state": - await self._clear_save_vm_stated() - await self._export_config() - await super().stop() - - # Release the datapath only once QEMU is gone: the process holds - # the adapter TAPs open (it is the guest side), and uBridge's tap - # delete refuses a held device ("Device or resource busy") — - # deleting them while QEMU still ran leaked the persistent TAPs, - # because that best-effort delete is suppressed. Order: process - # side first, then the devices (the same lesson as IOU's reverse - # stop order). - # - # It stays under the lock: the process monitor calls stop() on its - # own when QEMU dies, so an API stop and a process-death stop run - # concurrently in the normal case — and the TAP sweep walks the - # adapter map across awaits, which the second stop was free to - # clear underneath it ("dictionary changed size during - # iteration"). Serialized, it finds uBridge already gone and - # sweeps nothing. - await self._stop_ubridge() + if self.on_close == "shutdown_signal": + await self._control_vm("system_powerdown") + await gns3server.utils.asyncio.wait_for_process_termination(self._process, timeout=120) + else: + self._process.terminate() + await gns3server.utils.asyncio.wait_for_process_termination(self._process, timeout=3) + except ProcessLookupError: + pass + except asyncio.TimeoutError: + if self._process: + try: + self._process.kill() + except ProcessLookupError: + pass + if self._process.returncode is None: + log.warning(f'QEMU VM "{self._name}" PID={self._process.pid} is still running') + self._process = None + self._stop_cpulimit() + self._stop_swtpm() + if self.on_close != "save_vm_state": + await self._clear_save_vm_stated() + await self._export_config() + await super().stop() + finally: + # Release the datapath once QEMU is gone: the process holds + # the adapter TAPs open (it is the guest side), and uBridge's + # tap delete refuses a held device ("Device or resource + # busy") — deleting them while QEMU still ran leaked the + # persistent TAPs, because that best-effort delete is + # suppressed. Order: process side first, then the devices + # (the same lesson as IOU's reverse stop order). The finally + # keeps a mid-stop raise (a failed savevm, an export error) + # from skipping the teardown entirely. + # + # It stays under the lock: the process monitor calls stop() on + # its own when QEMU dies, so an API stop and a process-death + # stop run concurrently in the normal case — and the TAP + # sweep walks the adapter map across awaits, which the second + # stop was free to clear underneath it ("dictionary changed + # size during iteration"). Serialized, it finds uBridge + # already gone and sweeps nothing. + await self._stop_ubridge() async def _open_qemu_monitor_connection_vm(self, timeout=10): """ @@ -1665,6 +1681,12 @@ class QemuVM(KernelDatapathMixin, BaseNode): except IndexError: raise QemuError(f'Adapter {adapter_number} does not exist on QEMU VM "{self._name}"') + # The compute-side backstop of the controller's duplicate-port guard: + # a port carries at most one NIO, and overwriting the one a link + # still owns orphans that link's teardown (its host state leaks). + if adapter.get_nio(0) is not None: + raise QemuError(f'Adapter {adapter_number} of QEMU VM "{self._name}" already has a link') + if self.is_running(): try: await self._connect_nio(adapter_number, nio) @@ -1777,8 +1799,17 @@ class QemuVM(KernelDatapathMixin, BaseNode): if self.ubridge: anchor = self._kernel_host_ifc(adapter_number) if isinstance(nio, NIOBridge) and anchor is not None: - # Kernel link: capture on the adapter's TAP anchor (AF_PACKET). - await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + # Kernel link: capture on the adapter's TAP anchor (AF_PACKET, + # one capture per uBridge process — see _reserve_kernel_capture; + # a failed start rolls the port's flag back so the port cannot + # later stop the winner's capture). + try: + self._reserve_kernel_capture(anchor) + await self._ubridge_send(f'capture start_kernel {anchor} "{output_file}"') + except Exception: + self._release_kernel_capture(anchor) + nio.stop_packet_capture() + raise else: await self._ubridge_send( 'bridge start_capture {name} "{output_file}"'.format( @@ -1801,8 +1832,14 @@ class QemuVM(KernelDatapathMixin, BaseNode): nio.stop_packet_capture() if self.ubridge: - if isinstance(nio, NIOBridge) and self._kernel_host_ifc(adapter_number) is not None: - await self._ubridge_send("capture stop_kernel") + anchor = self._kernel_host_ifc(adapter_number) + if isinstance(nio, NIOBridge) and anchor is not None: + # Process-wide and argument-less: only the port owning the + # slot may stop it (a second port's stop would kill this + # capture while its own flag still says capturing). + if self._kernel_capture_owned_by(anchor): + await self._ubridge_send("capture stop_kernel") + self._release_kernel_capture(anchor) else: await self._ubridge_send("bridge stop_capture {name}".format(name=f"QEMU-{self._id}-{adapter_number}")) diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py index fdff74d6d..4322a53ee 100644 --- a/gns3server/compute/ubridge/tc_probe.py +++ b/gns3server/compute/ubridge/tc_probe.py @@ -49,14 +49,16 @@ from .ubridge_error import UbridgeError log = logging.getLogger(__name__) -# binary identity -> parsed report (None = probed and unusable) -_cache: dict[tuple[str, int, int], Optional[dict[str, str]]] = {} +# binary identity (path, mtime, size, ctime — ctime catches chmod/setcap, +# which change neither mtime nor size) -> parsed report (None = probed and +# unusable) +_cache: dict[tuple[str, int, int, int], Optional[dict[str, str]]] = {} # binary identity -> whether the tap module works (None = unknown) -_tap_cache: dict[tuple[str, int, int], Optional[bool]] = {} +_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {} # binary identity -> whether iol_bridge can bind a port to a TAP (None = unknown) -_iol_tap_cache: dict[tuple[str, int, int], Optional[bool]] = {} +_iol_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {} # binary identity -> whether the bridge module can release a named TAP NIO (None = unknown) -_bridge_tap_cache: dict[tuple[str, int, int], Optional[bool]] = {} +_bridge_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {} _lock = asyncio.Lock() @@ -74,7 +76,7 @@ async def probe_tc_capabilities(timeout: float = 15.0): return None try: stat = os.stat(path) - key = (path, stat.st_mtime_ns, stat.st_size) + key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns) except OSError: return None @@ -137,7 +139,7 @@ async def probe_tap_support(timeout: float = 15.0): return None try: stat = os.stat(path) - key = (path, stat.st_mtime_ns, stat.st_size) + key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns) except OSError: return None @@ -215,7 +217,7 @@ async def probe_iol_tap_support(timeout: float = 15.0): return None try: stat = os.stat(path) - key = (path, stat.st_mtime_ns, stat.st_size) + key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns) except OSError: return None @@ -297,7 +299,7 @@ async def probe_bridge_tap_support(timeout: float = 15.0): return None try: stat = os.stat(path) - key = (path, stat.st_mtime_ns, stat.st_size) + key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns) except OSError: return None @@ -346,7 +348,10 @@ async def _probe_bridge_tap(path, config, timeout): try: return await asyncio.wait_for(ask(), timeout=timeout) - except (OSError, asyncio.TimeoutError, ValueError) as e: + except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e: + # UbridgeError also covers hypervisor.start()/connect() failing + # (unlike the siblings, ask() re-raises those): an unusable + # binary means unknown, never a failed /capabilities response. log.debug("uBridge bridge-tap probe failed: %s", e) return None finally: diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 909792ac8..6d2fa355e 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -803,14 +803,18 @@ class Link: if not reported: return True for caps in reported: + # Field-wise .get: the controller stores the compute's raw + # capabilities verbatim, so a divergent or hand-crafted payload + # with a partial ubridge_tc dict must read as "not supported" + # rather than raising a KeyError into the API response. if filter_type in FILTER_EBPF_MODES: - if FILTER_EBPF_MODES[filter_type] not in caps["ebpf_modes"]: + if FILTER_EBPF_MODES[filter_type] not in (caps.get("ebpf_modes") or []): return False elif filter_type == "bpf": - if not caps["cbpf"]: + if not caps.get("cbpf"): return False elif filter_type in FILTER_NETEM_KEYWORDS: - if FILTER_NETEM_KEYWORDS[filter_type] not in caps["netem"]: + if FILTER_NETEM_KEYWORDS[filter_type] not in (caps.get("netem") or []): return False # the remaining types (delay, packet_loss, corrupt) run on any # build that reports tc capabilities at all diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 8e607f1b5..925cbd569 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -70,7 +70,17 @@ def _is_unix_socket_docker(node): """ environment = (node.properties or {}).get("environment") or "" - return "GNS3_UNIX_SOCKET_NIO" in environment and not is_iol_runner_environment(environment) + if is_iol_runner_environment(environment): + return False + for line in environment.splitlines(): + line = line.strip().rstrip(",") + if line.startswith("GNS3_UNIX_SOCKET_NIO="): + # Mirror the compute's parsing exactly (VendorDockerVM): the + # marker must start the line, and only a truthy value enables + # the unix-socket wiring — with 0/false/no the container wires + # adapters the standard way and can carry a kernel link. + return line.split("=", 1)[1].strip().lower() in ("1", "true", "yes") + return False class UDPLink(Link): diff --git a/gns3server/schemas/compute/nios.py b/gns3server/schemas/compute/nios.py index d3be033e1..a41b7eda1 100644 --- a/gns3server/schemas/compute/nios.py +++ b/gns3server/schemas/compute/nios.py @@ -91,8 +91,11 @@ class BridgeNIO(BaseModel): """ type: BridgeNIOType + # Required-but-nullable: the NIO factories read nio_settings["bridge"], + # and the create/update routes drop unset fields — an omitted key would + # pass validation and then surface as a KeyError (500) during binding. bridge: Optional[str] = Field( - None, + ..., description="Kernel bridge name e.g. gns3a1b2c3d4e5; None when an Ethernet switch " "owns the anchor's bridge membership (absorbed link)", ) diff --git a/gns3server/utils/kernel_anchor.py b/gns3server/utils/kernel_anchor.py index 9cf72b77e..c96cddffa 100644 --- a/gns3server/utils/kernel_anchor.py +++ b/gns3server/utils/kernel_anchor.py @@ -59,12 +59,18 @@ def kernel_anchor_type(node_type, environment=None): its own key unchanged. """ - # The marker check mirrors utils.application_id.is_iol_runner_environment - # inline: this module sits below every node module (compute and - # controller alike import it during their own import), so it must not - # depend on anything that pulls the controller package in. - if node_type == "docker" and "GNS3_IOL_RUNNER=" in (environment or ""): - return "iol_docker" + # The marker check mirrors the compute's class selection + # (compute/docker/__init__.py, _select_node_class) inline: only a + # stripped, comma-trimmed line *beginning* with the marker selects + # IOLDockerVM, so e.g. NOTE=GNS3_IOL_RUNNER=1 must not be named as an + # IOL TAP anchor here. The check stays dependency-free on purpose: this + # module sits below every node module (compute and controller alike + # import it during their own import), so it must not import them back. + if node_type == "docker": + for line in (environment or "").splitlines(): + line = line.strip().rstrip(",") + if line.startswith("GNS3_IOL_RUNNER="): + return "iol_docker" return node_type diff --git a/gns3server/utils/packet_filter_validation.py b/gns3server/utils/packet_filter_validation.py index cee97e59c..036921f2d 100644 --- a/gns3server/utils/packet_filter_validation.py +++ b/gns3server/utils/packet_filter_validation.py @@ -399,7 +399,12 @@ def split_kernel_only_features(filters: Dict[str, List[Any]]) -> Tuple[Dict[str, correl = 0 if correl: dropped.add("packet_loss correlation") - values = values[:1] + # The correlation slot is stripped even when it is zero: the + # relay filter takes exactly one argument (ubridge's + # packet_loss_setup rejects argc != 1), so a kept "0" would make + # the whole filter fail to load — only a non-zero value is worth + # reporting as a dropped feature. + values = values[:1] clean[filter_type] = values return clean, dropped diff --git a/pyproject.toml b/pyproject.toml index fc2d9b861..425402005 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,7 +184,6 @@ extend-immutable-calls = [ "gns3server/compute/dynamips/nodes/router.py" = ["F522", "RUF006"] "gns3server/compute/iou/iou_vm.py" = ["C901", "S108", "S324"] "gns3server/compute/iou/utils/iou_export.py" = ["C901", "RUF003"] -"gns3server/compute/kernel_datapath.py" = ["C901"] "gns3server/compute/project.py" = ["DTZ006", "S324"] "gns3server/compute/qemu/qemu_vm.py" = ["ASYNC220", "C901", "RUF003", "S603"] "gns3server/compute/ubridge/ubridge_hypervisor.py" = ["C901"] @@ -254,7 +253,6 @@ extend-immutable-calls = [ "tests/controller/test_project.py" = ["RUF015"] "tests/controller/test_snapshot.py" = ["RUF015"] "tests/controller/test_udp_link.py" = ["B006"] -"tests/e2e/test_docker_link_local_frames.py" = ["S603"] "tests/test_server.py" = ["E722"] "tests/test_topologies.py" = ["B011"] "tests/test_utils.py" = ["F403", "F405"] From 35cf9648822c96147babd8105724cfd15a711526 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 6 Oct 2026 19:55:31 +0800 Subject: [PATCH 091/101] test: cover the kernel-datapath review fixes Regression coverage added with the fixes: - capture-slot semantics: second-port refusal, failed-start rollback, and a non-owner port's stop not reaching `capture stop_kernel`; - QEMU stop/start teardown on a mid-body or pre-launch failure, IOU start cleanup and filter guards, Dynamips half-attach undo plus retry; - switch VLAN reconcile skipping a deferred anchor and rolling the ports_mapping back on failure; - Docker/QEMU occupied-port refusal, probe ctime invalidation, probe survival of a failing uBridge start; - env-marker dialects (mid-line mentions, falsy values) and the agent merge rejecting entries without a device_name. Test-body imports of product modules move to the shared header. --- tests/agent/mcp/test_device_tools.py | 44 ++++++-- .../compute/test_ethernet_switch_nodes.py | 100 ++++++++++++++++-- .../docker/test_docker_kernel_datapath.py | 75 ++++++++++++- tests/compute/docker/test_docker_vm.py | 18 ++++ .../docker/test_iol_docker_kernel_datapath.py | 3 +- .../dynamips/test_dynamips_kernel_datapath.py | 29 +++++ tests/compute/iou/test_iou_vm.py | 52 +++++++++ .../compute/qemu/test_qemu_kernel_datapath.py | 67 ++++++++++++ tests/compute/ubridge/test_tc_probe.py | 28 ++++- tests/controller/test_kernel_datapath_link.py | 47 +++++--- tests/e2e/test_docker_link_local_frames.py | 3 +- tests/utils/test_kernel_anchor.py | 3 + tests/utils/test_packet_filter_validation.py | 9 +- 13 files changed, 438 insertions(+), 40 deletions(-) diff --git a/tests/agent/mcp/test_device_tools.py b/tests/agent/mcp/test_device_tools.py index 3a4308053..e4ff08fc1 100644 --- a/tests/agent/mcp/test_device_tools.py +++ b/tests/agent/mcp/test_device_tools.py @@ -10,6 +10,8 @@ overwriting each other's commands and outputs. import json from unittest.mock import MagicMock, patch +import pytest + from gns3server.agent.gns3_copilot.utils.device_configs import ( merge_duplicate_device_configs, ) @@ -334,6 +336,27 @@ class TestDuplicateDeviceMerging: assert result[0]["status"] == "partial_success" assert set(result[0]["blocked_commands"]) == {"debug ip routing", "debug ospf events"} + def test_entry_without_device_name_fails_the_whole_batch(self): + """An unaddressable entry must abort with an error, not vanish.""" + from gns3server.agent.gns3_copilot.tools_v2.display_tools_nornir import ( + ExecuteMultipleDeviceCommands, + ) + + result = ExecuteMultipleDeviceCommands()._run( + json.dumps( + { + "project_id": PROJECT_ID, + "device_configs": [ + {"device_name": "R1", "commands": ["show version"]}, + {"commands": ["show clock"]}, + ], + } + ) + ) + + assert result[0]["status"] == "failed" + assert "device_configs[1]" in result[0]["error"] + class TestMergeDuplicateDeviceConfigs: """Unit tests for the shared normalization helper.""" @@ -361,12 +384,21 @@ class TestMergeDuplicateDeviceConfigs: assert merged == [{"device_name": "R1", "commands": ["a", "b"], "extra": 1}] assert original[0]["commands"] == ["a"] - def test_entry_without_device_name_is_dropped(self): - merged = merge_duplicate_device_configs( - [{"commands": ["a"]}, {"device_name": "R1", "commands": ["b"]}], - commands_field="commands", - ) - assert merged == [{"device_name": "R1", "commands": ["b"]}] + def test_entry_without_device_name_is_rejected(self): + # Silently dropping it would let the batch execute partially while + # every returned row reports success. + with pytest.raises(ValueError, match=r"device_configs\[0\]"): + merge_duplicate_device_configs( + [{"commands": ["a"]}, {"device_name": "R1", "commands": ["b"]}], + commands_field="commands", + ) + + def test_entry_that_is_not_an_object_is_rejected(self): + with pytest.raises(ValueError, match=r"device_configs\[1\]"): + merge_duplicate_device_configs( + [{"device_name": "R1", "commands": ["a"]}, "R2"], + commands_field="commands", + ) def test_missing_commands_field_defaults_to_empty(self): merged = merge_duplicate_device_configs( diff --git a/tests/api/routes/compute/test_ethernet_switch_nodes.py b/tests/api/routes/compute/test_ethernet_switch_nodes.py index b685b6a0d..3069c45de 100644 --- a/tests/api/routes/compute/test_ethernet_switch_nodes.py +++ b/tests/api/routes/compute/test_ethernet_switch_nodes.py @@ -22,7 +22,9 @@ import pytest_asyncio from fastapi import FastAPI, status from httpx import AsyncClient +from gns3server.compute.error import NodeError from gns3server.compute.project import Project +from gns3server.compute.ubridge.ubridge_error import UbridgeError from tests.utils import asyncio_patch # The builtin Ethernet switch talks to uBridge (brctl/bridge modules) instead of @@ -927,8 +929,6 @@ class TestEthernetSwitchNodesRoutes: proceeds — a lost race is not an error. """ - from gns3server.compute.ubridge.ubridge_error import UbridgeError - anchor, peer = "gs1a2b3c4d5e0", "gs1a2b3c4d5e1" node = compute_project.get_node(ethernet_switch["node_id"]) exists = {"value": False} @@ -1217,9 +1217,94 @@ class TestEthernetSwitchNodesRoutes: with self._sysfs(anchor_exists=True): await compute_client.post(url, json=self._anchor_params(anchor)) + node = compute_project.get_node(ethernet_switch["node_id"]) + node._ubridge_send.reset_mock() + + ports = [{"name": f"Ethernet{i}", "port_number": i, "type": "access", "vlan": 1} for i in range(8)] + ports[0]["vlan"] = 10 + response = await compute_client.put( + app.url_path_for( + "compute:update_ethernet_switch", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + ), + json={"ports_mapping": ports}, + ) + assert response.status_code == status.HTTP_200_OK + + br = node._bridge_name + commands = [c.args[0] for c in node._ubridge_send.call_args_list] + assert commands == [ + f'brctl vlan_del "{br}" "{anchor}" 1', + f'brctl vlan_add "{br}" "{anchor}" 10 pvid untagged', + ], commands + assert not any("delif" in c or "addif" in c for c in commands) + + async def test_ethernet_switch_update_ports_skips_a_deferred_anchor( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + A kernel port whose peer is stopped has no interface yet (the join is + deferred by design): a VLAN change must not send commands that would + fail ENODEV and abort the whole reconcile — the port keeps the new + settings in the mapping and they apply when the peer starts. + """ + + anchor = "gv00010203e0p0" + url = app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ) + with self._sysfs(anchor_exists=False): + await compute_client.post(url, json=self._anchor_params(anchor)) + node = compute_project.get_node(ethernet_switch["node_id"]) + assert node._kernel_ports.get(0) == anchor # deferred but registered node._ubridge_send.reset_mock() + ports = [{"name": f"Ethernet{i}", "port_number": i, "type": "access", "vlan": 1} for i in range(8)] + ports[0]["vlan"] = 10 + with self._sysfs(anchor_exists=False): + response = await compute_client.put( + app.url_path_for( + "compute:update_ethernet_switch", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + ), + json={"ports_mapping": ports}, + ) + + assert response.status_code == status.HTTP_200_OK + commands = [c.args[0] for c in node._ubridge_send.call_args_list] + assert not any("vlan_" in c for c in commands), commands + + async def test_ethernet_switch_update_ports_rolls_the_mapping_back_on_failure( + self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict + ) -> None: + """ + A failed VLAN reconcile must not keep the new mapping: the retry would + then diff it against itself and report success with the VLANs never + applied. + """ + + await compute_client.post( + app.url_path_for( + "compute:create_ethernet_switch_nio", + project_id=ethernet_switch["project_id"], + node_id=ethernet_switch["node_id"], + adapter_number="0", + port_number="0", + ), + json=self._udp_params(), + ) + + node = compute_project.get_node(ethernet_switch["node_id"]) + before = [dict(port) for port in node.ports_mapping] + node._ubridge_send.side_effect = NodeError("207-reconcile failed") + ports = [{"name": f"Ethernet{i}", "port_number": i, "type": "access", "vlan": 1} for i in range(8)] ports[0]["vlan"] = 10 response = await compute_client.put( @@ -1230,12 +1315,7 @@ class TestEthernetSwitchNodesRoutes: ), json={"ports_mapping": ports}, ) - assert response.status_code == status.HTTP_200_OK - br = node._bridge_name - commands = [c.args[0] for c in node._ubridge_send.call_args_list] - assert commands == [ - f'brctl vlan_del "{br}" "{anchor}" 1', - f'brctl vlan_add "{br}" "{anchor}" 10 pvid untagged', - ], commands - assert not any("delif" in c or "addif" in c for c in commands) + assert response.status_code == status.HTTP_409_CONFLICT + assert response.json()["message"] == "207-reconcile failed" + assert [dict(port) for port in node.ports_mapping] == before diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index c50c35c6a..853c1999a 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -899,7 +899,6 @@ async def test_update_nio_kernel_clears_netem(vm): async def test_apply_netem_reset_tolerates_missing_qdisc(vm): vm._ubridge_hypervisor = MagicMock() - from gns3server.compute.ubridge.ubridge_error import UbridgeError async def refuse(command): raise UbridgeError("207-Could not reset qdisc on gv0: No such file or directory") @@ -914,7 +913,6 @@ async def test_apply_netem_reset_tolerates_missing_qdisc(vm): async def test_apply_netem_reset_reraises_real_errors(vm): vm._ubridge_hypervisor = MagicMock() - from gns3server.compute.ubridge.ubridge_error import UbridgeError async def refuse(command): raise UbridgeError("206-Could not reset qdisc on gv0: Operation not permitted") @@ -997,6 +995,79 @@ async def test_start_capture_second_concurrent_fails(vm): with pytest.raises(UbridgeError, match="already in progress"): await vm.start_capture(0, "/tmp/capture.pcap") + # the failed start must not leave the port marked as capturing: a stuck + # flag refuses the retry and lets this port's stop kill the winner + assert nio.capturing is False + assert vm._kernel_capture_ifc is None + + +@pytest.mark.asyncio +async def test_start_capture_second_port_on_the_same_node_is_refused(vm): + """ + uBridge serves one kernel capture per process while the server models + captures per port: a second port's capture must be refused with a clear + error and its flag rolled back — otherwise its stop would issue the + process-wide, argument-less ``capture stop_kernel`` and kill the first + port's capture while its own flag still says capturing. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + vm._ethernet_adapters.append(type(vm._ethernet_adapters[0])()) + host_ifc1, _ = vm._veth_names(0, 0) + host_ifc2, _ = vm._veth_names(1, 0) + vm._kernel_veths[(0, 0)] = host_ifc1 + vm._kernel_veths[(1, 0)] = host_ifc2 + nio1 = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + nio2 = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio1) + vm._ethernet_adapters[1].add_nio(0, nio2) + + await vm.start_capture(0, "/tmp/capture.pcap") + + with pytest.raises(DockerError, match="one kernel capture per node"): + await vm.start_capture(1, "/tmp/capture2.pcap") + + assert nio1.capturing is True + assert nio2.capturing is False + vm._ubridge_send.assert_any_call(f'capture start_kernel {host_ifc1} "/tmp/capture.pcap"') + assert not any("stop_kernel" in str(c) for c in vm._ubridge_send.call_args_list) + + +@pytest.mark.asyncio +async def test_stop_capture_on_a_port_that_does_not_own_the_slot_skips_stop_kernel(vm): + """ + ``capture stop_kernel`` is process-wide and takes no interface: a stop on + a port whose flag says capturing while another port owns the slot must + not reach uBridge — it would stop the owner's capture. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._ubridge_send = AsyncioMagicMock() + vm.status = "started" + vm._ethernet_adapters.append(type(vm._ethernet_adapters[0])()) + host_ifc1, _ = vm._veth_names(0, 0) + host_ifc2, _ = vm._veth_names(1, 0) + vm._kernel_veths[(0, 0)] = host_ifc1 + vm._kernel_veths[(1, 0)] = host_ifc2 + nio1 = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + nio2 = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + vm._ethernet_adapters[0].add_nio(0, nio1) + vm._ethernet_adapters[1].add_nio(0, nio2) + + await vm.start_capture(0, "/tmp/capture.pcap") # port 0 owns the slot + nio2.start_packet_capture("/tmp/capture2.pcap") # a stale second flag + + await vm.stop_capture(1) + + assert nio2.capturing is False + assert not any("stop_kernel" in str(c) for c in vm._ubridge_send.call_args_list) + + # the owner's capture is untouched and still stoppable + await vm.stop_capture(0) + vm._ubridge_send.assert_any_call("capture stop_kernel") + @pytest.mark.asyncio async def test_connect_nio_kernel_restarts_capture(vm): diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 51a1e4176..dbddda0ae 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -1642,6 +1642,24 @@ async def test_adapter_add_nio_binding_1(vm): assert vm._ethernet_adapters[0].get_nio(0) == nio +@pytest.mark.asyncio +async def test_adapter_add_nio_binding_refuses_an_occupied_port(vm): + """ + The compute-side backstop of the controller's duplicate-port guard: a + second NIO for a port that already carries one is refused instead of + silently overwriting the binding the first link's teardown still needs. + """ + + first = vm.manager.create_nio({"type": "nio_udp", "lport": 4242, "rport": 4343, "rhost": "127.0.0.1"}) + await vm.adapter_add_nio_binding(0, first) + + second = vm.manager.create_nio({"type": "nio_udp", "lport": 4244, "rport": 4345, "rhost": "127.0.0.1"}) + with pytest.raises(DockerError, match="already has a link"): + await vm.adapter_add_nio_binding(0, second) + + assert vm._ethernet_adapters[0].get_nio(0) is first + + @pytest.mark.asyncio async def test_adapter_add_nio_binding_sets_carrier(vm): diff --git a/tests/compute/docker/test_iol_docker_kernel_datapath.py b/tests/compute/docker/test_iol_docker_kernel_datapath.py index b3f692706..3491c128e 100644 --- a/tests/compute/docker/test_iol_docker_kernel_datapath.py +++ b/tests/compute/docker/test_iol_docker_kernel_datapath.py @@ -22,6 +22,7 @@ import pytest_asyncio from gns3server.compute.docker import Docker from gns3server.compute.docker.docker_error import DockerError from gns3server.compute.docker.iol_docker_vm import IOLDockerVM +from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_anchor_type from tests.utils import AsyncioMagicMock NODE_ID = "00010203-0405-0607-0809-0a0b0c0d0e0f" @@ -385,8 +386,6 @@ def test_anchor_names_follow_the_iol_docker_key(): never the docker veth namespace. """ - from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_anchor_type - environment = "GNS3_IOL_RUNNER=1\nGNS3_IOL_STARTUP_CONFIG=cfg.txt" assert kernel_anchor_type("docker", environment) == "iol_docker" assert kernel_anchor_type("docker", "GNS3_UNIX_SOCKET_NIO=1") == "docker" diff --git a/tests/compute/dynamips/test_dynamips_kernel_datapath.py b/tests/compute/dynamips/test_dynamips_kernel_datapath.py index d09932eea..d1af1c808 100644 --- a/tests/compute/dynamips/test_dynamips_kernel_datapath.py +++ b/tests/compute/dynamips/test_dynamips_kernel_datapath.py @@ -309,6 +309,35 @@ async def test_slot_add_nio_binding_attach_failure_leaves_the_port_unbound(route assert router._slots[0].get_nio(0) is None +@pytest.mark.asyncio +async def test_attach_kernel_nio_failure_undoes_the_binding_so_a_retry_works(router): + """ + A failure after the hypervisor binding (the enslave step here) must undo + the half-attach: the _tap_nios entry would otherwise short-circuit every + retry, and a link delete reads the still-unbound adapter as "nothing to + do" — a silent black hole until the router restarts. + """ + + router.status = "started" + router._kernel_taps[(0, 0)] = TAP00 + nio = await _nio(router) + + real_attach = router._kernel_attach + router._kernel_attach = AsyncioMagicMock(side_effect=UbridgeError("208-Device or resource busy")) + with pytest.raises(UbridgeError): + await router._attach_kernel_nio(0, 0, nio) + router._kernel_attach = real_attach + + # the half-attach went away with the failure... + assert (0, 0) not in router._tap_nios + assert any("slot_remove_nio_binding" in c for c in _hypervisor_commands(router)) + + # ...so the retry wires the port for real + await router._attach_kernel_nio(0, 0, nio) + assert (0, 0) in router._tap_nios + router._ubridge_send.assert_any_call(f'brctl addif "{BRIDGE}" "{TAP00}"') + + @pytest.mark.asyncio async def test_attach_kernel_nio_without_anchor_on_a_running_node_raises(router): """ diff --git a/tests/compute/iou/test_iou_vm.py b/tests/compute/iou/test_iou_vm.py index 67a0fc5f1..509240c00 100644 --- a/tests/compute/iou/test_iou_vm.py +++ b/tests/compute/iou/test_iou_vm.py @@ -26,9 +26,11 @@ from unittest.mock import MagicMock, call import pytest import pytest_asyncio +from gns3server.compute.error import NodeError from gns3server.compute.iou import IOU from gns3server.compute.iou.iou_error import IOUError from gns3server.compute.iou.iou_vm import IOUVM, IOUL1KeepaliveProtocol +from gns3server.compute.ubridge.ubridge_error import UbridgeError from tests.utils import AsyncioMagicMock, asyncio_patch @@ -151,6 +153,26 @@ async def test_start_failure_stops_l1_responder(vm): vm._stop_l1_keepalive_responder.assert_called_once_with() +@pytest.mark.asyncio +async def test_start_failure_in_the_tap_setup_stops_ubridge(vm): + """ + A raise before the IOU launch (the tap datapath setup here) must take + the uBridge datapath down again: a node reporting a failed start must + not keep a uBridge process and its anchor TAPs alive. + """ + + vm._check_requirements = AsyncioMagicMock(return_value=True) + vm._check_iou_license = AsyncioMagicMock(return_value=True) + vm._start_ubridge = AsyncioMagicMock() + vm._prepare_tap_datapath = AsyncioMagicMock(side_effect=IOUError("tap create refused")) + vm._stop_ubridge = AsyncioMagicMock() + + with pytest.raises(IOUError, match="tap create refused"): + await vm.start() + + assert vm._stop_ubridge.call_count == 1 + + @pytest.mark.asyncio async def test_start_with_iourc(vm, tmpdir, config): @@ -580,6 +602,36 @@ def test_get_legacy_vm_workdir(): assert IOU.get_legacy_vm_workdir(42, "bla") == "iou/device-42" +@pytest.mark.asyncio +async def test_ubridge_apply_filters_rejects_kernel_only(vm): + """ + The IOL relay path carries the base guard: a kernel-only filter reaching + a relay port is a clear NodeError, not a late uBridge failure. + """ + + vm._ubridge_send = AsyncioMagicMock() + with pytest.raises(NodeError, match="kernel-datapath"): + await vm._ubridge_apply_filters("bridge0 0 0", {"rate": ["512kbit"]}) + vm._ubridge_send.assert_not_called() + + +@pytest.mark.asyncio +async def test_ubridge_apply_filters_degrades_a_bad_bpf_line(vm): + """ + A BPF line that no longer compiles on this uBridge's libpcap is a + warning, not a hard failure — the same degradation as the base relay + path (the controller validated the syntax with tcpdump at update time). + """ + + async def refuse(command): + if "add_packet_filter" in command: + raise UbridgeError("Cannot compile filter 'icmp': syntax error") + return ["OK"] + + vm._ubridge_send = refuse + await vm._ubridge_apply_filters("bridge0 0 0", {"bpf": ["icmp"]}) + + @pytest.mark.asyncio async def test_invalid_iou_file(vm, iourc_file): diff --git a/tests/compute/qemu/test_qemu_kernel_datapath.py b/tests/compute/qemu/test_qemu_kernel_datapath.py index ba6ea96ec..9c5c211c0 100644 --- a/tests/compute/qemu/test_qemu_kernel_datapath.py +++ b/tests/compute/qemu/test_qemu_kernel_datapath.py @@ -376,6 +376,73 @@ async def test_stop_stops_the_process_before_deleting_the_taps(vm): assert order == ["process", "taps"] +@pytest.mark.asyncio +async def test_stop_releases_the_datapath_even_when_the_stop_body_raises(vm): + """ + The finally keeps a mid-stop raise (an export error here) from skipping + the uBridge/TAP release — the datapath would otherwise outlive a failed + stop with nothing left to reclaim it. + """ + + _running(vm) + stopped = AsyncioMagicMock() + + async def failing_export(): + raise QemuError("qemu-img is missing") + + with ( + patch("gns3server.utils.asyncio.wait_for_process_termination", new=AsyncioMagicMock()), + patch.object(vm, "_export_config", new=failing_export), + patch.object(vm, "_stop_ubridge", new=stopped), + asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._clear_save_vm_stated"), + ): + with pytest.raises(QemuError, match="qemu-img is missing"): + await vm.stop() + + assert stopped.call_count == 1 + + +@pytest.mark.asyncio +async def test_start_failure_before_the_launch_tears_down_the_datapath(vm): + """ + A raise before the QEMU launch (a broken qemu_path in _build_command) + must take the uBridge/TAP datapath down again: a node reporting a failed + start must not keep a uBridge process and the anchor TAPs alive. + """ + + stopped = AsyncioMagicMock() + with ( + patch.object(vm, "check_available_ram"), + patch.object(vm, "_prepare_tap_datapath", new=AsyncioMagicMock()), + patch.object(vm, "_build_command", new=AsyncioMagicMock(side_effect=QemuError("qemu-img is missing"))), + patch.object(vm, "_stop_ubridge", new=stopped), + ): + with pytest.raises(QemuError, match="qemu-img is missing"): + await vm.start() + + assert stopped.call_count == 1 + + +@pytest.mark.asyncio +async def test_add_nio_binding_refuses_an_occupied_adapter(vm): + """ + The compute-side backstop of the controller's duplicate-port guard: a + second NIO for an adapter that already carries one is refused instead of + silently overwriting the binding the first link's teardown still needs. + """ + + vm._tap_datapath = True + vm._kernel_taps[(0, 0)] = TAP0 + first = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE}) + await vm.adapter_add_nio_binding(0, first) + + second = vm.manager.create_nio({"type": "nio_bridge", "bridge": "gns3otherbridge"}) + with pytest.raises(QemuError, match="already has a link"): + await vm.adapter_add_nio_binding(0, second) + + assert vm._ethernet_adapters[0].get_nio(0) is first + + @pytest.mark.asyncio async def test_concurrent_stops_sweep_every_tap_exactly_once(vm): """ diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py index 1cdeba245..45275368c 100644 --- a/tests/compute/ubridge/test_tc_probe.py +++ b/tests/compute/ubridge/test_tc_probe.py @@ -19,6 +19,8 @@ The standalone `tc capabilities` probe behind the compute /capabilities payload: spawn a throwaway uBridge, ask once, cache by binary identity. """ +import os + import pytest from gns3server.compute.ubridge import tc_probe @@ -91,11 +93,13 @@ def probe_env(monkeypatch, tmp_path): tc_probe._iol_tap_cache.clear() tc_probe._bridge_tap_cache.clear() FakeHypervisor.last = None - # a real file on disk: the cache is keyed on its (path, mtime, size) + # a real file on disk: the cache is keyed on its (path, mtime, size, + # ctime) binary = tmp_path / "ubridge" binary.write_bytes(b"") monkeypatch.setattr(tc_probe.shutil, "which", lambda name: str(binary)) monkeypatch.setattr(tc_probe, "Hypervisor", FakeHypervisor) + return str(binary) async def test_probe_returns_parsed_report(): @@ -134,6 +138,28 @@ async def test_probe_missing_binary_means_unknown(monkeypatch): assert FakeHypervisor.spawned == 0 +async def test_probe_reprobes_when_the_binary_attributes_change(probe_env): + + assert await probe_tc_capabilities() is not None + # chmod touches ctime but neither mtime nor size: a setcap/chmod remedy + # after a permission-related probe failure must be able to flip the + # cached verdict without a server restart. + os.chmod(probe_env, 0o600) + assert await probe_tc_capabilities() is not None + assert FakeHypervisor.spawned == 2 + + +async def test_probe_bridge_tap_unknown_when_ubridge_will_not_start(monkeypatch): + """start() can raise UbridgeError (unusable binary): unknown, never a + failed /capabilities response.""" + + async def failing_start(self): + raise UbridgeError("uBridge executable version must be >= 2.4.0") + + monkeypatch.setattr(FakeHypervisor, "start", failing_start) + assert await probe_bridge_tap_support() is None + + async def test_probe_empty_reply_means_unknown(): FakeHypervisor.reply = [] diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 7d326f896..0f34d6ed6 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -26,9 +26,13 @@ import pytest from gns3server.config import Config from gns3server.controller.controller_error import ControllerError +from gns3server.controller.link import FILTERS from gns3server.controller.node import Node from gns3server.controller.ports.ethernet_port import EthernetPort +from gns3server.controller.ports.serial_port import SerialPort from gns3server.controller.udp_link import UDPLink +from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_cascade_names +from tests.utils import AsyncioMagicMock def _node(project, compute, name, node_type="docker", status="stopped", environment=None): @@ -231,8 +235,6 @@ async def test_kernel_datapath_not_eligible_for_dynamips_serial_ports(project): decided by the controller's port matrix. """ - from gns3server.controller.ports.serial_port import SerialPort - compute = MagicMock() compute.id = "compute-1" compute.capabilities = {"ubridge_tap": True} @@ -253,8 +255,6 @@ async def test_kernel_datapath_not_eligible_for_serial_ports(project): kernel path would fail at the compute (no anchor on a serial bay). """ - from gns3server.controller.ports.serial_port import SerialPort - compute = MagicMock() compute.id = "compute-1" compute.capabilities = {"ubridge_tap": True, "ubridge_iol_tap": True} @@ -431,6 +431,33 @@ async def test_kernel_datapath_not_eligible_for_unix_socket_containers(project): assert link._kernel_datapath_eligible(node1, node2) is False +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_when_unix_socket_marker_is_falsy(project): + """ + GNS3_UNIX_SOCKET_NIO=0 (like false/no) leaves the container on the + standard veth wiring — the compute parses the value, so the controller + must not exclude it with a substring match, or the link silently stays + on the relay forever. + """ + + link, node1, node2 = await _kernel_link(project) + node2._properties = {"environment": "GNS3_UNIX_SOCKET_NIO=0"} + assert link._kernel_datapath_eligible(node1, node2) is True + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_when_the_marker_is_mid_line(project): + """ + A mid-line mention (NOTE=GNS3_UNIX_SOCKET_NIO=1) is not the marker: + the compute's line-based parsing does not select VendorDockerVM for it, + so the controller must not exclude the node either. + """ + + link, node1, node2 = await _kernel_link(project) + node2._properties = {"environment": "NOTE=GNS3_UNIX_SOCKET_NIO=1"} + assert link._kernel_datapath_eligible(node1, node2) is True + + @pytest.mark.asyncio async def test_iol_runner_containers_gate_on_the_bridge_tap_capabilities(project): """ @@ -476,8 +503,6 @@ async def test_iol_runner_container_on_a_switch_gates_and_names_the_iol_docker_a a late error. """ - from gns3server.utils.kernel_anchor import kernel_anchor_name - compute = MagicMock() compute.id = "compute-1" compute.capabilities = {"ubridge_bridge_tap": True, "ubridge_tap": True} @@ -880,8 +905,6 @@ async def test_available_filters_unconstrained_without_report(project): validation is the guard. """ - from gns3server.controller.link import FILTERS - link, _node1, _node2 = await _kernel_link(project) link._link_data = [{"type": "nio_bridge"}] assert [f["type"] for f in link.available_filters()] == [f["type"] for f in FILTERS] @@ -946,8 +969,6 @@ async def test_switch_link_prepare_emits_anchor_and_external_bridge(project): interface. """ - from gns3server.utils.kernel_anchor import kernel_anchor_name - compute = MagicMock() compute.id = "compute-1" switch = _node(project, compute, "sw1", node_type="ethernet_switch") @@ -987,8 +1008,6 @@ async def test_switch_link_repushes_the_switch_nio_on_node_start(project): replaced). Relay links do nothing. """ - from tests.utils import AsyncioMagicMock - compute = MagicMock() compute.id = "compute-1" switch = _node(project, compute, "sw1", node_type="ethernet_switch") @@ -1031,8 +1050,6 @@ async def test_prepare_cascade_link_emits_paired_anchor_nios(project): once like any other kernel link. """ - from gns3server.utils.kernel_anchor import kernel_cascade_names - compute = MagicMock() compute.id = "compute-1" sw1 = _node(project, compute, "sw1", node_type="ethernet_switch") @@ -1071,8 +1088,6 @@ async def test_node_started_repushes_both_cascade_ends(project): re-checks and re-joins its own half. """ - from tests.utils import AsyncioMagicMock - compute = MagicMock() compute.id = "compute-1" sw1 = _node(project, compute, "sw1", node_type="ethernet_switch") diff --git a/tests/e2e/test_docker_link_local_frames.py b/tests/e2e/test_docker_link_local_frames.py index 8ac4a5404..42743d83a 100644 --- a/tests/e2e/test_docker_link_local_frames.py +++ b/tests/e2e/test_docker_link_local_frames.py @@ -251,7 +251,8 @@ def _llprobe(): if cc is None: pytest.skip("cannot build the raw-frame probe (no C compiler found)") try: - subprocess.run([cc, "-static", "-O2", "-o", binary, source], check=True, capture_output=True) + # S603: resolved compiler path, fixed argv, no shell + subprocess.run([cc, "-static", "-O2", "-o", binary, source], check=True, capture_output=True) # noqa: S603 except subprocess.CalledProcessError as e: pytest.skip(f"cannot build the raw-frame probe (cc -static): {e}") return binary diff --git a/tests/utils/test_kernel_anchor.py b/tests/utils/test_kernel_anchor.py index 7f5558d1a..247237644 100644 --- a/tests/utils/test_kernel_anchor.py +++ b/tests/utils/test_kernel_anchor.py @@ -45,8 +45,11 @@ def test_anchor_type_keys_iol_runner_containers_apart(): assert kernel_anchor_type("docker", "GNS3_IOL_RUNNER=1") == "iol_docker" assert kernel_anchor_type("docker", "GNS3_IOL_STARTUP_CONFIG=cfg.txt\nGNS3_IOL_RUNNER=1") == "iol_docker" + assert kernel_anchor_type("docker", "GNS3_IOL_RUNNER=1,") == "iol_docker" assert kernel_anchor_type("docker") == "docker" assert kernel_anchor_type("docker", "GNS3_UNIX_SOCKET_NIO=1") == "docker" + # a mid-line mention is not the marker (mirrors _select_node_class) + assert kernel_anchor_type("docker", "NOTE=GNS3_IOL_RUNNER=1") == "docker" assert kernel_anchor_type("qemu") == "qemu" diff --git a/tests/utils/test_packet_filter_validation.py b/tests/utils/test_packet_filter_validation.py index 2bada6f13..0cbf970b3 100644 --- a/tests/utils/test_packet_filter_validation.py +++ b/tests/utils/test_packet_filter_validation.py @@ -360,8 +360,13 @@ class TestNetemExtensionFilters: {"delay": [100, 20, ""], "packet_loss": [10, 0], "corrupt": [2], "bpf": ["icmp"]} ) assert dropped == set() - # zero correlation and empty distribution are kept as-is (no-ops) - assert clean == {"delay": [100, 20, ""], "packet_loss": [10, 0], "corrupt": [2], "bpf": ["icmp"]} + # Zero correlation and an empty distribution carry no feature, so + # neither is reported as dropped. The packet_loss correlation slot + # is stripped all the same: the relay filter takes exactly one + # argument (ubridge's packet_loss_setup rejects argc != 1), while + # the delay's empty 3rd value is an unquoted empty token that + # tokenizes away — harmless kept as-is. + assert clean == {"delay": [100, 20, ""], "packet_loss": [10], "corrupt": [2], "bpf": ["icmp"]} def test_kernel_only_features_empty(self): assert split_kernel_only_features({}) == ({}, set()) From 1542a222f4df5f77c23905ac3a17588c0bd674a5 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 6 Oct 2026 21:17:50 +0800 Subject: [PATCH 092/101] refactor: share the anchor-TAP lifecycle in the kernel-datapath mixin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The persistent-TAP lifecycle — probe, sweep, create, register, delete — was implemented four times, once per node type, and the copies had drifted (probe mechanism, _tap_datapath reset, keep-vs-recreate policy). The mechanical half moves into KernelDatapathMixin: - _probe_persistent_taps(probe_name): the scratch-TAP probe, returning the refusal for the caller's own warning (QEMU, Dynamips); - _create_anchor_taps(ports, set_owner, keep_existing): sweep + create + optional ownership handover + carrier down + registry, with the ports enumerated by the node type and named through the new _tap_name hook; - _delete_anchor_taps(): the suppressed tap sweep + per-link bridge cleanup + registry clear. Each node type keeps what its fd holder dictates: the release step before the tap sweep (the QEMU process, the IOL bridge, the hypervisor's NIO bindings, the container port bridges), its probe policy, and its registry lifetime. QEMU's now-unconditional registry clear aligns it with the other three; everything else is behavior-preserving. --- gns3server/compute/docker/iol_docker_vm.py | 32 ++----- gns3server/compute/dynamips/nodes/router.py | 47 +++-------- gns3server/compute/iou/iou_vm.py | 36 +++----- gns3server/compute/kernel_datapath.py | 92 +++++++++++++++++++++ gns3server/compute/qemu/qemu_vm.py | 66 +++------------ 5 files changed, 133 insertions(+), 140 deletions(-) diff --git a/gns3server/compute/docker/iol_docker_vm.py b/gns3server/compute/docker/iol_docker_vm.py index e6d2bf70a..5687caca7 100644 --- a/gns3server/compute/docker/iol_docker_vm.py +++ b/gns3server/compute/docker/iol_docker_vm.py @@ -489,25 +489,13 @@ class IOLDockerVM(VendorDockerVM): return self._tap_datapath = True - for adapter_number in range(0, len(self._ethernet_adapters)): - for port_number in range(0, self._ethernet_adapters[adapter_number].interfaces): - tap = self._tap_name(adapter_number, port_number) - # A persistent TAP outlives a crash: sweep any leftover of a - # previous run before recreating it (a surviving device would - # also keep its stale tc qdisc). - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._ubridge_send(f'tap create "{tap}"') - # Carrier off until a link attaches (the anchor contract). - await self._ubridge_send(f'link set "{tap}" down') - self._kernel_taps[(adapter_number, port_number)] = tap - log.debug( - "IOL container '%s': anchor TAP %s created for bay %d unit %d", - self._name, - tap, - adapter_number, - port_number, - ) + # Every bay/unit anchors. No set_owner: the container port bridges + # hold these fds, and only while a kernel link is attached. + await self._create_anchor_taps( + (adapter_number, port_number) + for adapter_number in range(0, len(self._ethernet_adapters)) + for port_number in range(0, self._ethernet_adapters[adapter_number].interfaces) + ) async def _ensure_anchor(self, adapter_number, port_number): """ @@ -650,11 +638,7 @@ class IOLDockerVM(VendorDockerVM): for port_number in range(0, adapter.interfaces): with contextlib.suppress(UbridgeError): await self._ubridge_send(f"bridge delete {self._bridge_name(adapter_number, port_number)}") - for tap in self._kernel_taps.values(): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._remove_kernel_bridges() - self._kernel_taps.clear() + await self._delete_anchor_taps() self._tap_datapath = False self._ubridge_tc_caps = None await super()._stop_ubridge() diff --git a/gns3server/compute/dynamips/nodes/router.py b/gns3server/compute/dynamips/nodes/router.py index de0063d43..439d9c10f 100644 --- a/gns3server/compute/dynamips/nodes/router.py +++ b/gns3server/compute/dynamips/nodes/router.py @@ -42,7 +42,6 @@ from ...base_node import BaseNode from ...error import NodeError from ...kernel_datapath import KernelDatapathMixin from ...nios.nio_bridge import NIOBridge -from ...ubridge.ubridge_error import UbridgeError from ..adapters.adapter import ETHERNET_ADAPTERS, ETHERNET_WICS from ..dynamips_error import DynamipsError from ..nios.nio_tap import NIOTAP @@ -1648,19 +1647,15 @@ class Router(KernelDatapathMixin, BaseNode): # uBridge (re)started: the tc capabilities must be probed again. self._ubridge_tc_caps = None if not self._tap_datapath: - probe = f"gd{self._id.replace('-', '')[:8]}prob" - try: - await self._ubridge_send(f'tap create "{probe}"') - except UbridgeError as e: + probe_error = await self._probe_persistent_taps(f"gd{self._id.replace('-', '')[:8]}prob") + if probe_error is not None: message = ( - f"Router '{self._name}': uBridge cannot create persistent TAPs ({e}); " + f"Router '{self._name}': uBridge cannot create persistent TAPs ({probe_error}); " "this node runs on the relay datapath and cannot carry kernel links" ) log.warning(message) self.project.emit("log.warning", {"message": message}) return - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{probe}"') self._tap_datapath = True await self._create_taps() @@ -1675,31 +1670,15 @@ class Router(KernelDatapathMixin, BaseNode): async def _create_taps(self): """ - Create the persistent TAP every Ethernet slot/port owns. Each TAP - starts DOWN (carrier off until a kernel link attaches); uBridge - creates the device and hands ownership to this user, so the - unprivileged Dynamips hypervisor can open it. A port whose anchor - already exists (a restart, or an adapter hot-added to a running - router) keeps it — an anchor is never recreated under a live link. - A persistent TAP outlives its creator, so a leftover from a previous - run (crash, kill) is swept first. + Create the persistent TAP every Ethernet slot/port owns (WIC ports + included), handed to this user so the unprivileged Dynamips + hypervisor can open the device. Unlike QEMU's, these anchors outlive + a node stop — the hypervisor keeps its NIO bindings — so a port that + already holds one keeps it: an anchor is never recreated under a + live link (a restart, or an adapter hot-added to a running router). """ - for slot_number, port_number in self._ethernet_slot_ports(): - if (slot_number, port_number) in self._kernel_taps: - continue - tap = self._tap_name(slot_number, port_number) - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._ubridge_send(f'tap create "{tap}"') - try: - await self._ubridge_send(f"tap set_owner {tap} {os.getuid()}") - except UbridgeError as e: - # Only root could open the TAP now: the hypervisor would fail - # to bind its NIO, which is worth surfacing at that point. - log.warning("Router '%s': could not hand TAP %s to uid %s: %s", self._name, tap, os.getuid(), e) - await self._ubridge_send(f'link set "{tap}" down') - self._kernel_taps[(slot_number, port_number)] = tap + await self._create_anchor_taps(self._ethernet_slot_ports(), set_owner=True, keep_existing=True) async def _attach_kernel_nio(self, slot_number, port_number, nio): """ @@ -1792,11 +1771,7 @@ class Router(KernelDatapathMixin, BaseNode): with contextlib.suppress(DynamipsError, OSError): await tap_nio.delete() self._tap_nios.clear() - for tap in self._kernel_taps.values(): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._remove_kernel_bridges() - self._kernel_taps.clear() + await self._delete_anchor_taps() self._ubridge_tc_caps = None await super()._stop_ubridge() diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index aad00f0bc..f11cac8af 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -929,27 +929,15 @@ class IOUVM(KernelDatapathMixin, BaseNode): return self._tap_datapath = True - await self._create_taps() - - async def _create_taps(self): - """ - Create the persistent TAP every Ethernet bay/unit owns (four units - per bay, the IOU adapter model). uBridge holds a device's fd only - while a kernel link binds the port to it (``iol_bridge - add_nio_tap``); until then the TAP sits DOWN with no holder, and it - survives link churn untouched — links attach to it, never recreate - it. A persistent TAP outlives its creator, so a leftover from a - previous run (crash, kill) is swept first. - """ - - for adapter_number, adapter in enumerate(self._ethernet_adapters): - for port_number in adapter.ports.keys(): - tap = self._tap_name(adapter_number, port_number) - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._ubridge_send(f'tap create "{tap}"') - await self._ubridge_send(f'link set "{tap}" down') - self._kernel_taps[(adapter_number, port_number)] = tap + # Every Ethernet bay/unit anchors (four units per bay, the IOU + # adapter model; serial bays never do). No set_owner: uBridge holds + # these fds itself, and only while a kernel link binds the port — + # until then, and after, the TAP survives link churn untouched. + await self._create_anchor_taps( + (adapter_number, port_number) + for adapter_number, adapter in enumerate(self._ethernet_adapters) + for port_number in adapter.ports.keys() + ) async def _attach_kernel_nio(self, adapter_number, port_number, nio): """ @@ -988,11 +976,7 @@ class IOUVM(KernelDatapathMixin, BaseNode): if self.ubridge: with contextlib.suppress(UbridgeError): await self._ubridge_send(f"iol_bridge delete {self._iol_bridge_name()}") - for tap in self._kernel_taps.values(): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._remove_kernel_bridges() - self._kernel_taps.clear() + await self._delete_anchor_taps() self._ubridge_tc_caps = None await super()._stop_ubridge() diff --git a/gns3server/compute/kernel_datapath.py b/gns3server/compute/kernel_datapath.py index fd5fced43..967d813bc 100644 --- a/gns3server/compute/kernel_datapath.py +++ b/gns3server/compute/kernel_datapath.py @@ -34,6 +34,9 @@ A node type using this mixin provides: or None when the adapter has none (a vendor container without a veth); * ``_kernel_anchors()`` — every anchor it currently owns, used to recognise the polymorphic marker anchor; +* ``_tap_name(adapter_number, port_number)`` — the deterministic anchor TAP + name of a port (the shared utils.kernel_anchor contract), for the node + types whose anchor is a persistent TAP; * optionally ``_relay_carrier_bridge(adapter_number, port_number)`` — the uBridge bridge carrying the adapter's TAP relay, for node types whose relay datapath still rides a uBridge-owned TAP instead of an anchor; @@ -91,6 +94,15 @@ class KernelDatapathMixin: return None + def _tap_name(self, adapter_number, port_number=0): + """ + Returns the deterministic anchor TAP name of a port (the shared + utils.kernel_anchor naming contract — the controller names a peer's + anchor with the same function when an Ethernet switch absorbs it). + """ + + raise NotImplementedError + def _kernel_error(self, message): """ Returns the node module's error for a kernel-datapath failure @@ -130,6 +142,86 @@ class KernelDatapathMixin: state = "on" if connected else "off" await self._ubridge_send(f"bridge set_nio_tap_carrier {bridge_name} {state}") + # ------------------------------------------------------------------ + # Anchor TAP lifecycle + # ------------------------------------------------------------------ + + async def _probe_persistent_taps(self, probe_name): + """ + Probe the node's uBridge for the persistent-TAP module with a + scratch TAP that is deleted again. Returns None when the module + works, or the UbridgeError that refused it — an old build answers + "Unknown command"/"Unknown module", a build without CAP_NET_ADMIN + fails outright. The caller composes its own warning and fallback: + the legacy relay datapath keeps working either way. + """ + + try: + await self._ubridge_send(f'tap create "{probe_name}"') + except UbridgeError as e: + return e + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{probe_name}"') + return None + + async def _create_anchor_taps(self, ports, set_owner=False, keep_existing=False): + """ + Create the persistent anchor TAP of every port, named through + ``_tap_name`` and registered in ``_kernel_taps``. + + *ports* is an iterable of ``(adapter_number, port_number)`` keys — a + node type enumerates whichever of its ports anchor (QEMU's whole + adapters, IOU's bay/units, Dynamips' Ethernet slot ports, the IOL + container's bay/units). A persistent TAP outlives its creator, so a + leftover of a previous run (crash, kill) is swept first; each TAP is + created and starts DOWN (carrier off until a link attaches). + ``set_owner`` hands the device to this user for the node types whose + unprivileged emulator opens it itself (QEMU, Dynamips); + ``keep_existing`` leaves ports that already hold an anchor alone — + for a node whose anchors outlive its stop (Dynamips) or an adapter + hot-added to a running node, where an anchor must never be recreated + under a live link. + """ + + for key in ports: + if keep_existing and key in self._kernel_taps: + continue + tap = self._tap_name(*key) + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._ubridge_send(f'tap create "{tap}"') + if set_owner: + try: + await self._ubridge_send(f"tap set_owner {tap} {os.getuid()}") + except UbridgeError as e: + # Only root could open the TAP then: the emulator would + # fail to attach its netdev, which is worth surfacing + # here rather than as a launch failure. + log.warning( + "Node '%s' [%s]: could not hand TAP %s to uid %s: %s", self._name, self._id, tap, os.getuid(), e + ) + await self._ubridge_send(f'link set "{tap}" down') + self._kernel_taps[key] = tap + log.debug("Node '%s' [%s]: anchor TAP %s created for port %s", self._name, self._id, tap, key) + + async def _delete_anchor_taps(self): + """ + Delete every registered anchor TAP and the per-link kernel bridges + this node still holds, then drop the registry. The caller releases + whatever holds each TAP's fd first (the QEMU process, the IOL + bridge, the hypervisor's NIO bindings, the container port bridges): + a held device answers EBADFD and the best-effort delete below would + leak it. The sweep is suppressed per TAP — an anchor already gone + is not an error — and the registry is cleared either way. + """ + + if self.ubridge: + for tap in self._kernel_taps.values(): + with contextlib.suppress(UbridgeError): + await self._ubridge_send(f'tap delete "{tap}"') + await self._remove_kernel_bridges() + self._kernel_taps.clear() + # ------------------------------------------------------------------ # Link attach / detach # ------------------------------------------------------------------ diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 37485fc53..dac2c219f 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -20,7 +20,6 @@ order to run a QEMU VM. """ import asyncio -import contextlib import json import logging import math @@ -54,7 +53,6 @@ from ..kernel_datapath import KernelDatapathMixin from ..nios.nio_bridge import NIOBridge from ..nios.nio_tap import NIOTAP from ..nios.nio_udp import NIOUDP -from ..ubridge.ubridge_error import UbridgeError from .qemu_error import QemuError from .utils.qcow2 import Qcow2, Qcow2Error from .utils.ziputils import pack_zip, unpack_zip @@ -1565,65 +1563,26 @@ class QemuVM(KernelDatapathMixin, BaseNode): if not self.ubridge: return - probe = f"gq{self._id.replace('-', '')[:8]}prob" - try: - await self._ubridge_send(f'tap create "{probe}"') - except UbridgeError as e: - log.warning("QEMU VM '%s': uBridge cannot create persistent TAPs (%s)", self._name, e) + probe_error = await self._probe_persistent_taps(f"gq{self._id.replace('-', '')[:8]}prob") + if probe_error is not None: + log.warning("QEMU VM '%s': uBridge cannot create persistent TAPs (%s)", self._name, probe_error) self.project.emit( "log.warning", { - "message": f"QEMU VM '{self._name}': uBridge does not support persistent TAPs ({e}); " + "message": f"QEMU VM '{self._name}': uBridge does not support persistent TAPs ({probe_error}); " "this node runs on the legacy relay datapath and cannot carry kernel links" }, ) return - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{probe}"') self._tap_datapath = True - await self._create_taps() - - async def _create_taps(self): - """ - Create the persistent TAP every adapter owns. Each adapter is born as - a TAP whatever its NIO type — the anchor role Docker's veth host end - plays — so a link attaches to (or detaches from) a running VM without - touching its interfaces. uBridge creates the device and hands - ownership to this user, so the unprivileged QEMU process can open it; - the TAP starts DOWN (carrier off until a link attaches). A persistent - TAP outlives its creator, so a leftover from a previous run (crash, - kill) is swept first. - """ - - for adapter_number, adapter in enumerate(self._ethernet_adapters): - tap = self._tap_name(adapter_number) - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._ubridge_send(f'tap create "{tap}"') - try: - await self._ubridge_send(f"tap set_owner {tap} {os.getuid()}") - except UbridgeError as e: - # Only root could open the TAP now: the VM would fail to - # attach its netdev, which is worth surfacing here rather - # than as a launch failure. - log.warning("QEMU VM '%s': could not hand TAP %s to uid %s: %s", self._name, tap, os.getuid(), e) - await self._ubridge_send(f'link set "{tap}" down') - self._kernel_taps[(adapter_number, 0)] = tap - - async def _remove_taps(self): - """ - Delete the adapter TAPs together with the per-link kernel bridges - this VM still holds: a stopped VM has no links attached, so those - bridges are orphans whose ports just disappeared. - """ - - if self.ubridge: - for tap in self._kernel_taps.values(): - with contextlib.suppress(UbridgeError): - await self._ubridge_send(f'tap delete "{tap}"') - await self._remove_kernel_bridges() - self._kernel_taps.clear() + # Every adapter anchors (port 0 only — QEMU adapters are + # single-port). The shared lifecycle sweeps leftovers first, creates + # each TAP and hands it to this user (unprivileged QEMU opens it + # itself), leaving it down until a link attaches. + await self._create_anchor_taps( + ((adapter_number, 0) for adapter_number, _adapter in enumerate(self._ethernet_adapters)), set_owner=True + ) async def _stop_ubridge(self): """ @@ -1635,8 +1594,7 @@ class QemuVM(KernelDatapathMixin, BaseNode): probed again. """ - if self.ubridge: - await self._remove_taps() + await self._delete_anchor_taps() self._ubridge_tc_caps = None await super()._stop_ubridge() From 96be37baa1f03c0b99b406f67be9ccc4f448113e Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Tue, 6 Oct 2026 22:41:59 +0800 Subject: [PATCH 093/101] test: cover the namespace-failure veth cleanup and BridgeNIO field validation Both fixes from the code-review pass were carried by other tests' side effects: the move_to_ns failure path (the branch that used to skip the delete_veth) and the required-but-nullable bridge field on the NIO create routes. --- tests/api/routes/compute/test_docker_nodes.py | 25 +++++++++++++++++ .../docker/test_docker_kernel_datapath.py | 28 ++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/tests/api/routes/compute/test_docker_nodes.py b/tests/api/routes/compute/test_docker_nodes.py index 40cb31679..c706f34b3 100644 --- a/tests/api/routes/compute/test_docker_nodes.py +++ b/tests/api/routes/compute/test_docker_nodes.py @@ -226,6 +226,31 @@ class TestDockerNodesRoutes: assert response.status_code == status.HTTP_201_CREATED assert response.json()["type"] == "nio_udp" + async def test_docker_nio_create_bridge_requires_the_field_but_takes_null( + self, app: FastAPI, compute_client: AsyncClient, vm: dict + ) -> None: + """ + BridgeNIO.bridge is required-but-nullable: an omitted key is a 422 + (the NIO factories read it, and the routes drop unset fields — an + omitted key used to pass validation and die as a KeyError during + binding), while an explicit null is the absorbed-link shape. + """ + + url = app.url_path_for( + "compute:create_docker_node_nio", + project_id=vm["project_id"], + node_id=vm["node_id"], + adapter_number="0", + port_number="0", + ) + + response = await compute_client.post(url, json={"type": "nio_bridge"}) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + + response = await compute_client.post(url, json={"type": "nio_bridge", "bridge": None}) + assert response.status_code == status.HTTP_201_CREATED + assert response.json()["bridge"] is None + async def test_docker_nio_batch_create(self, app: FastAPI, compute_client: AsyncClient, vm: dict) -> None: """ Exercise the project-wide batch NIO endpoint: bind two NIOs on the same diff --git a/tests/compute/docker/test_docker_kernel_datapath.py b/tests/compute/docker/test_docker_kernel_datapath.py index 853c1999a..def336b51 100644 --- a/tests/compute/docker/test_docker_kernel_datapath.py +++ b/tests/compute/docker/test_docker_kernel_datapath.py @@ -33,7 +33,7 @@ from gns3server.compute.docker import Docker from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error from gns3server.compute.docker.docker_vm import DockerVM from gns3server.compute.nios.nio_bridge import NIOBridge -from gns3server.compute.ubridge.ubridge_error import UbridgeError +from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError from tests.utils import AsyncioMagicMock BRIDGE = "gns3a1b2c3d4e5f" @@ -261,6 +261,32 @@ async def test_add_ubridge_kernel_connection_cleans_up_on_failure(vm): assert (0, 0) not in vm._kernel_veths +@pytest.mark.asyncio +async def test_create_veth_cleans_up_when_the_namespace_move_fails(vm): + """ + ``move_to_ns`` failing (the container died between create and move) + raises UbridgeNamespaceError — and must take the same cleanup path as + any other failure: the old branch re-raised before the delete_veth, + leaving the half-created pair behind until the next start's sweep. + """ + + vm._ubridge_hypervisor = MagicMock() + vm._namespace = 42 + + async def failing_send(command): + if command.startswith("docker move_to_ns"): + raise UbridgeError("Could not move interface: No such process") + + vm._ubridge_send = AsyncioMagicMock(side_effect=failing_send) + + with pytest.raises(UbridgeNamespaceError): + await vm._create_veth(0) + + host_ifc, _ = vm._veth_names(0, 0) + vm._ubridge_send.assert_any_call(f'docker delete_veth "{host_ifc}"') + assert vm._kernel_veths == {} + + # --------------------------------------------------------------------------- # Link attach / detach # --------------------------------------------------------------------------- From 6459122113c892b9251eff851345868d2f4fc6a1 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 7 Oct 2026 13:19:38 +0800 Subject: [PATCH 094/101] test(e2e): run every traffic checkpoint dual-stack (IPv6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live suite asserted the datapath over IPv4 only. Give every guest a ULA IPv6 address next to its v4 one and twin every v4 assertion point, sharing the impairment window (one filter application, both families asserted) so the runtime grows by minutes, not double. - harness: configure_ios(eth_ipv6=), ping() dispatches to `ping ipv6`, docker_bring_up_ipv6 (the Docker daemon leaves eth0 with disable_ipv6=1 on v6-less bridge networks, so the interfaces-file route cannot work), family-aware docker_wait_address, pcap_records scans every record (a leading DAD/MLD frame must not mask the sampled ethertype), and pcap_icmp_rtt measures the wire round trip from capture timestamps; - scenarios: connectivity/delay/suspend/isolation/churn checks in both families, captures assert 86dd next to 0800, the libpcap bpf drops are asserted family-selective (icmp vs icmp6, each killing its own family and leaving the other alive), the eBPF classifier modes on both families, marker v6 twins with an echo-specific expression (bare icmp6 also matches DAD/MLD and would break the exact counts), the v6 sharkd replay, and the link-local matrix gains the 86dd all-nodes row; serial (HDLC) legs stay v4-only; - the dynamips-emulated classic IOS under-reports its `ping ipv6` RTT by ~40 % of a delayed round trip (pcap-verified: the wire carries the v6 echo exchange in the same ~230 ms as v4 while the console reports ~130 ms), so the dynamips-family scenarios assert the v6 delay from the capture's own timestamps — recorded in tests/e2e/README.md. Full suite: 17 passed in 58m50s. --- tests/e2e/README.md | 10 + tests/e2e/harness.py | 126 ++++++++-- tests/e2e/test_docker_kernel_datapath.py | 221 ++++++++++++++++-- tests/e2e/test_docker_link_local_frames.py | 14 +- tests/e2e/test_dynamips_kernel_datapath.py | 61 ++++- tests/e2e/test_ethernet_switch_cascade.py | 57 ++++- .../test_ethernet_switch_kernel_datapath.py | 71 +++++- tests/e2e/test_iol_docker_kernel_datapath.py | 60 ++++- tests/e2e/test_iou_kernel_datapath.py | 111 ++++++++- tests/e2e/test_marker_kernel_datapath.py | 143 ++++++++++-- tests/e2e/test_qemu_kernel_datapath.py | 40 +++- 11 files changed, 809 insertions(+), 105 deletions(-) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index ef5a1ef82..fba5b9671 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -66,6 +66,16 @@ Current scenarios: The Docker scenarios run a digest-pinned `alpine:3`, so every developer tests the same bytes: a cached copy in the local Docker daemon costs no network, a cache miss pulls through the server's own image-pull route, and with neither cache nor registry the scenario skips (`GNS3_E2E_DOCKER_IMAGE` overrides the reference). Their guests are addressed the way a GNS3 user does it — the node's persistent `/etc/network/interfaces`, which the container's `init.sh` applies at boot — and driven through the Docker daemon socket the server itself uses. +## Dual-stack coverage + +Every scenario runs its traffic checkpoints **dual-stack**: the guests carry a ULA IPv6 address next to their IPv4 one (a `fd00:a:b::h` mirror of the v4 `10.a.b.h`), and each checkpoint pings both families — the v6 twin rides the same impairment window as its v4 counterpart rather than a second filter application, so the impairment set is applied once and asserted twice. The serial (HDLC) legs stay v4-only: they are not Ethernet wires. + +The family-sensitive spots get dedicated v6 assertions: the libpcap `bpf` drop filters are asserted to kill their own family and leave the other family alive (`icmp` vs `icmp6`); capture windows carry both families and assert the `86dd` ethertype next to `0800`; the eBPF classifier modes (family-blind by construction) are measured on both families. Marker twins use an echo-specific expression (`icmp6 and (ip6[40] == 128 or ip6[40] == 129)`) because a bare `icmp6` also matches the guests' own DAD/MLD/NS/NA control frames and would break the exact-count asserts — the per-family exact counts (v4 marker sees only v4 frames, v6 marker only v6) double as the cross-family check. + +Docker guests get their v6 address from `harness.docker_bring_up_ipv6` after every start rather than from the interfaces file: on bridge networks without IPv6 the Docker daemon leaves the container interface with `disable_ipv6=1`, so busybox ifup's inet6 stanza would fail with EPERM at boot. The helper switches the interface on, silences DAD transmissions and adds the address; since it does not survive a stop/start, it is re-applied at every container start. + +One platform quirk is recorded here because it looks like a datapath bug and is not one: while a netem delay is applied, the dynamips-emulated classic IOS **under-reports its own `ping ipv6` RTT** by ~40 % of the delayed round trip — a pcap on the anchor shows the v6 echo request→reply taking the same ~230 ms as v4 while the console reports ~130 ms, and with a small single-owned delay the report swallows most of the signal (reproducible; the order of the two pings does not matter; the v4 ping on the same platform reports honestly). The dynamips-family scenarios (dynamips, ethernet switch, cascade) therefore assert their **v6 delay from the capture's own timestamps** (`harness.pcap_icmp_rtt` — the wire truth, immune to any guest's stopwatch), while the absolute v6 ping values are asserted where the guests read the real clock (the Docker scenarios' busybox guests and the native IOL/IOS-XE guests). + Every kernel scenario also asserts the L2-anchor spec's server-side guarantees (`docs/design/ubridge-l2-anchor-spec.md` §E): anchors and bridges carry no L3 identity (`assert_pure_l2`), joined ports are FORWARDING (`assert_forwarding`), and an idle link with the guests silenced stays silent for 5 s (`assert_idle_silence`) — the guests are the IOS consoles' interfaces shut, the Docker guests' links down, or all four cascade routers silenced, whichever the scenario runs. These skip on a uBridge without `link l2only` (probed once by `harness.l2only_supported()`). ## Host environment notes diff --git a/tests/e2e/harness.py b/tests/e2e/harness.py index 217fe2dbc..5dc48b990 100644 --- a/tests/e2e/harness.py +++ b/tests/e2e/harness.py @@ -702,9 +702,14 @@ def ios_console(server, project_id, node_id): return IOSConsole(host, node["console"]) -def configure_ios(console, hostname, eth_ip=None, serial_ip=None, clock=False, eth_if="f1/0", serial_if="s2/0"): +def configure_ios( + console, hostname, eth_ip=None, eth_ipv6=None, serial_ip=None, clock=False, eth_if="f1/0", serial_if="s2/0" +): """Baseline config on a freshly booted router: hostname, no console spam, - optional interface addresses.""" + optional interface addresses. ``eth_ipv6`` configures the same interface + dual-stack (a /64 prefix); ipv6 unicast-routing is deliberately left off — + the routers stay pure hosts, so no RAs go out and the link stays quiet + (directly connected pings need no forwarding).""" console.run("enable") console.run("conf t") console.run(f"hostname {hostname}") @@ -712,9 +717,12 @@ def configure_ios(console, hostname, eth_ip=None, serial_ip=None, clock=False, e # global form: console logging off in one step, no line sub-mode to lose # characters in console.run("no logging console") - if eth_ip: + if eth_ip or eth_ipv6: console.run(f"interface {eth_if}") - console.run(f"ip address {eth_ip} 255.255.255.0") + if eth_ip: + console.run(f"ip address {eth_ip} 255.255.255.0") + if eth_ipv6: + console.run(f"ipv6 address {eth_ipv6}/64") console.run("no shutdown") console.run("exit") if serial_ip: @@ -737,8 +745,12 @@ FAST_RTT_MS = 100 def ping(console, target, repeat=3, timeout=1): - """Run an IOS ping; returns {success, avg, raw} (success in percent).""" - out = console.run(f"ping {target} repeat {repeat} timeout {timeout}", timeout=repeat * timeout + 20) + """Run an IOS ping; returns {success, avg, raw} (success in percent). + + An IPv6 literal needs IOS's ``ping ipv6`` form; the summary lines the + parsers read are the same shape for both families.""" + family = "ipv6 " if ":" in target else "" + out = console.run(f"ping {family}{target} repeat {repeat} timeout {timeout}", timeout=repeat * timeout + 20) success = re.search(r"Success rate is (\d+) percent", out) rtt = re.search(r"round-trip min/avg/max = (\d+)/(\d+)/(\d+)", out) return {"success": int(success.group(1)) if success else 0, "avg": int(rtt.group(2)) if rtt else None, "raw": out} @@ -968,7 +980,10 @@ def configure_docker_interfaces(node, address, netmask="255.255.255.0", adapter= user does: write the persistent ``/etc/network/interfaces`` the server bind-mounts into the container and init.sh applies with busybox ifup at boot (CIDR is not understood — separate address/netmask lines). Call - before starting the node.""" + before starting the node. + + IPv4 only: an inet6 stanza would never apply — see + ``docker_bring_up_ipv6``.""" directory = node.get("node_directory") if not directory or not os.path.isdir(directory): pytest.skip(f"cannot reach the node directory {directory!r} — this scenario needs the compute on this host") @@ -978,12 +993,45 @@ def configure_docker_interfaces(node, address, netmask="255.255.255.0", adapter= f.write(f"auto eth{adapter}\niface eth{adapter} inet static\n\taddress {address}\n\tnetmask {netmask}\n") -def docker_wait_address(daemon, container_id, address, timeout=30): - """Wait until the container's eth0 carries *address* — init.sh applies - the interfaces file (busybox ifup) a moment after container start.""" +def docker_bring_up_ipv6(daemon, container_id, address6, adapter=0): + """Give the container's eth{adapter} its global IPv6 address. Call + after every container start (the address does not survive a stop: the + netns is rebuilt). + + IPv6 cannot ride the interfaces file here: on bridge networks without + IPv6 the Docker daemon leaves the container interface with + ``disable_ipv6=1``, which makes busybox ifup's ``ip -6 addr add`` fail + with EPERM before init.sh's ``ifup -a`` ever finishes the stanza — the + interface must be switched on by hand first. DAD transmissions are + silenced before the address is added: the kernel still holds the + address tentative for ~1 s (measured; ~2 s with the default one DAD + solicitation), but nothing here tests DAD and the solicitations are one + less source of boot-time chatter on the wire.""" + code, output = daemon.exec( + container_id, + [ + "sh", + "-c", + f"echo 0 > /proc/sys/net/ipv6/conf/eth{adapter}/dad_transmits" + f" && echo 0 > /proc/sys/net/ipv6/conf/eth{adapter}/disable_ipv6" + f" && ip -6 addr add {address6}/64 dev eth{adapter}", + ], + ) + assert code == 0, output + + +def docker_wait_address(daemon, container_id, address, timeout=30, adapter=0): + """Wait until the container's eth{adapter} carries *address* (either + family) — init.sh applies the interfaces file (busybox ifup) a moment + after container start. A v6 address also has to leave the tentative + state (a tentative address answers no neighbour solicitation, so pings + to it cannot resolve).""" + family = "-6" if ":" in address else "-4" def present(): - _code, output = daemon.exec(container_id, ["ip", "-4", "-o", "addr", "show", "dev", "eth0"]) + _code, output = daemon.exec(container_id, ["ip", family, "-o", "addr", "show", "dev", f"eth{adapter}"]) + if family == "-6": + return address.lower() in output.lower() and "tentative" not in output.lower() return address in output if not wait_until(present, timeout=timeout, interval=0.5): @@ -1319,11 +1367,14 @@ def wait_until(predicate, timeout=10, interval=0.3): def pcap_records(path): - """Count the records of a pcap file and collect the ethertype of the - first few (hex, e.g. '0800') — enough to prove a capture is real.""" + """Count the records of a pcap file and collect the distinct ethertypes + across all of them (hex, e.g. '0800') — enough to prove a capture is + real. Scans every record, not a leading sample: a dual-stack host's + own IPv6 control frames (DAD/MLD) can precede the traffic under test, + and a truncated sample would let them mask it.""" import struct - ethertypes = [] + ethertypes = set() with open(path, "rb") as f: header = f.read(24) if len(header) < 24: @@ -1339,6 +1390,49 @@ def pcap_records(path): if len(data) < caplen: break count += 1 - if len(ethertypes) < 5 and caplen >= 14: - ethertypes.append(data[12:14].hex()) + if caplen >= 14: + ethertypes.add(data[12:14].hex()) return count, ethertypes + + +def pcap_icmp_rtt(path, ethertype): + """The mean echo request->reply RTT (ms) in a capture, from the + capture's own timestamps — the wire truth, independent of whatever a + guest's ping reports (the dynamips-emulated classic IOS under-reports + its ``ping ipv6`` RTT by roughly 40 % of a delayed round trip; this + primitive is what the dynamips-family scenarios assert instead). + + Pairs each echo request with the next echo reply (``ethertype`` is + '0800' or '86dd'; ICMP echo type 8/0 for v4, 128/129 for v6 with the + fixed 40-byte IPv6 header — no extension headers in these pings). + Returns None when the capture holds no complete exchange.""" + import struct + + rows = [] + with open(path, "rb") as f: + header = f.read(24) + if len(header) < 24: + return None + endian = "<" if header[:4] in (b"\xd4\xc3\xb2\xa1", b"\x4d\x3c\xb2\xa1") else ">" + while True: + rec = f.read(16) + if len(rec) < 16: + break + ts_s, ts_us, caplen, _orig = struct.unpack(endian + "IIII", rec) + data = f.read(caplen) + if len(data) < caplen: + break + if caplen < 14 or data[12:14].hex() != ethertype: + continue + icmp_off = 14 + ((data[14] & 0xF) * 4 if ethertype == "0800" else 40) + if caplen >= icmp_off + 1: + rows.append((ts_s + ts_us / 1e6, data[icmp_off])) + request_type = 8 if ethertype == "0800" else 128 + deltas, pending = [], None + for ts, icmp_type in rows: + if icmp_type == request_type: + pending = ts + elif icmp_type == request_type + 1 and pending is not None: + deltas.append((ts - pending) * 1000) + pending = None + return sum(deltas) / len(deltas) if deltas else None diff --git a/tests/e2e/test_docker_kernel_datapath.py b/tests/e2e/test_docker_kernel_datapath.py index 04c7d5e50..f3473ef55 100644 --- a/tests/e2e/test_docker_kernel_datapath.py +++ b/tests/e2e/test_docker_kernel_datapath.py @@ -49,6 +49,13 @@ server itself uses. The scenarios cover: The image is pinned by digest (``harness.ensure_docker_image``): a cached copy costs no network, a miss pulls through the server's own pull route, and no cache + no registry skips the scenario. + +Every traffic assertion runs dual-stack: the guests carry a ULA IPv6 +address next to their IPv4 one and each checkpoint pings both families — +including the filter matrix (netem and the classifiers are family-blind; +the libpcap bpf drops are deliberately not) and the relay markers (a v6 +twin marker with an echo-specific expression, since a bare ``icmp6`` +would also match the guests' own DAD/MLD control frames). """ import os @@ -65,6 +72,16 @@ pytestmark = pytest.mark.e2e ETH = (0, 0) R1_IP = "10.1.1.1" R2_IP = "10.1.1.2" +# the dual-stack half: a ULA mirroring the v4 addressing (10.1.1.x -> fd00:1:1::x) +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" + +# exact ICMPv6 echo match for markers: a bare "icmp6" would also match the +# guests' own DAD/MLD/NS/NA control frames and break the exact-count asserts +_ICMP6_ECHO_BPF = "icmp6 and (ip6[40] == 128 or ip6[40] == 129)" +# busybox ping frame lengths (56 data bytes): 14 Ethernet + 20/40 IP + 8 ICMP + 56 +PING_FRAME_LEN = 98 +PING_FRAME_LEN6 = 118 def _resolve_container(daemon, node): @@ -78,8 +95,9 @@ def _resolve_container(daemon, node): def _create_topology(server, pid, image, name): - """Two containers with addressed eth0s. The harness reaches their guests - through the Docker daemon socket — the same socket the server uses.""" + """Two containers with dual-stack addressed eth0s. The harness reaches + their guests through the Docker daemon socket — the same socket the + server uses.""" compute = server.compute daemon = harness.DockerDaemon() nodes = [] @@ -91,12 +109,17 @@ def _create_topology(server, pid, image, name): def _start_addresses(compute, pid, daemon, nodes, addresses): - """Start the containers and wait for init.sh to have applied their - addresses (busybox ifup runs a moment after container start).""" + """Start the containers and wait for init.sh to have applied their v4 + addresses (busybox ifup runs a moment after container start); the v6 + addresses are brought up by the harness after each start (see + ``harness.docker_bring_up_ipv6`` — an interface-side disable_ipv6 the + Docker daemon sets makes the interfaces-file route impossible).""" for node in nodes: compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start") - for node, address in zip(nodes, addresses, strict=False): + for node, (address, address6) in zip(nodes, addresses, strict=False): harness.docker_wait_address(daemon, node["container_id"], address) + harness.docker_bring_up_ipv6(daemon, node["container_id"], address6) + harness.docker_wait_address(daemon, node["container_id"], address6) def _put_filters(compute, pid, lid, filters): @@ -134,9 +157,14 @@ def _silence_guest(daemon, container_id): assert code == 0, output -def _unsilence_guest(daemon, container_id): +def _unsilence_guest(daemon, container_id, address6): + """Reverse of ``_silence_guest``. disable_ipv6=1 removed the v6 + addresses and busybox ifup does not re-run, so the guest's global + address must be brought up again by hand (the v4 address stayed + configured throughout).""" code, output = daemon.exec(container_id, ["ip", "link", "set", "eth0", "up"]) assert code == 0, output + harness.docker_bring_up_ipv6(daemon, container_id, address6) def test_docker_kernel_fast_path(): @@ -164,7 +192,7 @@ def test_docker_kernel_fast_path(): assert not harness.tap_exists(a1), "the veth pair is born at container start" print(".. starting containers") - _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + _start_addresses(compute, pid, daemon, (n1, n2), ((R1_IP, R1_IP6), (R2_IP, R2_IP6))) # Kernel objects: the veth host ends exist and the per-link bridge # enslaves exactly the two of them. @@ -187,6 +215,9 @@ def test_docker_kernel_fast_path(): baseline = harness.docker_wait_ping(daemon, c1, R2_IP) assert baseline["loss"] == 0, baseline["raw"] assert baseline["avg"] is not None and baseline["avg"] < 50, baseline + baseline6 = harness.docker_wait_ping(daemon, c1, R2_IP6) + assert baseline6["loss"] == 0, baseline6["raw"] + assert baseline6["avg"] is not None and baseline6["avg"] < 50, baseline6 # §E.2: with the guests silenced the segment is silent for 5 s — # the host anchors do not generate the MLD/DAD flood an unhardened @@ -195,12 +226,14 @@ def test_docker_kernel_fast_path(): _silence_guest(daemon, c1) _silence_guest(daemon, c2) harness.assert_idle_silence(a1, a2, bridge) - _unsilence_guest(daemon, c1) - _unsilence_guest(daemon, c2) + _unsilence_guest(daemon, c1, R1_IP6) + _unsilence_guest(daemon, c2, R2_IP6) assert harness.docker_wait_ping(daemon, c1, R2_IP, attempts=5)["loss"] == 0 + assert harness.docker_wait_ping(daemon, c1, R2_IP6, attempts=5)["loss"] == 0 # delay 100: netem on both host ends (each impairs one direction), - # one-way ~100 ms => RTT grows by ~200 ms + # one-way ~100 ms => RTT grows by ~200 ms — netem is family-blind, + # both families ride the same qdisc _put_filters(compute, pid, lid, {"delay": [100]}) assert "netem" in harness.qdiscs(a1) and "netem" in harness.qdiscs(a2), ( harness.qdiscs(a1), @@ -208,10 +241,14 @@ def test_docker_kernel_fast_path(): ) delayed = harness.docker_wait_ping(daemon, c1, R2_IP) assert delayed["loss"] == 0 and delayed["avg"] >= 150, (baseline, delayed) + delayed6 = harness.docker_wait_ping(daemon, c1, R2_IP6) + assert delayed6["loss"] == 0 and delayed6["avg"] >= 150, (baseline6, delayed6) _put_filters(compute, pid, lid, {}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.docker_wait_ping(daemon, c1, R2_IP) assert fast["loss"] == 0 and fast["avg"] < 50, fast + fast6 = harness.docker_wait_ping(daemon, c1, R2_IP6) + assert fast6["loss"] == 0 and fast6["avg"] < 50, fast6 # suspend: the host end admin-downs (carrier loss into the # container), the link is dead; resume restores @@ -219,19 +256,25 @@ def test_docker_kernel_fast_path(): assert not harness.tap_up(a1) dead = harness.docker_ping(daemon, c1, R2_IP, count=3, timeout=1) assert dead["loss"] == 100, dead["raw"] + dead6 = harness.docker_ping(daemon, c1, R2_IP6, count=3, timeout=1) + assert dead6["loss"] == 100, dead6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) assert harness.tap_up(a1) assert harness.docker_wait_ping(daemon, c1, R2_IP)["loss"] == 0 + assert harness.docker_wait_ping(daemon, c1, R2_IP6)["loss"] == 0 - # capture: AF_PACKET on the anchor writes a real pcap + # capture: AF_PACKET on the anchor writes a real pcap — both + # families inside the window, both ethertypes on the wire capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) harness.docker_ping(daemon, c1, R2_IP) + harness.docker_ping(daemon, c1, R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # link delete detaches both ends (the veths survive — they are the # adapters); re-creating enslaves them again and the traffic returns @@ -243,6 +286,7 @@ def test_docker_kernel_fast_path(): bridge = harness.link_bridge_name(lid) assert link["kernel_datapath"] is True, link assert harness.docker_wait_ping(daemon, c1, R2_IP, attempts=5)["loss"] == 0 + assert harness.docker_wait_ping(daemon, c1, R2_IP6, attempts=5)["loss"] == 0 # container stop: the veth host ends are deleted explicitly (a veth # outlives its container's netns) and the bridge goes with them; a @@ -253,9 +297,12 @@ def test_docker_kernel_fast_path(): assert harness.bridge_members(bridge) in (None, [a2]), harness.bridge_members(bridge) compute.call("POST", f"/projects/{pid}/nodes/{n1_id}/start") harness.docker_wait_address(daemon, c1, R1_IP) + harness.docker_bring_up_ipv6(daemon, c1, R1_IP6) + harness.docker_wait_address(daemon, c1, R1_IP6) assert harness.tap_exists(a1) assert harness.wait_until(lambda: harness.bridge_members(bridge) == sorted([a1, a2]), timeout=15) assert harness.docker_wait_ping(daemon, c1, R2_IP, attempts=5)["loss"] == 0 + assert harness.docker_wait_ping(daemon, c1, R2_IP6, attempts=5)["loss"] == 0 except BaseException: harness.release(server, pid, failed=True) raise @@ -293,9 +340,11 @@ def test_docker_kernel_filter_matrix(): lid = link["link_id"] assert link["kernel_datapath"] is True, link - _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + _start_addresses(compute, pid, daemon, (n1, n2), ((R1_IP, R1_IP6), (R2_IP, R2_IP6))) baseline = harness.docker_wait_ping(daemon, c1, R2_IP) assert baseline["loss"] == 0, baseline["raw"] + baseline6 = harness.docker_wait_ping(daemon, c1, R2_IP6) + assert baseline6["loss"] == 0, baseline6["raw"] # The kernel link offers the kernel-only types the compute's uBridge # reports it can run (all of them on this uBridge). @@ -314,6 +363,9 @@ def test_docker_kernel_filter_matrix(): delayed = harness.docker_ping(daemon, c1, R2_IP) print(f".. {delayed['avg']} ms RTT") assert delayed["loss"] == 0 and delayed["avg"] >= 150, (baseline, delayed) + delayed6 = harness.docker_ping(daemon, c1, R2_IP6) + print(f".. {delayed6['avg']} ms RTT (v6)") + assert delayed6["loss"] == 0 and delayed6["avg"] >= 150, (baseline6, delayed6) print(".. delay 50 jitter 20 distribution normal") _put_filters(compute, pid, lid, {"delay": [50, 20, "normal"]}) @@ -325,6 +377,9 @@ def test_docker_kernel_filter_matrix(): jittered = harness.docker_ping(daemon, c1, R2_IP, count=10) print(f".. {jittered['avg']} ms RTT") assert jittered["loss"] == 0 and jittered["avg"] >= 65, jittered + jittered6 = harness.docker_ping(daemon, c1, R2_IP6, count=10) + print(f".. {jittered6['avg']} ms RTT (v6)") + assert jittered6["loss"] == 0 and jittered6["avg"] >= 65, jittered6 _clear_filters(compute, pid, lid, a1, a2) print(".. packet_loss 30") @@ -334,6 +389,9 @@ def test_docker_kernel_filter_matrix(): # round trip = 1 - 0.7^2 ~ 51 %; the stochastic draw must land # somewhere between "clearly lossy" and "not a black hole" assert 20 <= lossy["loss"] <= 80, lossy["raw"] + lossy6 = harness.docker_ping(daemon, c1, R2_IP6, count=20) + print(f".. {lossy6['loss']} % round-trip loss (v6)") + assert 20 <= lossy6["loss"] <= 80, lossy6["raw"] _clear_filters(compute, pid, lid, a1, a2) print(".. corrupt 30") @@ -344,6 +402,9 @@ def test_docker_kernel_filter_matrix(): corrupt = harness.docker_ping(daemon, c1, R2_IP, count=20) print(f".. {corrupt['loss']} % round-trip loss") assert 20 <= corrupt["loss"] <= 80, corrupt["raw"] + corrupt6 = harness.docker_ping(daemon, c1, R2_IP6, count=20) + print(f".. {corrupt6['loss']} % round-trip loss (v6)") + assert 20 <= corrupt6["loss"] <= 80, corrupt6["raw"] _clear_filters(compute, pid, lid, a1, a2) print(".. duplicate 50") @@ -352,6 +413,8 @@ def test_docker_kernel_filter_matrix(): # duplicated frames are absorbed by the stack: no loss, traffic flows dups = harness.docker_ping(daemon, c1, R2_IP, count=5) assert dups["loss"] == 0, dups["raw"] + dups6 = harness.docker_ping(daemon, c1, R2_IP6, count=5) + assert dups6["loss"] == 0, dups6["raw"] _clear_filters(compute, pid, lid, a1, a2) # -- netem extensions (kernel-only) -------------------------------- @@ -360,6 +423,8 @@ def test_docker_kernel_filter_matrix(): assert "reorder" in harness.qdiscs(a1), harness.qdiscs(a1) reordered = harness.docker_ping(daemon, c1, R2_IP, count=10) assert reordered["loss"] <= 60, reordered["raw"] + reordered6 = harness.docker_ping(daemon, c1, R2_IP6, count=10) + assert reordered6["loss"] <= 60, reordered6["raw"] _clear_filters(compute, pid, lid, a1, a2) print(".. delay 50 + seed 42 + limit 5000") @@ -367,6 +432,8 @@ def test_docker_kernel_filter_matrix(): assert "limit 5000" in harness.qdiscs(a1), harness.qdiscs(a1) seeded = harness.docker_ping(daemon, c1, R2_IP) assert seeded["loss"] == 0 and seeded["avg"] >= 80, seeded + seeded6 = harness.docker_ping(daemon, c1, R2_IP6) + assert seeded6["loss"] == 0 and seeded6["avg"] >= 80, seeded6 _clear_filters(compute, pid, lid, a1, a2) print(".. gemodel 100/0/30 (steady-state bad, 30 % loss)") @@ -375,6 +442,9 @@ def test_docker_kernel_filter_matrix(): gemodel = harness.docker_ping(daemon, c1, R2_IP, count=20) print(f".. {gemodel['loss']} % round-trip loss") assert 20 <= gemodel["loss"] <= 90, gemodel["raw"] + gemodel6 = harness.docker_ping(daemon, c1, R2_IP6, count=20) + print(f".. {gemodel6['loss']} % round-trip loss (v6)") + assert 20 <= gemodel6["loss"] <= 90, gemodel6["raw"] _clear_filters(compute, pid, lid, a1, a2) print(".. rate 512kbit (1400-byte pings)") @@ -384,15 +454,28 @@ def test_docker_kernel_filter_matrix(): rated = harness.docker_ping(daemon, c1, R2_IP, size=1400) print(f".. {rated['avg']} ms RTT") assert rated["loss"] == 0 and rated["avg"] >= 30, (baseline, rated) + rated6 = harness.docker_ping(daemon, c1, R2_IP6, size=1400) + print(f".. {rated6['avg']} ms RTT (v6)") + assert rated6["loss"] == 0 and rated6["avg"] >= 30, (baseline6, rated6) _clear_filters(compute, pid, lid, a1, a2) # -- cls_bpf match-drop -------------------------------------------- if caps.get("cbpf"): - print(".. bpf 'icmp' drops everything") + print(".. bpf 'icmp' drops v4 ICMP only") _put_filters(compute, pid, lid, {"bpf": ["icmp"]}) assert "clsact" in harness.qdiscs(a1), harness.qdiscs(a1) dropped = harness.docker_ping(daemon, c1, R2_IP, count=4) assert dropped["loss"] == 100, dropped["raw"] + # the libpcap expression is family-specific: v6 ICMP sails through + v6_alive = harness.docker_ping(daemon, c1, R2_IP6, count=4) + assert v6_alive["loss"] == 0, v6_alive["raw"] + + print(".. bpf 'icmp6' drops v6 ICMP only") + _put_filters(compute, pid, lid, {"bpf": ["icmp6"]}) + v4_alive = harness.docker_ping(daemon, c1, R2_IP, count=4) + assert v4_alive["loss"] == 0, v4_alive["raw"] + dropped6 = harness.docker_ping(daemon, c1, R2_IP6, count=4) + assert dropped6["loss"] == 100, dropped6["raw"] print(".. bpf 'greater 150' discriminates by frame size") _put_filters(compute, pid, lid, {"bpf": ["greater 150"]}) @@ -400,6 +483,12 @@ def test_docker_kernel_filter_matrix(): assert small["loss"] == 0, small["raw"] big = harness.docker_ping(daemon, c1, R2_IP, count=4, size=300) assert big["loss"] == 100, big["raw"] + # the v6 echo frame is 118 bytes — still under the threshold; + # size=300 pushes it to 362, over it + small6 = harness.docker_ping(daemon, c1, R2_IP6, count=4) + assert small6["loss"] == 0, small6["raw"] + big6 = harness.docker_ping(daemon, c1, R2_IP6, count=4, size=300) + assert big6["loss"] == 100, big6["raw"] _clear_filters(compute, pid, lid, a1, a2) else: print(".. uBridge reports no cbpf: skipping the bpf filter checks") @@ -413,9 +502,16 @@ def test_docker_kernel_filter_matrix(): print(f".. {nth['loss']} % round-trip loss") # kernel counts per direction: 1 - (2/3)^2 = 55.6 % round trip assert 30 <= nth["loss"] <= 80, nth["raw"] + # the classifier draws per packet, not per header — v6 rides + # the same distribution + nth6 = harness.docker_ping(daemon, c1, R2_IP6, count=20) + print(f".. {nth6['loss']} % round-trip loss (v6)") + assert 30 <= nth6["loss"] <= 80, nth6["raw"] _put_filters(compute, pid, lid, {"frequency_drop": [-1]}) all_dropped = harness.docker_ping(daemon, c1, R2_IP, count=4) assert all_dropped["loss"] == 100, all_dropped["raw"] + all_dropped6 = harness.docker_ping(daemon, c1, R2_IP6, count=4) + assert all_dropped6["loss"] == 100, all_dropped6["raw"] _clear_filters(compute, pid, lid, a1, a2) if "quota" in modes: print(".. quota 3000 bytes hard cutoff") @@ -426,8 +522,13 @@ def test_docker_kernel_filter_matrix(): assert cut["loss"] >= 40, cut["raw"] after = harness.docker_ping(daemon, c1, R2_IP, count=4) assert after["loss"] == 100, after["raw"] + cut6 = harness.docker_ping(daemon, c1, R2_IP6, count=5, size=1400) + assert cut6["loss"] >= 40, cut6["raw"] + after6 = harness.docker_ping(daemon, c1, R2_IP6, count=4) + assert after6["loss"] == 100, after6["raw"] _clear_filters(compute, pid, lid, a1, a2) assert harness.docker_ping(daemon, c1, R2_IP, count=4)["loss"] == 0 + assert harness.docker_ping(daemon, c1, R2_IP6, count=4)["loss"] == 0 if "window" in modes: print(".. window_drop [1000, 2000, 100] — one outage with passes on both sides") _put_filters(compute, pid, lid, {"window_drop": [1000, 2000, 100]}) @@ -438,8 +539,19 @@ def test_docker_kernel_filter_matrix(): missing = set(range(25)) - replied assert 0 in replied and 24 in replied, (sorted(replied), output) assert missing and max(missing) - min(missing) >= 5, (sorted(replied), output) + # the outage window is one-shot from the moment the filter is + # applied (documented semantics), so the v6 stream needs the + # schedule re-armed — re-sending the same window is a no-op + # except that it restarts the schedule + _put_filters(compute, pid, lid, {"window_drop": [1000, 2000, 100]}) + code, output = daemon.exec(c1, ["ping", "-c", "25", "-i", "0.2", "-W", "1", R2_IP6], timeout=30) + replied6 = {int(m.group(1)) for m in re.finditer(r"seq=(\d+)", output)} + missing6 = set(range(25)) - replied6 + assert 0 in replied6 and 24 in replied6, (sorted(replied6), output) + assert missing6 and max(missing6) - min(missing6) >= 5, (sorted(replied6), output) _clear_filters(compute, pid, lid, a1, a2) assert harness.docker_ping(daemon, c1, R2_IP, count=4)["loss"] == 0 + assert harness.docker_ping(daemon, c1, R2_IP6, count=4)["loss"] == 0 except BaseException: harness.release(server, pid, failed=True) raise @@ -471,7 +583,7 @@ def test_docker_relay_control(): link = compute.create_link(pid, (n1_id, *ETH), (n2_id, *ETH)) assert link["kernel_datapath"] is False, link - _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + _start_addresses(compute, pid, daemon, (n1, n2), ((R1_IP, R1_IP6), (R2_IP, R2_IP6))) # No kernel wiring: no per-link bridge, the veth host end is just a # relay endpoint (up, pure L2 — the hardening is a creation-time @@ -484,6 +596,8 @@ def test_docker_relay_control(): relay_ping = harness.docker_wait_ping(daemon, n1["container_id"], R2_IP) assert relay_ping["loss"] == 0, relay_ping["raw"] + relay_ping6 = harness.docker_wait_ping(daemon, n1["container_id"], R2_IP6) + assert relay_ping6["loss"] == 0, relay_ping6["raw"] # available_filters hides the kernel-only types on the relay offered = _filter_types(compute, pid, link["link_id"]) @@ -502,9 +616,14 @@ def test_docker_relay_control(): delayed = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5) assert delayed["loss"] == 0, delayed["raw"] assert delayed["avg"] >= 150, (relay_ping, delayed) + delayed6 = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=5) + assert delayed6["loss"] == 0, delayed6["raw"] + assert delayed6["avg"] >= 150, (relay_ping6, delayed6) _put_filters(compute, pid, link["link_id"], {}) fast = harness.docker_wait_ping(daemon, n1["container_id"], R2_IP) assert fast["loss"] == 0 and fast["avg"] < 50, (relay_ping, fast) + fast6 = harness.docker_wait_ping(daemon, n1["container_id"], R2_IP6) + assert fast6["loss"] == 0 and fast6["avg"] < 50, (relay_ping6, fast6) # frequency_drop counts every packet crossing the bridge, both # directions sharing one counter, so on an alternating ping stream @@ -516,24 +635,43 @@ def test_docker_relay_control(): dropped = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=10) print(f".. {dropped['loss']} % round-trip loss") assert dropped["loss"] >= 50, dropped["raw"] + dropped6 = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=10) + print(f".. {dropped6['loss']} % round-trip loss (v6)") + assert dropped6["loss"] >= 50, dropped6["raw"] _put_filters(compute, pid, link["link_id"], {"frequency_drop": [-1]}) blackholed = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) assert blackholed["loss"] == 100, blackholed["raw"] + blackholed6 = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=4) + assert blackholed6["loss"] == 100, blackholed6["raw"] # bpf on the relay is uBridge's libpcap userspace match-drop — a - # different engine from the kernel datapath's cls_bpf - print(".. relay filters: bpf 'icmp' drops every ICMP frame") + # different engine from the kernel datapath's cls_bpf, same + # family-specific expression semantics + print(".. relay filters: bpf 'icmp' drops v4 ICMP only") _put_filters(compute, pid, link["link_id"], {"bpf": ["icmp"]}) icmp_dropped = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) assert icmp_dropped["loss"] == 100, icmp_dropped["raw"] + v6_alive = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=4) + assert v6_alive["loss"] == 0, v6_alive["raw"] + print(".. relay filters: bpf 'icmp6' drops v6 ICMP only") + _put_filters(compute, pid, link["link_id"], {"bpf": ["icmp6"]}) + v4_alive = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) + assert v4_alive["loss"] == 0, v4_alive["raw"] + icmp6_dropped = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=4) + assert icmp6_dropped["loss"] == 100, icmp6_dropped["raw"] print(".. relay filters: bpf 'greater 150' discriminates by frame size") _put_filters(compute, pid, link["link_id"], {"bpf": ["greater 150"]}) small = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4) assert small["loss"] == 0, small["raw"] big = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=4, size=300) assert big["loss"] == 100, big["raw"] + small6 = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=4) + assert small6["loss"] == 0, small6["raw"] + big6 = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=4, size=300) + assert big6["loss"] == 100, big6["raw"] _put_filters(compute, pid, link["link_id"], {}) assert harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5)["loss"] == 0 + assert harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=5)["loss"] == 0 # capture on the relay: the same REST call as the kernel datapath's, # hosted by the relay node's uBridge bridge (bridge start_capture) @@ -543,18 +681,20 @@ def test_docker_relay_control(): "POST", f"/projects/{pid}/links/{link['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} ) harness.docker_ping(daemon, n1["container_id"], R2_IP) + harness.docker_ping(daemon, n1["container_id"], R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{link['link_id']}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # markers on the relay are uBridge's `mark` packet filter on the # relay bridge (bridge add_packet_filter ... mark), signalled over # the same dedicated marker WS — the relay engine's observability # path (the kernel datapath's marker add_kernel is the other one) - print(".. relay marker: 5 ICMP echoes through a mark filter on the relay bridge") + print(".. relay markers: 5 v4 + 5 v6 ICMP echoes through mark filters on the relay bridge") marker_ws = harness.WebSocketCollector(server, f"/projects/{pid}/notifications/markers/ws") try: m = compute.call( @@ -563,25 +703,54 @@ def test_docker_relay_control(): {"name": "m-icmp", "bpf": "icmp", "tag": 4242, "capture_node_id": n1_id}, ) assert m["capture_node_id"] == n1_id and m["enabled"] is True, m + m6 = compute.call( + "POST", + f"/projects/{pid}/links/{link['link_id']}/markers", + {"name": "m-icmp6", "bpf": _ICMP6_ECHO_BPF, "tag": 4243, "capture_node_id": n1_id}, + ) + assert m6["capture_node_id"] == n1_id and m6["enabled"] is True, m6 result = harness.docker_ping(daemon, n1["container_id"], R2_IP, count=5) assert result["loss"] == 0, result["raw"] - harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 10, timeout=10) + result6 = harness.docker_ping(daemon, n1["container_id"], R2_IP6, count=5) + assert result6["loss"] == 0, result6["raw"] + harness.wait_until( + lambda: len(_matches(marker_ws, "m-icmp")) >= 10 and len(_matches(marker_ws, "m-icmp6")) >= 10, + timeout=10, + ) + # exact counts prove the two filters do not cross-fire: each + # marker sees its own family's 5 requests + 5 replies and + # nothing of the other's icmp = _matches(marker_ws, "m-icmp") assert len(icmp) == 10, len(icmp) # 5 requests + 5 replies + icmp6 = _matches(marker_ws, "m-icmp6") + assert len(icmp6) == 10, len(icmp6) assert {e["dir"] for e in icmp} == {"tx", "rx"}, [e["dir"] for e in icmp] + assert {e["dir"] for e in icmp6} == {"tx", "rx"}, [e["dir"] for e in icmp6] for event in icmp: assert event["node_id"] == n1_id, event assert event["link_id"] == link["link_id"], event assert event["tag"] == 4242, event - assert event["len"] == 98, event # busybox ping frame - # the marker's pcap on the host holds exactly the matched frames + assert event["len"] == PING_FRAME_LEN, event # busybox ping frame + for event in icmp6: + assert event["node_id"] == n1_id, event + assert event["link_id"] == link["link_id"], event + assert event["tag"] == 4243, event + assert event["len"] == PING_FRAME_LEN6, event # v6 echo is 20 bytes longer + # each marker's pcap on the host holds exactly its matched frames markers_dir = os.path.join(compute.call("GET", f"/projects/{pid}")["path"], "project-files", "markers") pcap = os.path.join(markers_dir, f"{n1_id}_{link['link_id']}_m-icmp.pcap") + pcap6 = os.path.join(markers_dir, f"{n1_id}_{link['link_id']}_m-icmp6.pcap") assert os.path.exists(pcap), os.listdir(markers_dir) + assert os.path.exists(pcap6), os.listdir(markers_dir) count, ethertypes = harness.pcap_records(pcap) assert count == 10, count assert set(ethertypes) == {"0800"}, ethertypes - # deleting the marker removes its pcap with it + count6, ethertypes6 = harness.pcap_records(pcap6) + assert count6 == 10, count6 + assert set(ethertypes6) == {"86dd"}, ethertypes6 + # deleting a marker removes its pcap with it + compute.call("DELETE", f"/projects/{pid}/links/{link['link_id']}/markers/m-icmp6") + assert harness.wait_until(lambda: not os.path.exists(pcap6), timeout=10), pcap6 compute.call("DELETE", f"/projects/{pid}/links/{link['link_id']}/markers/m-icmp") assert harness.wait_until(lambda: not os.path.exists(pcap), timeout=10), pcap finally: @@ -618,9 +787,10 @@ def test_docker_relay_to_kernel_reopen_upgrade(): assert link["kernel_datapath"] is False, link bridge = harness.link_bridge_name(link["link_id"]) - _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + _start_addresses(compute, pid, daemon, (n1, n2), ((R1_IP, R1_IP6), (R2_IP, R2_IP6))) assert harness.bridge_members(bridge) is None assert harness.docker_wait_ping(daemon, n1["container_id"], R2_IP)["loss"] == 0 + assert harness.docker_wait_ping(daemon, n1["container_id"], R2_IP6)["loss"] == 0 # Flip the datapath choice and reopen the project: every link # re-runs _prepare, which re-evaluates eligibility live. @@ -634,9 +804,10 @@ def test_docker_relay_to_kernel_reopen_upgrade(): assert link["kernel_datapath"] is True, link assert not harness.tap_exists(a1), "containers are stopped after the reopen" - _start_addresses(compute, pid, daemon, (n1, n2), (R1_IP, R2_IP)) + _start_addresses(compute, pid, daemon, (n1, n2), ((R1_IP, R1_IP6), (R2_IP, R2_IP6))) assert harness.wait_until(lambda: harness.bridge_members(bridge) == sorted([a1, a2]), timeout=15) assert harness.docker_wait_ping(daemon, n1["container_id"], R2_IP, attempts=5)["loss"] == 0 + assert harness.docker_wait_ping(daemon, n1["container_id"], R2_IP6, attempts=5)["loss"] == 0 except BaseException: harness.release(server, pid, failed=True) raise diff --git a/tests/e2e/test_docker_link_local_frames.py b/tests/e2e/test_docker_link_local_frames.py index 42743d83a..216bb407d 100644 --- a/tests/e2e/test_docker_link_local_frames.py +++ b/tests/e2e/test_docker_link_local_frames.py @@ -36,7 +36,12 @@ asserts, through the server's own wiring path: * the bridge-level ``group_fwd_mask`` stays 0 (the fix is per-port only); * the live matrix from the guest: ordinary multicast crosses (wire sanity), STP / LACP / EAPOL / LLDP cross, and PAUSE does not — ``br_handle_frame``'s - ``case 0x01`` drops it unconditionally, so PFC stays unsupported by design; + ``case 0x01`` drops it unconditionally, so PFC stays unsupported by design. + The matrix is dual-stack at the frame level: the v4 all-hosts control row + has a v6 twin over ethertype 0x86dd (the all-nodes multicast MAC), and the + guests deliberately stay v4-addressed — a dual-stack guest's own MLD + report would target exactly that MAC and contaminate the per-MAC count + window the row is measured with; * a container stop/start replays the NIO wiring and the mask is re-applied (the reconnect story), with a fresh LACP round surviving it. @@ -65,15 +70,18 @@ ETH = (0, 0) R1_IP = "10.2.1.1" R2_IP = "10.2.1.2" -# (destination MAC, ethertype) — what the guest protocol would emit +# (destination MAC, ethertype) — what the guest protocol would emit. The +# v4 control frame's v6 twin is the all-nodes multicast: the same "ordinary +# multicast crosses" sanity, over the v6 ethertype. CONTROL = ("01005e000001", "0800") # ordinary multicast: must always cross +CONTROL6 = ("333300000001", "86dd") # ordinary v6 multicast (ff02::1): must always cross STP = ("0180c2000000", "0026") # STP BPDU LACP = ("0180c2000002", "8809") # LACP / slow protocols EAPOL = ("0180c2000003", "888e") # 802.1X EAPOL LLDP = ("0180c200000e", "88cc") # LLDP / DCBX PAUSE = ("0180c2000001", "8808") # 802.3x PAUSE — kernel hard-drop -MUST_CROSS = [CONTROL, STP, LACP, EAPOL, LLDP] +MUST_CROSS = [CONTROL, CONTROL6, STP, LACP, EAPOL, LLDP] MUST_NOT_CROSS = [PAUSE] RECV_WINDOW = 12 diff --git a/tests/e2e/test_dynamips_kernel_datapath.py b/tests/e2e/test_dynamips_kernel_datapath.py index 5d170eb98..9e1bd6154 100644 --- a/tests/e2e/test_dynamips_kernel_datapath.py +++ b/tests/e2e/test_dynamips_kernel_datapath.py @@ -39,6 +39,10 @@ per-link bridge, nothing enslaved — and still pings, so the kernel objects above can only come from the kernel datapath. (The anchors themselves exist on both: they are the adapter's port interfaces, created by node start whenever the tap module is available, exactly like QEMU's TAPs.) + +Every Ethernet checkpoint runs dual-stack (the routers carry a ULA v6 +address next to their v4 one and each checkpoint pings both families); the +serial leg stays v4-only — it is HDLC on the relay, not an Ethernet wire. """ import re @@ -56,6 +60,11 @@ SLOTS = {"slot1": "PA-2FE-TX", "slot2": "PA-4T+"} ETH = (1, 0) SERIAL = (2, 0) +# the dual-stack half of the Ethernet leg: a ULA mirroring 10.1.1.x. The +# serial leg stays v4-only (HDLC, not an Ethernet wire). +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" + def _pick_image(server): images = server.compute.dynamips_images() @@ -134,13 +143,19 @@ def test_dynamips_kernel_datapath(): harness.assert_forwarding(t1) harness.assert_forwarding(t2) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_ipv6=R2_IP6) out = c1.run("show ip int brief") assert re.search(r"FastEthernet1/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + out = c1.run("show ipv6 interface brief") + # IOS prints v6 addresses upper-case (and on their own line): match + # the literal case-insensitively + assert R1_IP6.upper() in out.upper(), out print(".. both routers configured, pinging over the kernel link") baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + baseline6 = harness.wait_ping(c1, R2_IP6) + assert baseline6["success"] == 100, baseline6["raw"] # L2-anchor spec §E.2: shut the guest interfaces (silencing IOS's own # CDP/keepalives — the only legitimate speakers on the segment) and @@ -153,9 +168,11 @@ def test_dynamips_kernel_datapath(): for console in (c1, c2): _eth_shutdown(console, shut=False) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 # delay 100: netem on both anchors (each impairs one direction), - # one-way ~100 ms => RTT grows by ~200 ms + # one-way ~100 ms => RTT grows by ~200 ms — netem is family-blind, + # both families ride the same qdisc compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"delay": [100]}}) assert "netem" in harness.qdiscs(t1) and "netem" in harness.qdiscs(t2), ( harness.qdiscs(t1), @@ -163,30 +180,54 @@ def test_dynamips_kernel_datapath(): ) delayed = harness.wait_ping(c1, "10.1.1.2") assert delayed["success"] == 100, delayed["raw"] + delayed6 = harness.wait_ping(c1, R2_IP6) + assert delayed6["success"] == 100, delayed6["raw"] + print(f".. delay 100: v4 {baseline['avg']}->{delayed['avg']} ms, v6 {baseline6['avg']}->{delayed6['avg']} ms") assert delayed["avg"] >= 150, (baseline, delayed) + # The v6 twin is asserted from the capture's own timestamps, not from + # the guest's report: the dynamips-emulated classic IOS under-reports + # its ping ipv6 RTT by ~40 % of a delayed round trip (reproducible: + # report ~125-135 ms while a pcap on the anchor shows the v6 echo + # request->reply taking the same ~230 ms as v4 — the wire is correct, + # the reporter is not; the pings' order does not matter). + capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) + harness.ping(c1, R2_IP6) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") + wire6 = harness.pcap_icmp_rtt(capture["capture_file_path"], "86dd") + print(f".. delay 100 (capture): v6 wire round trip {wire6:.0f} ms") + assert wire6 is not None and wire6 >= 60, wire6 compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(t1), harness.qdiscs(t1) fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast + fast6 = harness.wait_ping(c1, R2_IP6) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, fast6 # suspend: anchor admin-down, traffic dead; resume restores it compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": True}) assert not harness.tap_up(t1) dead = harness.ping(c1, "10.1.1.2", repeat=3) assert dead["success"] == 0, dead["raw"] + dead6 = harness.ping(c1, R2_IP6, repeat=3) + assert dead6["success"] == 0, dead6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) assert harness.tap_up(t1) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 - # capture: AF_PACKET on the anchor writes a real pcap + # capture: AF_PACKET on the anchor writes a real pcap — both + # families inside the window, both ethertypes on the wire capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) harness.ping(c1, "10.1.1.2") + harness.ping(c1, R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # serial link between the same routers: per-port exclusion keeps it # on the relay, where the real traffic still crosses @@ -208,6 +249,7 @@ def test_dynamips_kernel_datapath(): bridge = harness.link_bridge_name(lid) assert link["kernel_datapath"] is True, link assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 # stop / start keeps the whole wiring (the Dynamips hypervisor # outlives ``vm stop``); the restarted router just reboots and is @@ -220,8 +262,9 @@ def test_dynamips_kernel_datapath(): c1.close() c1 = _boot(server, pid, r1_id) assert harness.bridge_members(bridge) == sorted([t1, t2]) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_ipv6=R1_IP6) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 c1.close() c2.close() @@ -281,10 +324,12 @@ def test_dynamips_relay_control(): # datapath one: relay anchors are pure L2 too harness.assert_pure_l2(t1) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_ipv6=R2_IP6) relay_ping = harness.wait_ping(c1, "10.1.1.2") assert relay_ping["success"] == 100, relay_ping["raw"] + relay_ping6 = harness.wait_ping(c1, R2_IP6) + assert relay_ping6["success"] == 100, relay_ping6["raw"] c1.close() c2.close() diff --git a/tests/e2e/test_ethernet_switch_cascade.py b/tests/e2e/test_ethernet_switch_cascade.py index c474a6124..fe72c8575 100644 --- a/tests/e2e/test_ethernet_switch_cascade.py +++ b/tests/e2e/test_ethernet_switch_cascade.py @@ -52,8 +52,15 @@ No relay control twin here: on the relay datapath a cascade is just two per-port uBridge UDP relays, the machinery ``test_ethernet_switch_kernel_datapath.py::test_ethernet_switch_relay_control`` already drives. + +Every checkpoint runs dual-stack: each router interface carries a ULA v6 +address next to its v4 one and every ping is doubled (both VLANs tagged +across the trunk in both families, the mode flip and suspend killing both, +the two-sided delay measured on both). """ +import time + import pytest from tests.e2e import harness @@ -68,6 +75,9 @@ ETH20 = (1, 1) IP10 = "10.1.10.%s" IP20 = "10.1.20.%s" +# the dual-stack half: ULAs mirroring the v4 subnets +IP10_6 = "fd00:1:10::%s" +IP20_6 = "fd00:1:20::%s" def _pick_image(server): @@ -110,10 +120,12 @@ def _set_ports(compute, pid, switch_id, cascade_type): compute.call("PUT", f"/projects/{pid}/nodes/{switch_id}", {"properties": {"ports_mapping": _ports(cascade_type)}}) -def _add_eth(console, ip, ifname): +def _add_eth(console, ip, ifname, ipv6=None): console.run("conf t") console.run(f"interface {ifname}") console.run(f"ip address {ip} 255.255.255.0") + if ipv6: + console.run(f"ipv6 address {ipv6}/64") console.run("no shutdown") console.run("end") @@ -202,18 +214,23 @@ def test_ethernet_switch_cascade_kernel_datapath(): harness.bridge_members(br2) ) - harness.configure_ios(c1, "R1", eth_ip=IP10 % 1, eth_if="f1/0") - _add_eth(c1, IP20 % 1, "f1/1") - harness.configure_ios(c2, "R2", eth_ip=IP10 % 2, eth_if="f1/0") - _add_eth(c2, IP20 % 2, "f1/1") + harness.configure_ios(c1, "R1", eth_ip=IP10 % 1, eth_if="f1/0", eth_ipv6=IP10_6 % 1) + _add_eth(c1, IP20 % 1, "f1/1", ipv6=IP20_6 % 1) + harness.configure_ios(c2, "R2", eth_ip=IP10 % 2, eth_if="f1/0", eth_ipv6=IP10_6 % 2) + _add_eth(c2, IP20 % 2, "f1/1", ipv6=IP20_6 % 2) print(".. both routers configured, pinging across the cascade") # A trunk cascade carries both VLANs at once — two simultaneous - # adjacencies over the same veth pair is the tagged-frames proof. + # adjacencies over the same veth pair is the tagged-frames proof — + # in both families. baseline10 = harness.wait_ping(c1, IP10 % 2) assert baseline10["success"] == 100, baseline10["raw"] baseline20 = harness.wait_ping(c1, IP20 % 2) assert baseline20["success"] == 100, baseline20["raw"] + baseline10_6 = harness.wait_ping(c1, IP10_6 % 2) + assert baseline10_6["success"] == 100, baseline10_6["raw"] + baseline20_6 = harness.wait_ping(c1, IP20_6 % 2) + assert baseline20_6["success"] == 100, baseline20_6["raw"] # L2-anchor spec §E.2: with all four router interfaces shut, every # host-side device of this fabric stays silent for 5 s — the four @@ -228,6 +245,8 @@ def test_ethernet_switch_cascade_kernel_datapath(): _eth_shutdown(console, shut=False) assert harness.wait_ping(c1, IP10 % 2, attempts=5)["success"] == 100 assert harness.wait_ping(c1, IP20 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP10_6 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP20_6 % 2, attempts=5)["success"] == 100 else: print(".. uBridge without link l2only: skipping the §E.2 assertion") @@ -251,12 +270,17 @@ def test_ethernet_switch_cascade_kernel_datapath(): assert harness.bridge_members(br1) == sorted([a1_10, a1_20, e0]), harness.bridge_members(br1) assert harness.bridge_members(br2) == sorted([a2_10, a2_20, e1]), harness.bridge_members(br2) assert harness.wait_ping(c1, IP10 % 2)["success"] == 100 + assert harness.wait_ping(c1, IP10_6 % 2)["success"] == 100 isolated = harness.ping(c1, IP20 % 2, repeat=3) assert isolated["success"] == 0, isolated["raw"] + isolated6 = harness.ping(c1, IP20_6 % 2, repeat=3) + assert isolated6["success"] == 0, isolated6["raw"] _set_ports(compute, pid, sw1_id, "dot1q") _set_ports(compute, pid, sw2_id, "dot1q") rejoined = harness.wait_ping(c1, IP20 % 2, attempts=5) assert rejoined["success"] == 100, rejoined["raw"] + rejoined6 = harness.wait_ping(c1, IP20_6 % 2, attempts=5) + assert rejoined6["success"] == 100, rejoined6["raw"] # Two-sided impairment: each veth end is its own interface, so the # filter lands on both — one netem per direction, RTT ~= 2 x delay. @@ -266,20 +290,38 @@ def test_ethernet_switch_cascade_kernel_datapath(): delayed = harness.wait_ping(c1, IP10 % 2) assert delayed["success"] == 100, delayed["raw"] assert 150 <= delayed["avg"] <= 350, (baseline10, delayed) + # the v6 twin reads the capture's own timestamps: the guest's + # ping ipv6 under-reports a delayed round trip on dynamips (see + # test_dynamips_kernel_datapath), so this asserts the wire itself + capture = compute.call( + "POST", f"/projects/{pid}/links/{cascade['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} + ) + harness.ping(c1, IP10_6 % 2) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{cascade['link_id']}/capture/stop") + wire6 = harness.pcap_icmp_rtt(capture["capture_file_path"], "86dd") + print(f".. delay 100: v4 {baseline10['avg']}->{delayed['avg']} ms, v6 wire {wire6:.0f} ms") + assert wire6 is not None and wire6 >= 60, wire6 compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"filters": {}}) assert "netem" not in harness.qdiscs(e0) and "netem" not in harness.qdiscs(e1) fast = harness.wait_ping(c1, IP10 % 2) assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast + fast6 = harness.wait_ping(c1, IP10_6 % 2) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, fast6 # suspend: both ends admin-down, both VLANs die; resume restores. compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"suspend": True}) assert not harness.tap_up(e0) and not harness.tap_up(e1) assert harness.ping(c1, IP10 % 2, repeat=3)["success"] == 0 assert harness.ping(c1, IP20 % 2, repeat=3)["success"] == 0 + assert harness.ping(c1, IP10_6 % 2, repeat=3)["success"] == 0 + assert harness.ping(c1, IP20_6 % 2, repeat=3)["success"] == 0 compute.call("PUT", f"/projects/{pid}/links/{cascade['link_id']}", {"suspend": False}) assert harness.tap_up(e0) and harness.tap_up(e1) assert harness.wait_ping(c1, IP10 % 2, attempts=5)["success"] == 100 assert harness.wait_ping(c1, IP20 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP10_6 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP20_6 % 2, attempts=5)["success"] == 100 # Cascade link fault: the whole veth pair dies with the link (either # side's teardown), the host links and their anchors survive; a @@ -290,6 +332,7 @@ def test_ethernet_switch_cascade_kernel_datapath(): assert harness.bridge_members(br1) == sorted([a1_10, a1_20]), harness.bridge_members(br1) assert harness.bridge_members(br2) == sorted([a2_10, a2_20]), harness.bridge_members(br2) assert harness.ping(c1, IP10 % 2, repeat=3)["success"] == 0 + assert harness.ping(c1, IP10_6 % 2, repeat=3)["success"] == 0 cascade = compute.create_link(pid, (sw1_id, 0, 2), (sw2_id, 0, 2)) assert cascade["kernel_datapath"] is True, cascade n0, n1 = harness.cascade_end_names(cascade["link_id"]) @@ -297,6 +340,8 @@ def test_ethernet_switch_cascade_kernel_datapath(): assert harness.wait_until(lambda: harness.bridge_members(br2) == sorted([a2_10, a2_20, n1]), timeout=10) assert harness.wait_ping(c1, IP10 % 2, attempts=5)["success"] == 100 assert harness.wait_ping(c1, IP20 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP10_6 % 2, attempts=5)["success"] == 100 + assert harness.wait_ping(c1, IP20_6 % 2, attempts=5)["success"] == 100 c1.close() c2.close() diff --git a/tests/e2e/test_ethernet_switch_kernel_datapath.py b/tests/e2e/test_ethernet_switch_kernel_datapath.py index d42cd3a93..25bf19179 100644 --- a/tests/e2e/test_ethernet_switch_kernel_datapath.py +++ b/tests/e2e/test_ethernet_switch_kernel_datapath.py @@ -43,6 +43,10 @@ Real ICMP crossing the switch proves the datapath: isolated relay-configured instance: the same topology rides the per-port uBridge relays (the switch bridge's members are its own relay TAPs, never the routers' anchors) and still pings. + +Every checkpoint runs dual-stack: the routers carry a ULA v6 address next +to their v4 one and each checkpoint pings both families (VLAN isolation +and suspend kill both; the netem delays and the capture window carry both). """ import time @@ -57,6 +61,10 @@ pytestmark = pytest.mark.e2e SLOTS = {"slot1": "PA-2FE-TX"} ETH = (1, 0) +# the dual-stack half: a ULA mirroring 10.1.1.x +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" + def _pick_image(server): images = server.compute.dynamips_images() @@ -161,11 +169,13 @@ def test_ethernet_switch_kernel_fast_path(): harness.bridge_members(sw_bridge) ) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_ipv6=R2_IP6) print(".. both routers configured, pinging through the switch") baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + baseline6 = harness.wait_ping(c1, R2_IP6) + assert baseline6["success"] == 100, baseline6["raw"] # L2-anchor spec §E.2: with both routers shut, the absorbed anchors # and the switch bridge stay silent for 5 s (the bridge is created @@ -177,6 +187,7 @@ def test_ethernet_switch_kernel_fast_path(): for console in (c1, c2): _eth_shutdown(console, shut=False) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 else: print(".. uBridge without link l2only: skipping the §E.2 assertion") @@ -209,6 +220,8 @@ def test_ethernet_switch_kernel_fast_path(): assert harness.bridge_members(sw_bridge) == sorted([a1, a2]), harness.bridge_members(sw_bridge) isolated = harness.ping(c1, "10.1.1.2", repeat=3) assert isolated["success"] == 0, isolated["raw"] + isolated6 = harness.ping(c1, R2_IP6, repeat=3) + assert isolated6["success"] == 0, isolated6["raw"] assert harness.bridge_members(sw_bridge) == sorted([a1, a2]), harness.bridge_members(sw_bridge) compute.call( "PUT", @@ -224,6 +237,8 @@ def test_ethernet_switch_kernel_fast_path(): ) rejoined = harness.wait_ping(c1, "10.1.1.2") assert rejoined["success"] == 100, rejoined["raw"] + rejoined6 = harness.wait_ping(c1, R2_IP6) + assert rejoined6["success"] == 100, rejoined6["raw"] # Single-owned impairment: the filter lands on the absorbed anchor # (one interface — the switch's), RTT grows by ~100 ms one-way. @@ -233,10 +248,25 @@ def test_ethernet_switch_kernel_fast_path(): delayed = harness.wait_ping(c1, "10.1.1.2") assert delayed["success"] == 100, delayed["raw"] assert 60 <= delayed["avg"] <= 250, (baseline, delayed) + # the v6 twin reads the capture's own timestamps: the guest's + # ping ipv6 under-reports a delayed round trip on dynamips (see + # test_dynamips_kernel_datapath), and here the single-owned delay + # is small enough that the quirk swallows most of the signal + capture = compute.call( + "POST", f"/projects/{pid}/links/{link1['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} + ) + harness.ping(c1, R2_IP6) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{link1['link_id']}/capture/stop") + wire6 = harness.pcap_icmp_rtt(capture["capture_file_path"], "86dd") + print(f".. delay 100: v4 {baseline['avg']}->{delayed['avg']} ms, v6 wire {wire6:.0f} ms") + assert wire6 is not None and wire6 >= 60, wire6 compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast + fast6 = harness.wait_ping(c1, R2_IP6) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, fast6 # The same single-owned impairment on link2 — created switch-first, # the order whose peer NIO update runs LAST. The passive end's empty @@ -249,31 +279,54 @@ def test_ethernet_switch_kernel_fast_path(): delayed2 = harness.wait_ping(c1, "10.1.1.2") assert delayed2["success"] == 100, delayed2["raw"] assert 60 <= delayed2["avg"] <= 250, (baseline, delayed2) + # the capture must sit on the pinging node's anchor (link1's, i.e. + # R1's): a TX copy is timestamped post-qdisc, so on link2's own + # anchor — where this window's delay impairs the request direction — + # the request copy would already carry the delay and the measured + # delta would collapse to the turnaround. R1's anchor sees the + # request before the delay and the reply after it. + capture = compute.call( + "POST", f"/projects/{pid}/links/{link1['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} + ) + harness.ping(c1, R2_IP6) + time.sleep(1) + compute.call("POST", f"/projects/{pid}/links/{link1['link_id']}/capture/stop") + wire2_6 = harness.pcap_icmp_rtt(capture["capture_file_path"], "86dd") + print(f".. delay 100 (link2): v4 {baseline['avg']}->{delayed2['avg']} ms, v6 wire {wire2_6:.0f} ms") + assert wire2_6 is not None and wire2_6 >= 60, wire2_6 compute.call("PUT", f"/projects/{pid}/links/{link2['link_id']}", {"filters": {}}) assert "netem" not in harness.qdiscs(a2) fast2 = harness.wait_ping(c1, "10.1.1.2") assert fast2["success"] == 100 and fast2["avg"] < harness.FAST_RTT_MS, fast2 + fast2_6 = harness.wait_ping(c1, R2_IP6) + assert fast2_6["success"] == 100 and fast2_6["avg"] < harness.FAST_RTT_MS, fast2_6 # suspend: the anchor admin-downs, the link is dead; resume restores compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"suspend": True}) assert not harness.tap_up(a1) dead = harness.ping(c1, "10.1.1.2", repeat=3) assert dead["success"] == 0, dead["raw"] + dead6 = harness.ping(c1, R2_IP6, repeat=3) + assert dead6["success"] == 0, dead6["raw"] compute.call("PUT", f"/projects/{pid}/links/{link1['link_id']}", {"suspend": False}) assert harness.tap_up(a1) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 # capture (the controller hosts it on the switch — always-on local - # builtin): a pcap of ICMP records written off the absorbed anchor + # builtin): a pcap of ICMP records written off the absorbed anchor, + # both families inside the window capture = compute.call( "POST", f"/projects/{pid}/links/{link1['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"} ) harness.ping(c1, "10.1.1.2") + harness.ping(c1, R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{link1['link_id']}/capture/stop") count, ethertypes = harness.pcap_records(capture["capture_file_path"]) - assert count >= 4, capture["capture_file_path"] + assert count >= 8, capture["capture_file_path"] assert "0800" in ethertypes, ethertypes + assert "86dd" in ethertypes, ethertypes # link delete detaches the anchor (it survives — the router owns # it); re-creating re-joins it @@ -284,6 +337,7 @@ def test_ethernet_switch_kernel_fast_path(): assert link2["kernel_datapath"] is True, link2 assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) == sorted([a1, a2]), timeout=10) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 # router stop/start keeps the wiring: a Dynamips stop only halts the # emulated router — the hypervisor, the anchor and its bridge @@ -297,8 +351,9 @@ def test_ethernet_switch_kernel_fast_path(): c1.close() c1 = _boot(server, pid, r1_id) assert harness.wait_until(lambda: harness.bridge_members(sw_bridge) == sorted([a1, a2]), timeout=15) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_ipv6=R1_IP6) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 c1.close() c2.close() @@ -351,10 +406,12 @@ def test_ethernet_switch_relay_control(): harness.assert_pure_l2(member) harness.assert_pure_l2(sw_bridge) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_ipv6=R2_IP6) relay_ping = harness.wait_ping(c1, "10.1.1.2") assert relay_ping["success"] == 100, relay_ping["raw"] + relay_ping6 = harness.wait_ping(c1, R2_IP6) + assert relay_ping6["success"] == 100, relay_ping6["raw"] c1.close() c2.close() diff --git a/tests/e2e/test_iol_docker_kernel_datapath.py b/tests/e2e/test_iol_docker_kernel_datapath.py index 9100aaf9b..d64e8cb7f 100644 --- a/tests/e2e/test_iol_docker_kernel_datapath.py +++ b/tests/e2e/test_iol_docker_kernel_datapath.py @@ -36,6 +36,12 @@ bridge, kernel filters, suspend and capture. ``test_iol_docker_relay_control`` is the negative control on an isolated relay-configured instance: the same topology rides unix ↔ UDP through the uBridge relay (no anchors ever exist) and still pings. + +Every checkpoint runs dual-stack: the IOS-XE guests carry a ULA v6 address +next to their v4 one and each checkpoint pings both families — including +the TAP-anchor bpf spot check (the libpcap expression is family-specific: +each drop filter is asserted to kill its own family and leave the other +alive) and the eBPF nth classifier (family-blind, asserted on both). """ import re @@ -51,6 +57,10 @@ pytestmark = pytest.mark.e2e ETH = (0, 0) BOOT_TIMEOUT = 360 +# the dual-stack half: a ULA mirroring 10.1.1.x +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" + def _pick_image(server): images = server.compute.docker_images() @@ -143,13 +153,17 @@ def test_iol_docker_kernel_datapath(): harness.assert_forwarding(a1) harness.assert_forwarding(a2) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0", eth_ipv6=R2_IP6) out = c1.run("show ip int brief") assert re.search(r"Ethernet0/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + out = c1.run("show ipv6 interface brief") + assert R1_IP6.upper() in out.upper(), out # IOS prints v6 upper-case print(".. both routers configured, pinging through the port-bridge swap") baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + baseline6 = harness.wait_ping(c1, R2_IP6) + assert baseline6["success"] == 100, baseline6["raw"] # L2-anchor spec §E.2: with the guests shut the anchors and the # per-link bridge stay silent for 5 s. These anchors come from @@ -163,6 +177,7 @@ def test_iol_docker_kernel_datapath(): for console in (c1, c2): _eth_shutdown(console, shut=False) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 else: print(".. uBridge without link l2only: skipping the §E.2 assertion") @@ -176,10 +191,15 @@ def test_iol_docker_kernel_datapath(): delayed = harness.wait_ping(c1, "10.1.1.2") assert delayed["success"] == 100, delayed["raw"] assert delayed["avg"] >= 150, (baseline, delayed) + delayed6 = harness.wait_ping(c1, R2_IP6) + assert delayed6["success"] == 100, delayed6["raw"] + assert delayed6["avg"] >= 150, (baseline6, delayed6) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast + fast6 = harness.wait_ping(c1, R2_IP6) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, fast6 # Classifier spot check on the container's TAP anchor (the Docker # suite runs the full matrix on veth host ends): cls_bpf match-drop @@ -187,7 +207,7 @@ def test_iol_docker_kernel_datapath(): # same way. tc_caps = (caps or {}).get("ubridge_tc") or {} if tc_caps.get("cbpf"): - print(".. bpf 'icmp' drops everything on the TAP anchor") + print(".. bpf 'icmp' drops v4 ICMP only on the TAP anchor") compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"bpf": ["icmp"]}}) assert "clsact" in harness.qdiscs(a1) and "clsact" in harness.qdiscs(a2), ( harness.qdiscs(a1), @@ -195,9 +215,18 @@ def test_iol_docker_kernel_datapath(): ) blocked = harness.ping(c1, "10.1.1.2", repeat=4) assert blocked["success"] == 0, blocked["raw"] + v6_alive = harness.wait_ping(c1, R2_IP6) + assert v6_alive["success"] == 100, v6_alive["raw"] + print(".. bpf 'icmp6' drops v6 ICMP only on the TAP anchor") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"bpf": ["icmp6"]}}) + v4_alive = harness.wait_ping(c1, "10.1.1.2") + assert v4_alive["success"] == 100, v4_alive["raw"] + blocked6 = harness.ping(c1, R2_IP6, repeat=4) + assert blocked6["success"] == 0, blocked6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "clsact" not in harness.qdiscs(a1), harness.qdiscs(a1) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 else: print(".. uBridge reports no cbpf: skipping the bpf check") @@ -210,8 +239,13 @@ def test_iol_docker_kernel_datapath(): print(f".. {loss} % round-trip loss") # the kernel counts per direction: 1 - (2/3)^2 = 55.6 % round trip assert 20 <= loss <= 90, nth["raw"] + nth6 = harness.ping(c1, R2_IP6, repeat=9) + loss6 = 100 - nth6["success"] + print(f".. {loss6} % round-trip loss (v6)") + assert 20 <= loss6 <= 90, nth6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 else: print(".. uBridge reports no eBPF nth: skipping the frequency_drop check") @@ -221,19 +255,25 @@ def test_iol_docker_kernel_datapath(): assert not harness.tap_up(a1) dead = harness.ping(c1, "10.1.1.2", repeat=3) assert dead["success"] == 0, dead["raw"] + dead6 = harness.ping(c1, R2_IP6, repeat=3) + assert dead6["success"] == 0, dead6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) assert harness.tap_up(a1) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 - # capture: AF_PACKET on the anchor writes a real pcap + # capture: AF_PACKET on the anchor writes a real pcap — both + # families inside the window capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) harness.ping(c1, "10.1.1.2") + harness.ping(c1, R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # delete / re-create the kernel link: the port bridge's TAP leg is # released (stop -> delete_nio_tap), the anchor survives (the node @@ -247,6 +287,7 @@ def test_iol_docker_kernel_datapath(): bridge = harness.link_bridge_name(lid) assert link["kernel_datapath"] is True, link assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 # node stop: uBridge holds the anchor fds, so the port bridges go # first and the taps with them (a stopped container must not litter @@ -263,8 +304,9 @@ def test_iol_docker_kernel_datapath(): c1 = _boot(server, pid, n1_id) assert harness.tap_exists(a1) assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0", eth_ipv6=R1_IP6) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 c1.close() c2.close() @@ -322,10 +364,12 @@ def test_iol_docker_relay_control(): # the L2 hardening is a creation-time property, not a # datapath one: relay anchors are pure L2 too harness.assert_pure_l2(a1) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0", eth_ipv6=R2_IP6) relay_ping = harness.wait_ping(c1, "10.1.1.2") assert relay_ping["success"] == 100, relay_ping["raw"] + relay_ping6 = harness.wait_ping(c1, R2_IP6) + assert relay_ping6["success"] == 100, relay_ping6["raw"] c1.close() c2.close() diff --git a/tests/e2e/test_iou_kernel_datapath.py b/tests/e2e/test_iou_kernel_datapath.py index da4a22ddc..49a6bb48b 100644 --- a/tests/e2e/test_iou_kernel_datapath.py +++ b/tests/e2e/test_iou_kernel_datapath.py @@ -62,6 +62,16 @@ ETH = (0, 0) SERIAL = (2, 0) BOOT_TIMEOUT = 360 +# the dual-stack half of the Ethernet leg: a ULA mirroring 10.1.1.x. The +# serial leg stays v4-only (HDLC, not an Ethernet wire). +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" + +# exact ICMPv6 echo match for the relay marker: a bare "icmp6" would also +# match IOS's own control frames (DAD/MLD/NS/NA) and break the exact-count +# asserts +ICMP6_ECHO_BPF = "icmp6 and (ip6[40] == 128 or ip6[40] == 129)" + # L3 images only: an L2 switching image does not take an address on its # switchports. The first is the image the kernel datapath was validated # with, the second the matching 17.15 build. @@ -162,13 +172,17 @@ def test_iou_kernel_datapath(): harness.assert_forwarding(a1) harness.assert_forwarding(a2) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0", eth_ipv6=R2_IP6) out = c1.run("show ip int brief") assert re.search(r"Ethernet0/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + out = c1.run("show ipv6 interface brief") + assert R1_IP6.upper() in out.upper(), out # IOS prints v6 upper-case print(".. both routers configured, pinging through the [IOL fabric <-> TAP] port bridge") baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + baseline6 = harness.wait_ping(c1, R2_IP6) + assert baseline6["success"] == 100, baseline6["raw"] # L2-anchor spec §E.2: with the guests shut the anchors and the # per-link bridge stay silent for 5 s (these anchors are the `tap @@ -180,6 +194,7 @@ def test_iou_kernel_datapath(): for console in (c1, c2): _eth_shutdown(console, shut=False) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 else: print(".. uBridge without link l2only: skipping the §E.2 assertion") @@ -193,17 +208,22 @@ def test_iou_kernel_datapath(): delayed = harness.wait_ping(c1, "10.1.1.2") assert delayed["success"] == 100, delayed["raw"] assert delayed["avg"] >= 150, (baseline, delayed) + delayed6 = harness.wait_ping(c1, R2_IP6) + assert delayed6["success"] == 100, delayed6["raw"] + assert delayed6["avg"] >= 150, (baseline6, delayed6) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast + fast6 = harness.wait_ping(c1, R2_IP6) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, fast6 # Classifier spot check on a TAP anchor (the Docker suite runs the # full matrix on veth host ends): cls_bpf match-drop and the eBPF # stateful classifier attach to a tun/tap anchor the same way. tc_caps = (caps or {}).get("ubridge_tc") or {} if tc_caps.get("cbpf"): - print(".. bpf 'icmp' drops everything on the TAP anchor") + print(".. bpf 'icmp' drops v4 ICMP only on the TAP anchor") compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"bpf": ["icmp"]}}) assert "clsact" in harness.qdiscs(a1) and "clsact" in harness.qdiscs(a2), ( harness.qdiscs(a1), @@ -211,9 +231,18 @@ def test_iou_kernel_datapath(): ) blocked = harness.ping(c1, "10.1.1.2", repeat=4) assert blocked["success"] == 0, blocked["raw"] + v6_alive = harness.wait_ping(c1, R2_IP6) + assert v6_alive["success"] == 100, v6_alive["raw"] + print(".. bpf 'icmp6' drops v6 ICMP only on the TAP anchor") + compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {"bpf": ["icmp6"]}}) + v4_alive = harness.wait_ping(c1, "10.1.1.2") + assert v4_alive["success"] == 100, v4_alive["raw"] + blocked6 = harness.ping(c1, R2_IP6, repeat=4) + assert blocked6["success"] == 0, blocked6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "clsact" not in harness.qdiscs(a1), harness.qdiscs(a1) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 else: print(".. uBridge reports no cbpf: skipping the bpf check") @@ -226,8 +255,13 @@ def test_iou_kernel_datapath(): print(f".. {loss} % round-trip loss") # the kernel counts per direction: 1 - (2/3)^2 = 55.6 % round trip assert 20 <= loss <= 90, nth["raw"] + nth6 = harness.ping(c1, R2_IP6, repeat=9) + loss6 = 100 - nth6["success"] + print(f".. {loss6} % round-trip loss (v6)") + assert 20 <= loss6 <= 90, nth6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 else: print(".. uBridge reports no eBPF nth: skipping the frequency_drop check") @@ -237,19 +271,25 @@ def test_iou_kernel_datapath(): assert not harness.tap_up(a1) dead = harness.ping(c1, "10.1.1.2", repeat=3) assert dead["success"] == 0, dead["raw"] + dead6 = harness.ping(c1, R2_IP6, repeat=3) + assert dead6["success"] == 0, dead6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) assert harness.tap_up(a1) assert harness.wait_ping(c1, "10.1.1.2")["success"] == 100 + assert harness.wait_ping(c1, R2_IP6)["success"] == 100 - # capture: AF_PACKET on the anchor writes a real pcap + # capture: AF_PACKET on the anchor writes a real pcap — both + # families inside the window capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) harness.ping(c1, "10.1.1.2") + harness.ping(c1, R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # serial link between the same nodes: per-port exclusion keeps it on # the relay, where the real traffic still crosses (IOU-specific — @@ -274,6 +314,7 @@ def test_iou_kernel_datapath(): bridge = harness.link_bridge_name(lid) assert link["kernel_datapath"] is True, link assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 # node stop: uBridge holds the anchor fds, so the IOL bridge goes # first and the taps with it (a stopped node must not litter the @@ -294,13 +335,18 @@ def test_iou_kernel_datapath(): assert harness.tap_exists(a1) assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) assert harness.wait_until(lambda: "netem" in harness.qdiscs(a1), timeout=15), harness.qdiscs(a1) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0", eth_ipv6=R1_IP6) survivor = harness.wait_ping(c1, "10.1.1.2", attempts=5) assert survivor["success"] == 100, survivor["raw"] assert survivor["avg"] >= 150, (baseline, survivor) + survivor6 = harness.wait_ping(c1, R2_IP6, attempts=5) + assert survivor6["success"] == 100, survivor6["raw"] + assert survivor6["avg"] >= 150, (baseline6, survivor6) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) cleared = harness.ping(c1, "10.1.1.2", repeat=3) assert cleared["success"] == 100 and cleared["avg"] < harness.FAST_RTT_MS, (baseline, cleared) + cleared6 = harness.ping(c1, R2_IP6, repeat=3) + assert cleared6["success"] == 100 and cleared6["avg"] < harness.FAST_RTT_MS, (baseline6, cleared6) c1.close() c2.close() @@ -361,10 +407,12 @@ def test_iou_relay_control(): # the L2 hardening is a creation-time property, not a # datapath one: relay anchors are pure L2 too harness.assert_pure_l2(a1) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0") - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0") + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if="Ethernet0/0", eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if="Ethernet0/0", eth_ipv6=R2_IP6) relay_ping = harness.wait_ping(c1, "10.1.1.2") assert relay_ping["success"] == 100, relay_ping["raw"] + relay_ping6 = harness.wait_ping(c1, R2_IP6) + assert relay_ping6["success"] == 100, relay_ping6["raw"] # The relay link's filters really shape the wire on the IOU engine # too. delay 100: one delay line per direction (the port's IOL and @@ -378,9 +426,14 @@ def test_iou_relay_control(): delayed = harness.ping(c1, "10.1.1.2", repeat=5) assert delayed["success"] == 100, delayed["raw"] assert delayed["avg"] >= 150, (relay_ping, delayed) + delayed6 = harness.ping(c1, R2_IP6, repeat=5) + assert delayed6["success"] == 100, delayed6["raw"] + assert delayed6["avg"] >= 150, (relay_ping6, delayed6) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) fast = harness.ping(c1, "10.1.1.2", repeat=3) assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, (relay_ping, fast) + fast6 = harness.ping(c1, R2_IP6, repeat=3) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, (relay_ping6, fast6) # frequency_drop shares one counter between the port's two # directions: on an alternating ping stream one whole direction dies @@ -390,9 +443,14 @@ def test_iou_relay_control(): dropped = harness.ping(c1, "10.1.1.2", repeat=6) print(f".. success {dropped['success']} %") assert dropped["success"] <= 50, dropped["raw"] + dropped6 = harness.ping(c1, R2_IP6, repeat=6) + print(f".. success {dropped6['success']} % (v6)") + assert dropped6["success"] <= 50, dropped6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) recovered = harness.wait_ping(c1, "10.1.1.2") assert recovered["success"] == 100, recovered["raw"] + recovered6 = harness.wait_ping(c1, R2_IP6) + assert recovered6["success"] == 100, recovered6["raw"] # capture on the relay: iol_bridge start_capture on the port's IOL # location (the IOU engine's own capture path, distinct from the @@ -400,16 +458,18 @@ def test_iou_relay_control(): print(".. iol_bridge relay capture: start_capture writes the ICMP exchange") capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) harness.ping(c1, "10.1.1.2", repeat=5) + harness.ping(c1, R2_IP6, repeat=5) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # markers on the relay: iol_bridge add_packet_filter ... mark on the # port's IOL location, signalled over the dedicated marker WS - print(".. iol_bridge relay marker: 5 ICMP echoes through a mark filter") + print(".. iol_bridge relay markers: 5 v4 + 5 v6 ICMP echoes through mark filters") marker_ws = harness.WebSocketCollector(server, f"/projects/{pid}/notifications/markers/ws") try: m = compute.call( @@ -418,23 +478,50 @@ def test_iou_relay_control(): {"name": "m-icmp", "bpf": "icmp", "tag": 4242, "capture_node_id": n1_id}, ) assert m["capture_node_id"] == n1_id and m["enabled"] is True, m + m6 = compute.call( + "POST", + f"/projects/{pid}/links/{lid}/markers", + {"name": "m-icmp6", "bpf": ICMP6_ECHO_BPF, "tag": 4243, "capture_node_id": n1_id}, + ) + assert m6["capture_node_id"] == n1_id and m6["enabled"] is True, m6 result = harness.ping(c1, "10.1.1.2", repeat=5) assert result["success"] == 100, result["raw"] - harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 10, timeout=10) + result6 = harness.ping(c1, R2_IP6, repeat=5) + assert result6["success"] == 100, result6["raw"] + harness.wait_until( + lambda: len(_matches(marker_ws, "m-icmp")) >= 10 and len(_matches(marker_ws, "m-icmp6")) >= 10, + timeout=10, + ) + # exact counts across both families: neither marker sees the + # other family's frames icmp = _matches(marker_ws, "m-icmp") assert len(icmp) == 10, len(icmp) # 5 requests + 5 replies + icmp6 = _matches(marker_ws, "m-icmp6") + assert len(icmp6) == 10, len(icmp6) assert {e["dir"] for e in icmp} == {"tx", "rx"}, [e["dir"] for e in icmp] + assert {e["dir"] for e in icmp6} == {"tx", "rx"}, [e["dir"] for e in icmp6] for event in icmp: assert event["node_id"] == n1_id, event assert event["link_id"] == lid, event assert event["tag"] == 4242, event + for event in icmp6: + assert event["node_id"] == n1_id, event + assert event["link_id"] == lid, event + assert event["tag"] == 4243, event markers_dir = os.path.join(compute.call("GET", f"/projects/{pid}")["path"], "project-files", "markers") pcap = os.path.join(markers_dir, f"{n1_id}_{lid}_m-icmp.pcap") + pcap6 = os.path.join(markers_dir, f"{n1_id}_{lid}_m-icmp6.pcap") assert os.path.exists(pcap), os.listdir(markers_dir) + assert os.path.exists(pcap6), os.listdir(markers_dir) count, ethertypes = harness.pcap_records(pcap) assert count == 10, count assert set(ethertypes) == {"0800"}, ethertypes - # deleting the marker removes its pcap with it + count6, ethertypes6 = harness.pcap_records(pcap6) + assert count6 == 10, count6 + assert set(ethertypes6) == {"86dd"}, ethertypes6 + # deleting a marker removes its pcap with it + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-icmp6") + assert harness.wait_until(lambda: not os.path.exists(pcap6), timeout=10), pcap6 compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-icmp") assert harness.wait_until(lambda: not os.path.exists(pcap), timeout=10), pcap finally: diff --git a/tests/e2e/test_marker_kernel_datapath.py b/tests/e2e/test_marker_kernel_datapath.py index fba6b5d3f..ef988b279 100644 --- a/tests/e2e/test_marker_kernel_datapath.py +++ b/tests/e2e/test_marker_kernel_datapath.py @@ -40,6 +40,13 @@ the dedicated marker notification WebSocket. Real ICMP proves: marker under the tag is still capturing (skipped when sharkd is absent); * deleting a marker removes its pcap and silences its signals; deleting the project leaves no marker state behind. + +Everything runs dual-stack: alongside the v4 markers the link carries v6 +twin markers (one of them directioned) whose expression matches ICMPv6 +echoes only — a bare ``icmp6`` would also match the guests' own DAD/MLD +control frames and break the exact-count asserts. The exact counts across +both families double as the cross-family check: neither family's traffic +reaches the other family's markers. """ import json @@ -59,10 +66,19 @@ pytestmark = pytest.mark.e2e ETH = (0, 0) R1_IP = "10.1.1.1" R2_IP = "10.1.1.2" +# the dual-stack half: a ULA mirroring the v4 addressing (10.1.1.x -> fd00:1:1::x) +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" TAG = 4242 +TAG6 = 4243 -# a busybox `ping` default frame: 14 Ethernet + 20 IP + 8 ICMP + 56 data. +# busybox `ping` default frames: 14 Ethernet + 20/40 IP + 8 ICMP + 56 data. PING_FRAME_LEN = 98 +PING_FRAME_LEN6 = 118 + +# exact ICMPv6 echo match: a bare "icmp6" would also match the guests' own +# DAD/MLD/NS/NA control frames and break the exact-count asserts +ICMP6_ECHO_BPF = "icmp6 and (ip6[40] == 128 or ip6[40] == 129)" def _resolve_container(daemon, node): @@ -87,12 +103,15 @@ def _create_topology(server, pid, image): def _start_topology(compute, pid, daemon, nodes): - """Start the containers and wait for init.sh to have applied their - addresses (busybox ifup runs a moment after container start).""" + """Start the containers and wait for init.sh to have applied their v4 + addresses (busybox ifup runs a moment after container start); the v6 + addresses are brought up by the harness after each start.""" for node in nodes: compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start") - for node, address in zip(nodes, (R1_IP, R2_IP), strict=False): + for node, (address, address6) in zip(nodes, ((R1_IP, R1_IP6), (R2_IP, R2_IP6)), strict=False): harness.docker_wait_address(daemon, node["container_id"], address) + harness.docker_bring_up_ipv6(daemon, node["container_id"], address6) + harness.docker_wait_address(daemon, node["container_id"], address6) def _raw(compute, path): @@ -140,6 +159,8 @@ def test_marker_kernel_datapath(): _start_topology(compute, pid, daemon, (n1, n2)) baseline = harness.docker_wait_ping(daemon, c1, R2_IP) assert baseline["loss"] == 0, baseline["raw"] + baseline6 = harness.docker_wait_ping(daemon, c1, R2_IP6) + assert baseline6["loss"] == 0, baseline6["raw"] a1 = harness.docker_anchor_name(n1_id, *ETH) markers_dir = os.path.join(compute.call("GET", f"/projects/{pid}")["path"], "project-files", "markers") @@ -167,16 +188,39 @@ def test_marker_kernel_datapath(): {"name": "m-tx", "bpf": "icmp", "direction": "tx", "tag": 4344, "capture_node_id": n1_id}, ) assert m_tx["direction"] == "tx", m_tx + # the dual-stack twin: a v6 marker with an echo-specific expression + # (a bare "icmp6" would also match the guests' own NS/MLD/RA frames) + m6 = compute.call( + "POST", + f"/projects/{pid}/links/{lid}/markers", + {"name": "m-icmp6", "bpf": ICMP6_ECHO_BPF, "tag": TAG6, "capture_node_id": n1_id}, + ) + assert m6["capture_node_id"] == n1_id and m6["tag"] == TAG6 and m6["enabled"] is True, m6 + m_tx6 = compute.call( + "POST", + f"/projects/{pid}/links/{lid}/markers", + {"name": "m-tx6", "bpf": ICMP6_ECHO_BPF, "direction": "tx", "tag": 4345, "capture_node_id": n1_id}, + ) + assert m_tx6["direction"] == "tx", m_tx6 - print(".. sending 5 ICMP echoes and collecting marker.match signals") + print(".. sending 5 v4 + 5 v6 ICMP echoes and collecting marker.match signals") result = harness.docker_ping(daemon, c1, R2_IP, count=5) assert result["loss"] == 0, result["raw"] + result6 = harness.docker_ping(daemon, c1, R2_IP6, count=5) + assert result6["loss"] == 0, result6["raw"] - harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 10, timeout=10) + harness.wait_until( + lambda: len(_matches(marker_ws, "m-icmp")) >= 10 and len(_matches(marker_ws, "m-icmp6")) >= 10, timeout=10 + ) + # exact counts across both families: each marker sees its own + # family's 5 requests + 5 replies and nothing of the other's icmp = _matches(marker_ws, "m-icmp") assert len(icmp) == 10, len(icmp) # 5 requests (tx) + 5 replies (rx) + icmp6 = _matches(marker_ws, "m-icmp6") + assert len(icmp6) == 10, len(icmp6) assert not _matches(marker_ws, "m-tcp"), "a non-matching BPF must stay silent" assert len(_matches(marker_ws, "m-tx")) == 5, "the tx direction sees requests only" + assert len(_matches(marker_ws, "m-tx6")) == 5, "the v6 tx direction sees requests only" for event in icmp: assert event["node_id"] == n1_id, event @@ -184,38 +228,61 @@ def test_marker_kernel_datapath(): assert event["tag"] == TAG, event assert event["ts"] > 0, event assert event["len"] == PING_FRAME_LEN, event + for event in icmp6: + assert event["node_id"] == n1_id, event + assert event["link_id"] == lid, event + assert event["tag"] == TAG6, event + assert event["ts"] > 0, event + assert event["len"] == PING_FRAME_LEN6, event assert {e["dir"] for e in icmp} == {"tx", "rx"}, [e["dir"] for e in icmp] + assert {e["dir"] for e in icmp6} == {"tx", "rx"}, [e["dir"] for e in icmp6] assert {e["dir"] for e in _matches(marker_ws, "m-tx")} == {"tx"} + assert {e["dir"] for e in _matches(marker_ws, "m-tx6")} == {"tx"} # The dedicated channel carries matches; the main project stream must # not (and must still be alive — its link.updated proves the socket). assert main_ws.messages("marker.match") == [], main_ws.messages("marker.match")[:3] assert main_ws.messages("link.updated"), "the main channel is not receiving at all" - # Host pcap: exactly the matched frames, off the anchor + # Host pcaps: exactly the matched frames, off the anchor pcap = os.path.join(markers_dir, f"{n1_id}_{lid}_m-icmp.pcap") + pcap6 = os.path.join(markers_dir, f"{n1_id}_{lid}_m-icmp6.pcap") assert os.path.exists(pcap), os.listdir(markers_dir) + assert os.path.exists(pcap6), os.listdir(markers_dir) count, ethertypes = harness.pcap_records(pcap) assert count == 10, count assert set(ethertypes) == {"0800"}, ethertypes + count6, ethertypes6 = harness.pcap_records(pcap6) + assert count6 == 10, count6 + assert set(ethertypes6) == {"86dd"}, ethertypes6 # pause: this marker's signals stop, its pcap survives, and the # sibling marker's matches keep flowing (the channel is alive — - # silence is the pause, not a dead stream) + # silence is the pause, not a dead stream). Both families' streams + # run through the paused window: the count stays frozen for either. print(".. pausing m-icmp: its signals stop while the siblings keep firing") compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp", {"enabled": False}) harness.docker_ping(daemon, c1, R2_IP, count=3) - harness.wait_until(lambda: len(_matches(marker_ws, "m-tx")) >= 8, timeout=10) + harness.docker_ping(daemon, c1, R2_IP6, count=3) + harness.wait_until( + lambda: len(_matches(marker_ws, "m-tx")) >= 8 and len(_matches(marker_ws, "m-icmp6")) >= 16, timeout=10 + ) assert len(_matches(marker_ws, "m-icmp")) == 10, "a paused marker must not signal" + assert len(_matches(marker_ws, "m-icmp6")) == 16, "the 3 v6 pings must reach the v6 marker" assert os.path.exists(pcap) # resume: signals come back, pcap keeps appending print(".. resuming m-icmp") compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp", {"enabled": True}) harness.docker_ping(daemon, c1, R2_IP, count=2) - harness.wait_until(lambda: len(_matches(marker_ws, "m-icmp")) >= 14, timeout=10) + harness.docker_ping(daemon, c1, R2_IP6, count=2) + harness.wait_until( + lambda: len(_matches(marker_ws, "m-icmp")) >= 14 and len(_matches(marker_ws, "m-icmp6")) >= 20, timeout=10 + ) assert len(_matches(marker_ws, "m-icmp")) == 14 + assert len(_matches(marker_ws, "m-icmp6")) == 20 assert harness.pcap_records(pcap)[0] == 14 + assert harness.pcap_records(pcap6)[0] == 20 if shutil.which("sharkd") is None: print(".. sharkd is not installed: skipping the tag replay section") @@ -261,16 +328,66 @@ def test_marker_kernel_datapath(): status, gate = _raw(compute, f"/projects/{pid}/markers/tags/{TAG}/replay/range") assert status == 409, (status, gate) - # delete: the pcap goes with the marker and the signals stop - print(".. deleting m-icmp: pcap and signals go with it") + # the same replay over the v6 tag: identical mechanics, v6 decode + print(f".. pausing m-icmp6 and replaying tag {TAG6}") + compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp6", {"enabled": False}) + status, timeline6 = _raw(compute, f"/projects/{pid}/markers/tags/{TAG6}/replay/range") + assert status == 200, timeline6 + assert timeline6["tag"] == TAG6 + assert timeline6["frame_count"] == 20, timeline6["frame_count"] + assert timeline6["start"] and timeline6["end"], timeline6 + assert timeline6["sources"] == [ + { + "node_id": n1_id, + "link_id": lid, + "marker": "m-icmp6", + "data_link_type": "DLT_EN10MB", + "count": 20, + } + ], timeline6["sources"] + first6 = timeline6["frames"][0] + assert first6["src"] == R1_IP6 and first6["dst"] == R2_IP6, first6 + assert "ICMPv6" in (first6["proto"] or ""), first6 + + # display filter: echo requests only (5 + 3 + 2 of the 20) + query = urllib.parse.urlencode({"filter": "icmpv6.type == 128"}) + status, filtered6 = _raw(compute, f"/projects/{pid}/markers/tags/{TAG6}/replay/range?{query}") + assert status == 200, filtered6 + assert filtered6["frame_count"] == 10, filtered6["frame_count"] + # unknown link: an empty stream, same shape as a zero-match filter + query = urllib.parse.urlencode({"link": "00000000-0000-0000-0000-000000000000"}) + status, empty6 = _raw(compute, f"/projects/{pid}/markers/tags/{TAG6}/replay/range?{query}") + assert status == 200 and empty6["frame_count"] == 0 and empty6["frames"] == [], empty6 + # window endpoint: the whole capture inside one window + query = urllib.parse.urlencode({"ts": timeline6["start"], "window_ms": 120000, "limit": 1000}) + status, window6 = _raw(compute, f"/projects/{pid}/markers/tags/{TAG6}/replay/frames?{query}") + assert status == 200 and len(window6["frames"]) == 20, window6 + + # the gate: replay refuses while the tag is still capturing + compute.call("PUT", f"/projects/{pid}/links/{lid}/markers/m-icmp6", {"enabled": True}) + status, gate6 = _raw(compute, f"/projects/{pid}/markers/tags/{TAG6}/replay/range") + assert status == 409, (status, gate6) + + # delete: the pcaps go with their markers and the signals stop + print(".. deleting m-icmp and m-icmp6: pcaps and signals go with them") compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-icmp") assert harness.wait_until(lambda: not os.path.exists(pcap), timeout=10), pcap + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-icmp6") + assert harness.wait_until(lambda: not os.path.exists(pcap6), timeout=10), pcap6 harness.docker_ping(daemon, c1, R2_IP, count=2) - harness.wait_until(lambda: len(_matches(marker_ws, "m-tx")) >= 12, timeout=10) + harness.docker_ping(daemon, c1, R2_IP6, count=2) + # both still-attached directioned siblings prove the streams crossed + # and their events were delivered before the deleted markers are + # asserted frozen + harness.wait_until( + lambda: len(_matches(marker_ws, "m-tx")) >= 12 and len(_matches(marker_ws, "m-tx6")) >= 12, timeout=10 + ) assert len(_matches(marker_ws, "m-icmp")) == 14, "a deleted marker must not signal" + assert len(_matches(marker_ws, "m-icmp6")) == 20, "a deleted marker must not signal" compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-tcp") compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-tx") + compute.call("DELETE", f"/projects/{pid}/links/{lid}/markers/m-tx6") assert not os.listdir(markers_dir), os.listdir(markers_dir) except BaseException: harness.release(server, pid, failed=True) diff --git a/tests/e2e/test_qemu_kernel_datapath.py b/tests/e2e/test_qemu_kernel_datapath.py index 607b4387d..b3c18296b 100644 --- a/tests/e2e/test_qemu_kernel_datapath.py +++ b/tests/e2e/test_qemu_kernel_datapath.py @@ -39,6 +39,9 @@ relay attaches the anchor itself as an AF_PACKET endpoint, so it stays up and carrying while the link rides the uBridge relay. This differs from IOU's relay, where the fabric socket carries the traffic and the anchors sit idle: QEMU has no other leg, the TAP is the netdev either way. + +Every checkpoint runs dual-stack: the IOSv guests carry a ULA v6 address +next to their v4 one and each checkpoint pings both families. """ import re @@ -61,6 +64,10 @@ BOOT_TIMEOUT = 480 IMAGE_PREFERENCE = ("vios-adventerprisek9-m.spa.159-3.m12.qcow2",) GUEST_ETH_IF = "GigabitEthernet0/0" +# the dual-stack half: a ULA mirroring 10.1.1.x +R1_IP6 = "fd00:1:1::1" +R2_IP6 = "fd00:1:1::2" + def _pick_image(server): images = server.compute.qemu_images() @@ -154,13 +161,17 @@ def test_qemu_kernel_datapath(): harness.assert_forwarding(a1) harness.assert_forwarding(a2) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF) - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if=GUEST_ETH_IF) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF, eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if=GUEST_ETH_IF, eth_ipv6=R2_IP6) out = c1.run("show ip int brief") assert re.search(r"GigabitEthernet0/0\s+10\.1\.1\.1\s+\S+\s+\S+\s+up\s+up", out), out + out = c1.run("show ipv6 interface brief") + assert R1_IP6.upper() in out.upper(), out # IOS prints v6 upper-case print(".. both routers configured, pinging through the anchor TAPs") baseline = harness.wait_ping(c1, "10.1.1.2") assert baseline["success"] == 100, baseline["raw"] + baseline6 = harness.wait_ping(c1, R2_IP6) + assert baseline6["success"] == 100, baseline6["raw"] # L2-anchor spec §E.2: with the guests shut the anchors (QEMU's own # netdevs) and the per-link bridge stay silent for 5 s. @@ -171,6 +182,7 @@ def test_qemu_kernel_datapath(): for console in (c1, c2): _eth_shutdown(console, shut=False) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 else: print(".. uBridge without link l2only: skipping the §E.2 assertion") @@ -184,10 +196,15 @@ def test_qemu_kernel_datapath(): delayed = harness.wait_ping(c1, "10.1.1.2") assert delayed["success"] == 100, delayed["raw"] assert delayed["avg"] >= 150, (baseline, delayed) + delayed6 = harness.wait_ping(c1, R2_IP6) + assert delayed6["success"] == 100, delayed6["raw"] + assert delayed6["avg"] >= 150, (baseline6, delayed6) compute.call("PUT", f"/projects/{pid}/links/{lid}", {"filters": {}}) assert "netem" not in harness.qdiscs(a1), harness.qdiscs(a1) fast = harness.wait_ping(c1, "10.1.1.2") assert fast["success"] == 100 and fast["avg"] < harness.FAST_RTT_MS, fast + fast6 = harness.wait_ping(c1, R2_IP6) + assert fast6["success"] == 100 and fast6["avg"] < harness.FAST_RTT_MS, fast6 # suspend: anchor admin-down — the e1000 loses carrier, the bridge # port is disabled and nothing crosses; resume restores @@ -195,19 +212,24 @@ def test_qemu_kernel_datapath(): assert not harness.tap_up(a1) dead = harness.ping(c1, "10.1.1.2", repeat=3) assert dead["success"] == 0, dead["raw"] + dead6 = harness.ping(c1, R2_IP6, repeat=3) + assert dead6["success"] == 0, dead6["raw"] compute.call("PUT", f"/projects/{pid}/links/{lid}", {"suspend": False}) assert harness.tap_up(a1) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 - # capture: on the anchor, a real pcap + # capture: on the anchor, a real pcap — both families inside the window capture = compute.call("POST", f"/projects/{pid}/links/{lid}/capture/start", {"data_link_type": "DLT_EN10MB"}) harness.ping(c1, "10.1.1.2") + harness.ping(c1, R2_IP6) time.sleep(1) compute.call("POST", f"/projects/{pid}/links/{lid}/capture/stop") path = capture["capture_file_path"] count, ethertypes = harness.pcap_records(path) - assert count >= 4, f"{path}: {count} records" + assert count >= 8, f"{path}: {count} records" assert "0800" in ethertypes, (path, ethertypes) + assert "86dd" in ethertypes, (path, ethertypes) # delete / re-create the kernel link: the anchor leaves the bridge # but survives (the running VM holds it open — this is QEMU's own @@ -220,6 +242,7 @@ def test_qemu_kernel_datapath(): bridge = harness.link_bridge_name(lid) assert link["kernel_datapath"] is True, link assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 # node stop: QEMU exits and uBridge deletes every anchor of the # node together with the orphan bridges (a stopped node must not @@ -240,8 +263,9 @@ def test_qemu_kernel_datapath(): c1 = _boot(server, pid, n1_id) assert harness.tap_exists(a1) assert harness.bridge_members(bridge) == sorted([a1, a2]), harness.bridge_members(bridge) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF, eth_ipv6=R1_IP6) assert harness.wait_ping(c1, "10.1.1.2", attempts=5)["success"] == 100 + assert harness.wait_ping(c1, R2_IP6, attempts=5)["success"] == 100 c1.close() c2.close() @@ -307,10 +331,12 @@ def test_qemu_relay_control(): # the L2 hardening is a creation-time property, not a datapath # one: relay anchors are pure L2 too harness.assert_pure_l2(a1) - harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF) - harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if=GUEST_ETH_IF) + harness.configure_ios(c1, "R1", eth_ip="10.1.1.1", eth_if=GUEST_ETH_IF, eth_ipv6=R1_IP6) + harness.configure_ios(c2, "R2", eth_ip="10.1.1.2", eth_if=GUEST_ETH_IF, eth_ipv6=R2_IP6) relay_ping = harness.wait_ping(c1, "10.1.1.2") assert relay_ping["success"] == 100, relay_ping["raw"] + relay_ping6 = harness.wait_ping(c1, R2_IP6) + assert relay_ping6["success"] == 100, relay_ping6["raw"] c1.close() c2.close() From e1a27dadcfdd8bf5ab3afc6bca8fcfc976450017 Mon Sep 17 00:00:00 2001 From: Mark Paronyan Date: Wed, 7 Oct 2026 10:37:13 +0000 Subject: [PATCH 095/101] feat(api): add strict_names and batch node and link creation --- gns3server/api/routes/controller/links.py | 60 +++++++++++--- gns3server/api/routes/controller/nodes.py | 51 +++++++++++- gns3server/controller/controller_error.py | 18 +++++ gns3server/controller/node.py | 7 +- gns3server/controller/project.py | 51 +++++++----- gns3server/schemas/__init__.py | 5 +- gns3server/schemas/controller/links.py | 11 +++ gns3server/schemas/controller/nodes.py | 16 +++- tests/api/routes/controller/test_links.py | 39 +++++++++ tests/api/routes/controller/test_nodes.py | 98 +++++++++++++++++++++++ tests/controller/test_project.py | 40 +++++++++ 11 files changed, 359 insertions(+), 37 deletions(-) diff --git a/gns3server/api/routes/controller/links.py b/gns3server/api/routes/controller/links.py index b7fa2b40d..896cce8af 100644 --- a/gns3server/api/routes/controller/links.py +++ b/gns3server/api/routes/controller/links.py @@ -32,8 +32,9 @@ from fastapi.responses import FileResponse, StreamingResponse from gns3server import schemas from gns3server.agent.web_wireshark.manager import WebWiresharkManager from gns3server.controller import Controller -from gns3server.controller.controller_error import ControllerError +from gns3server.controller.controller_error import ControllerError, controller_error_status_code from gns3server.controller.link import _UNSET, Link +from gns3server.controller.project import Project from gns3server.db.repositories.rbac import RbacRepository from gns3server.utils.http_client import HTTPClient from gns3server.utils.port_allocator import link_id_to_port @@ -91,7 +92,7 @@ async def get_links(project_id: UUID) -> List[schemas.Link]: }, dependencies=[Depends(has_privilege("Link.Allocate"))], ) -async def create_link(project_id: UUID, link_create: schemas.LinkCreate) -> schemas.Link: +async def create_link(project_id: UUID, link_create: schemas.LinkCreate) -> dict: """ Create a new link. @@ -99,17 +100,22 @@ async def create_link(project_id: UUID, link_create: schemas.LinkCreate) -> sche """ project = await Controller.instance().get_loaded_project(str(project_id)) + return await _create_link(project, link_create) + + +async def _create_link(project: Project, link_create: schemas.LinkCreate) -> dict: + link = await project.add_link() link_data = jsonable_encoder(link_create, exclude_unset=True) - if "filters" in link_data: - await link.update_filters(link_data["filters"]) - if "link_style" in link_data: - await link.update_link_style(link_data["link_style"]) - if "suspend" in link_data: - await link.update_suspend(link_data["suspend"]) - if "show_filters_icon" in link_data: - await link.update_show_filters_icon(link_data["show_filters_icon"]) try: + if "filters" in link_data: + await link.update_filters(link_data["filters"]) + if "link_style" in link_data: + await link.update_link_style(link_data["link_style"]) + if "suspend" in link_data: + await link.update_suspend(link_data["suspend"]) + if "show_filters_icon" in link_data: + await link.update_show_filters_icon(link_data["show_filters_icon"]) for node in link_data["nodes"]: await link.add_node( project.get_node(node["node_id"]), @@ -123,6 +129,40 @@ async def create_link(project_id: UUID, link_create: schemas.LinkCreate) -> sche return link.asdict() +@router.post( + "/batch", + response_model=List[schemas.LinkBatchResult], + responses={ + 404: {"model": schemas.ErrorMessage, "description": "Could not find project"}, + }, + dependencies=[Depends(has_privilege("Link.Allocate"))], +) +async def create_links(project_id: UUID, links_create: List[schemas.LinkCreate]) -> List[schemas.LinkBatchResult]: + """ + Create several links in the order they are listed. + + The response lists one result per link in the same order. A link that fails to be created + does not stop the following ones. + + Required privilege: Link.Allocate + """ + + project = await Controller.instance().get_loaded_project(str(project_id)) + results = [] + for link_create in links_create: + try: + link = await _create_link(project, link_create) + results.append(schemas.LinkBatchResult(status_code=status.HTTP_201_CREATED, link=link)) + except ControllerError as e: + log.error(f"Could not create link in batch: {e}") + results.append( + schemas.LinkBatchResult( + status_code=controller_error_status_code(e), error=schemas.ErrorMessage(message=str(e)) + ) + ) + return results + + @router.get("/{link_id}/available_filters", dependencies=[Depends(has_privilege("Link.Audit"))]) async def get_filters(link: Link = Depends(dep_link)) -> List[dict]: """ diff --git a/gns3server/api/routes/controller/nodes.py b/gns3server/api/routes/controller/nodes.py index 8b22a9ce4..badf51bf2 100644 --- a/gns3server/api/routes/controller/nodes.py +++ b/gns3server/api/routes/controller/nodes.py @@ -34,7 +34,12 @@ from fastapi.routing import APIRoute from gns3server import schemas from gns3server.config import Config from gns3server.controller import Controller -from gns3server.controller.controller_error import ControllerBadRequestError, ControllerForbiddenError +from gns3server.controller.controller_error import ( + ControllerBadRequestError, + ControllerError, + ControllerForbiddenError, + controller_error_status_code, +) from gns3server.controller.node import Node from gns3server.controller.project import Project from gns3server.db.repositories.rbac import RbacRepository @@ -128,13 +133,20 @@ def _check_node_type(node: Node, *required_types: str) -> None: }, dependencies=[Depends(has_privilege("Node.Allocate"))], ) -async def create_node(node_create: schemas.NodeCreate, project: Project = Depends(dep_project)) -> schemas.Node: +async def create_node(node_create: schemas.NodeCreate, project: Project = Depends(dep_project)) -> dict: """ Create a new node. + Set strict_names to get a 409 error instead of an automatic rename when the node name is already used. + Required privilege: Node.Allocate """ + return await _create_node(project, node_create) + + +async def _create_node(project: Project, node_create: schemas.NodeCreate) -> dict: + controller = Controller.instance() compute = controller.get_compute(str(node_create.compute_id)) node_data = jsonable_encoder(node_create, exclude_unset=True) @@ -142,6 +154,41 @@ async def create_node(node_create: schemas.NodeCreate, project: Project = Depend return node.asdict() +@router.post( + "/batch", + response_model=List[schemas.NodeBatchResult], + responses={ + 404: {"model": schemas.ErrorMessage, "description": "Could not find project"}, + }, + dependencies=[Depends(has_privilege("Node.Allocate"))], +) +async def create_nodes( + nodes_create: List[schemas.NodeCreate], project: Project = Depends(dep_project) +) -> List[schemas.NodeBatchResult]: + """ + Create several nodes in the order they are listed. + + The response lists one result per node in the same order. A node that fails to be created + does not stop the following ones. + + Required privilege: Node.Allocate + """ + + results = [] + for node_create in nodes_create: + try: + node = await _create_node(project, node_create) + results.append(schemas.NodeBatchResult(status_code=status.HTTP_201_CREATED, node=node)) + except ControllerError as e: + log.error(f"Could not create node {node_create.name} in batch: {e}") + results.append( + schemas.NodeBatchResult( + status_code=controller_error_status_code(e), error=schemas.ErrorMessage(message=str(e)) + ) + ) + return results + + @router.get( "", response_model=List[schemas.Node], diff --git a/gns3server/controller/controller_error.py b/gns3server/controller/controller_error.py index c6f531498..01fac79b4 100644 --- a/gns3server/controller/controller_error.py +++ b/gns3server/controller/controller_error.py @@ -75,3 +75,21 @@ class ComputeConflictError(ComputeError): def response(self): return self._response + + +def controller_error_status_code(error: ControllerError) -> int: + """ + HTTP status code returned by the API for a controller error. + """ + + if isinstance(error, ControllerTimeoutError): + return 408 + if isinstance(error, ControllerUnauthorizedError): + return 401 + if isinstance(error, ControllerForbiddenError): + return 403 + if isinstance(error, ControllerNotFoundError): + return 404 + if isinstance(error, ControllerBadRequestError): + return 400 + return 409 diff --git a/gns3server/controller/node.py b/gns3server/controller/node.py index e96d4b704..7716aae28 100644 --- a/gns3server/controller/node.py +++ b/gns3server/controller/node.py @@ -100,7 +100,9 @@ class Node: "default_password", ] - def __init__(self, project, compute, name, node_id=None, node_type=None, template_id=None, **kwargs): + def __init__( + self, project, compute, name, node_id=None, node_type=None, template_id=None, strict_names=False, **kwargs + ): """ :param project: Project of the node :param compute: Compute where the server will run @@ -108,6 +110,7 @@ class Node: :param node_id: UUID of the node (integer) :param node_type: Type of emulator :param template_id: Template ID used to create this node + :param strict_names: Fail instead of renaming the node when the name is already used :param kwargs: Node properties """ @@ -125,7 +128,7 @@ class Node: self._label = None self._links = set() self._name = None - self.name = name + self._name = project.update_node_name(self, name, strict=strict_names) self._console = None self._console_type = None self._aux = None diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index c5aedf731..fb99cb4f4 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -557,17 +557,20 @@ class Project: if name in self._allocated_node_names: self._allocated_node_names.remove(name) - def update_allocated_node_name(self, base_name): + def update_allocated_node_name(self, base_name, strict=False): """ Updates a node name or generate a new if no node name is available. :param base_name: new node base name + :param strict: raise an error instead of renaming when the name is already taken """ if base_name is None: return None base_name = re.sub(r"[ ]", "", base_name) # remove spaces in node name + if strict and base_name in self._allocated_node_names: + raise ControllerError(f"Node name {base_name} is already used") if base_name in self._allocated_node_names: base_name = re.sub(r"[0-9]+$", "{0}", base_name) @@ -595,11 +598,11 @@ class Project: return name raise ControllerError("A node name could not be allocated (node limit reached?)") - def update_node_name(self, node, new_name): + def update_node_name(self, node, new_name, strict=False): if new_name and node.name != new_name: self.remove_allocated_node_name(node.name) - return self.update_allocated_node_name(new_name) + return self.update_allocated_node_name(new_name, strict=strict) return new_name @open_required @@ -630,26 +633,32 @@ class Project: node = await self.add_node(compute, name, node_id, node_type=node_type, **template) return node - async def _create_node(self, compute, name, node_id, node_type=None, allow_missing_image=False, **kwargs): + async def _create_node( + self, compute, name, node_id, node_type=None, allow_missing_image=False, strict_names=False, **kwargs + ): - node = Node(self, compute, name, node_id=node_id, node_type=node_type, **kwargs) - # Hold the lock across the check + POST + register so that concurrent - # node creations on the same compute don't all race past the check and - # each POST /projects (the compute-side sync handler then instantiated - # the Project N times). Once one creation registers the compute, the - # rest see it in the set and return immediately. - async with self._create_node_lock: - if compute not in self._project_created_on_compute: - if compute.id == "local": - data = {"name": self._name, "project_id": self._id, "path": self._path} - else: - data = {"name": self._name, "project_id": self._id} - if self._variables: - data["variables"] = self._variables - await compute.post("/projects", data=data) - self._project_created_on_compute.add(compute) + node = Node(self, compute, name, node_id=node_id, node_type=node_type, strict_names=strict_names, **kwargs) + try: + # Hold the lock across the check + POST + register so that concurrent + # node creations on the same compute don't all race past the check and + # each POST /projects (the compute-side sync handler then instantiated + # the Project N times). Once one creation registers the compute, the + # rest see it in the set and return immediately. + async with self._create_node_lock: + if compute not in self._project_created_on_compute: + if compute.id == "local": + data = {"name": self._name, "project_id": self._id, "path": self._path} + else: + data = {"name": self._name, "project_id": self._id} + if self._variables: + data["variables"] = self._variables + await compute.post("/projects", data=data) + self._project_created_on_compute.add(compute) - await node.create(allow_missing_image=allow_missing_image) + await node.create(allow_missing_image=allow_missing_image) + except BaseException: + self.remove_allocated_node_name(node.name) + raise self._nodes[node.id] = node return node diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index a5eb8406c..d9985eaea 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -35,6 +35,7 @@ from .controller.images import Image, ImageSyncJob, ImageSyncRequest, ImageType, from .controller.links import ( EthernetPortInfo, Link, + LinkBatchResult, LinkCapture, LinkCreate, LinkUpdate, @@ -43,7 +44,7 @@ from .controller.links import ( MarkerUpdate, UDPPortInfo, ) -from .controller.nodes import Node, NodeCapture, NodeCreate, NodeDuplicate, NodeUpdate +from .controller.nodes import Node, NodeBatchResult, NodeCapture, NodeCreate, NodeDuplicate, NodeUpdate from .controller.projects import ( NodeFile, Project, @@ -242,6 +243,7 @@ __all__ = [ "LLMModelConfigUpdate", "LLMModelConfigWithSource", "Link", + "LinkBatchResult", "LinkCapture", "LinkCreate", "LinkUpdate", @@ -256,6 +258,7 @@ __all__ = [ "NetmikoDeviceType", "NetmikoDeviceTypeList", "Node", + "NodeBatchResult", "NodeCapture", "NodeCreate", "NodeDuplicate", diff --git a/gns3server/schemas/controller/links.py b/gns3server/schemas/controller/links.py index ccb85d424..bf9632ece 100644 --- a/gns3server/schemas/controller/links.py +++ b/gns3server/schemas/controller/links.py @@ -20,6 +20,7 @@ from uuid import UUID, uuid4 from pydantic import BaseModel, Field, field_validator +from ..common import ErrorMessage from ..update import PartialUpdateModel from .labels import Label @@ -97,6 +98,16 @@ class Link(LinkBase): ) +class LinkBatchResult(BaseModel): + """ + Outcome of one link of a batch creation request. + """ + + status_code: int = Field(..., description="HTTP status code the same request would have returned on its own") + link: Optional[Link] = Field(None, description="The created link, set when status_code is 201") + error: Optional[ErrorMessage] = Field(None, description="The error, set when the link could not be created") + + class UDPPortInfo(BaseModel): """ UDP port information. diff --git a/gns3server/schemas/controller/nodes.py b/gns3server/schemas/controller/nodes.py index 6d731fc91..ff1d90d2c 100644 --- a/gns3server/schemas/controller/nodes.py +++ b/gns3server/schemas/controller/nodes.py @@ -20,7 +20,7 @@ from uuid import UUID, uuid4 from pydantic import BaseModel, Field -from ..common import ConsoleType, CustomAdapter, NodeStatus +from ..common import ConsoleType, CustomAdapter, ErrorMessage, NodeStatus from ..update import PartialUpdateModel from .labels import Label @@ -168,6 +168,10 @@ class NodeCreate(NodeBase): name: str node_type: NodeType node_id: UUID = Field(default_factory=uuid4) + strict_names: bool = Field( + False, + description="Return a 409 error instead of renaming the node when the requested name is already used", + ) class NodeUpdate(PartialUpdateModel, NodeBase): @@ -205,6 +209,16 @@ class Node(NodeBase): ) +class NodeBatchResult(BaseModel): + """ + Outcome of one node of a batch creation request. + """ + + status_code: int = Field(..., description="HTTP status code the same request would have returned on its own") + node: Optional[Node] = Field(None, description="The created node, set when status_code is 201") + error: Optional[ErrorMessage] = Field(None, description="The error, set when the node could not be created") + + class NodeDuplicate(BaseModel): """ Data to duplicate a node. diff --git a/tests/api/routes/controller/test_links.py b/tests/api/routes/controller/test_links.py index 518eb22f9..33a0ab811 100644 --- a/tests/api/routes/controller/test_links.py +++ b/tests/api/routes/controller/test_links.py @@ -81,6 +81,45 @@ class TestLinkRoutes: assert len(project.links) == 1 assert list(project.links.values())[0].filters == filters + async def test_create_links_batch( + self, app: FastAPI, client: AsyncClient, project: Project, nodes: Tuple[Node, Node] + ) -> None: + + node1, node2 = nodes + valid = { + "nodes": [ + {"node_id": node1.id, "adapter_number": 0, "port_number": 3}, + {"node_id": node2.id, "adapter_number": 2, "port_number": 4}, + ] + } + same_node = { + "nodes": [ + {"node_id": node1.id, "adapter_number": 0, "port_number": 3}, + {"node_id": node1.id, "adapter_number": 0, "port_number": 3}, + ] + } + unknown_node = { + "nodes": [ + {"node_id": str(uuid.uuid4()), "adapter_number": 0, "port_number": 3}, + {"node_id": node2.id, "adapter_number": 2, "port_number": 4}, + ] + } + + with asyncio_patch("gns3server.controller.udp_link.UDPLink.create"): + response = await client.post( + app.url_path_for("create_links", project_id=project.id), json=[same_node, unknown_node, valid] + ) + + assert response.status_code == status.HTTP_200_OK + results = response.json() + assert [r["status_code"] for r in results] == [409, 404, 201] + assert results[0]["link"] is None + assert results[0]["error"]["message"] + assert results[1]["error"]["message"] + assert results[2]["error"] is None + assert results[2]["link"]["link_id"] in project.links + assert len(project.links) == 1 + async def test_create_link_failure( self, app: FastAPI, client: AsyncClient, compute: Compute, project: Project ) -> None: diff --git a/tests/api/routes/controller/test_nodes.py b/tests/api/routes/controller/test_nodes.py index cf441a01f..ed89cfb18 100644 --- a/tests/api/routes/controller/test_nodes.py +++ b/tests/api/routes/controller/test_nodes.py @@ -34,6 +34,7 @@ from pydantic import SecretStr from gns3server.api.routes.controller.nodes import vnc_console, ws_console from gns3server.config import Config from gns3server.controller.compute import Compute +from gns3server.controller.controller_error import ControllerError from gns3server.controller.node import Node from gns3server.controller.project import Project from gns3server.services import auth_service @@ -72,6 +73,103 @@ class TestNodeRoutes: assert response.json()["name"] == "test" assert "name" not in response.json()["properties"] + async def test_create_node_taken_name_is_renamed( + self, app: FastAPI, client: AsyncClient, project: Project, compute: Compute + ) -> None: + + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + payload = {"name": "R1", "node_type": "vpcs", "compute_id": "example.com"} + + first = await client.post(app.url_path_for("create_node", project_id=project.id), json=payload) + second = await client.post(app.url_path_for("create_node", project_id=project.id), json=payload) + + assert first.json()["name"] == "R1" + assert second.status_code == status.HTTP_201_CREATED + assert second.json()["name"] == "R2" + + async def test_create_node_strict_names( + self, app: FastAPI, client: AsyncClient, project: Project, compute: Compute + ) -> None: + + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + payload = {"name": "R1", "node_type": "vpcs", "compute_id": "example.com", "strict_names": True} + + first = await client.post(app.url_path_for("create_node", project_id=project.id), json=payload) + second = await client.post(app.url_path_for("create_node", project_id=project.id), json=payload) + + assert first.status_code == status.HTTP_201_CREATED + assert second.status_code == status.HTTP_409_CONFLICT + assert "R1" in second.json()["message"] + assert len(project.nodes) == 1 + + async def test_create_nodes_batch( + self, app: FastAPI, client: AsyncClient, project: Project, compute: Compute + ) -> None: + + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + nodes = [{"name": "R{0}", "node_type": "vpcs", "compute_id": "example.com"} for _ in range(3)] + + response = await client.post(app.url_path_for("create_nodes", project_id=project.id), json=nodes) + + assert response.status_code == status.HTTP_200_OK + results = response.json() + assert [r["status_code"] for r in results] == [201, 201, 201] + assert [r["node"]["name"] for r in results] == ["R1", "R2", "R3"] + assert all(r["error"] is None for r in results) + assert len(project.nodes) == 3 + + async def test_create_nodes_batch_item_failures_keep_order( + self, app: FastAPI, client: AsyncClient, project: Project, compute: Compute + ) -> None: + + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + nodes = [ + {"name": "R1", "node_type": "vpcs", "compute_id": "example.com", "strict_names": True}, + {"name": "R1", "node_type": "vpcs", "compute_id": "example.com", "strict_names": True}, + {"name": "R1", "node_type": "vpcs", "compute_id": "unknown"}, + {"name": "R1", "node_type": "vpcs", "compute_id": "example.com"}, + ] + + response = await client.post(app.url_path_for("create_nodes", project_id=project.id), json=nodes) + + assert response.status_code == status.HTTP_200_OK + results = response.json() + assert [r["status_code"] for r in results] == [201, 409, 404, 201] + assert results[0]["node"]["name"] == "R1" + assert results[1]["node"] is None + assert "R1" in results[1]["error"]["message"] + assert results[2]["error"]["message"] + assert results[3]["node"]["name"] == "R2" + assert len(project.nodes) == 2 + + async def test_create_nodes_batch_failed_node_releases_name( + self, app: FastAPI, client: AsyncClient, project: Project, compute: Compute + ) -> None: + + compute.post = AsyncioMagicMock(side_effect=ControllerError("compute failure")) + nodes = [{"name": "R1", "node_type": "vpcs", "compute_id": "example.com", "strict_names": True}] + + response = await client.post(app.url_path_for("create_nodes", project_id=project.id), json=nodes) + + assert response.json()[0]["status_code"] == status.HTTP_409_CONFLICT + assert len(project.nodes) == 0 + + response_ok = MagicMock() + response_ok.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response_ok) + response = await client.post(app.url_path_for("create_nodes", project_id=project.id), json=nodes) + + assert response.json()[0]["status_code"] == status.HTTP_201_CREATED + assert response.json()[0]["node"]["name"] == "R1" + async def test_list_node(self, app: FastAPI, client: AsyncClient, project: Project, compute: Compute) -> None: response = MagicMock() diff --git a/tests/controller/test_project.py b/tests/controller/test_project.py index 92d51cd82..b41c7cb53 100644 --- a/tests/controller/test_project.py +++ b/tests/controller/test_project.py @@ -1118,6 +1118,46 @@ async def test_node_name(project): assert node.name == "R3" +@pytest.mark.asyncio +async def test_node_name_strict(project): + + compute = MagicMock() + compute.id = "local" + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + + node = await project.add_node(compute, "R1", None, node_type="vpcs", strict_names=True) + assert node.name == "R1" + with pytest.raises(ControllerError): + await project.add_node(compute, "R1", None, node_type="vpcs", strict_names=True) + assert len(project.nodes) == 1 + + node = await project.add_node(compute, "R1", None, node_type="vpcs", strict_names=False) + assert node.name == "R2" + node = await project.add_node(compute, "T{0}", None, node_type="vpcs", strict_names=True) + assert node.name == "T1" + node = await project.add_node(compute, "T{0}", None, node_type="vpcs", strict_names=True) + assert node.name == "T2" + + +@pytest.mark.asyncio +async def test_node_name_released_when_creation_fails(project): + + compute = MagicMock() + compute.id = "local" + compute.post = AsyncioMagicMock(side_effect=ControllerError("failure")) + + with pytest.raises(ControllerError): + await project.add_node(compute, "R1", None, node_type="vpcs") + + response = MagicMock() + response.json = {"console": 2048} + compute.post = AsyncioMagicMock(return_value=response) + node = await project.add_node(compute, "R1", None, node_type="vpcs", strict_names=True) + assert node.name == "R1" + + @pytest.mark.asyncio async def test_duplicate_node(project): From 96006dcf5828e42a3637e2117d76e0353765d82f Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 7 Oct 2026 22:39:39 +0800 Subject: [PATCH 096/101] docs(features): give packet filters their own doc with the evaluation-order diagram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Packet filters section of docker-kernel-datapath.md was the de-facto filter documentation every other datapath doc pointed at; move it to features/packet-filters.md as the node-type-neutral home (veth or TAP anchor), centered on a mermaid walkthrough of all 13 types through the anchor's egress chain: clsact first (the eBPF stateful classifier's nth / quota / window modes at prio 1, then the bpf expression lines at 10-99, first drop wins), the single netem qdisc behind it (enqueue draws, one time_to_send drained by the delay queue). Clarify a naming trap the table verbs invite: uBridge's tc module only borrows the iproute2 vocabulary — the tc ... tokens are its own hypervisor command names over hand-built netlink messages (RTM_NEWQDISC / RTM_NEWTFILTER), raw bpf() syscalls, embedded distribution tables and instruction array, libpcap as a library only; the tc binary is never invoked outside the test suite's oracles. Verified against the uBridge sources (hypervisor_tc.c, tc_impair.bpf.c, tc_ebpf.c), not just the commands gns3-server sends. docker-kernel-datapath.md keeps a pointer section; the iol doc's filter reference and docs/README.md's tree and catalog point at the new doc. --- docs/README.md | 4 + docs/features/docker-kernel-datapath.md | 37 +------ docs/features/iol-docker-kernel-datapath.md | 2 +- docs/features/packet-filters.md | 109 ++++++++++++++++++++ 4 files changed, 116 insertions(+), 36 deletions(-) create mode 100644 docs/features/packet-filters.md diff --git a/docs/README.md b/docs/README.md index 6c0887b2a..9257914ec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,6 +36,7 @@ docs/ ├── openapi.json # OpenAPI specification ├── features/ # Feature documentation │ ├── compute-controller-setup.md # Controller + Compute architecture & configuration +│ ├── packet-filters.md # Link impairment filters: 13 kernel types + relay │ ├── statistics-api.md # Aggregated statistics API for monitoring │ ├── vnc-websocket-console.md # Browser-based VNC console via WebSocket │ └── web-wireshark-business-process.md # Web Wireshark (Docker + xpra packet capture) @@ -90,6 +91,9 @@ Real-time traffic insight via per-link BPF markers and project-level inherited d ### Marker Tag Replay (`features/marker-tag-replay.md`) Aggregate playback across links keyed by `tag`: once every marker under a tag is paused, their pcaps merge into one timestamp-ordered timeline; frames are decoded on demand via tshark into an isomorphic JSON protocol tree. The cross-link delta of the same packet measures the intermediate node's forwarding latency. +### Packet Filters (`features/packet-filters.md`) +Link impairment filters on both datapaths: the 13 kernel types (one netem qdisc, cls_bpf match-drop, the eBPF stateful classifier) with the per-frame evaluation order, plus the 5 relay userspace types; reconcile semantics, capability gating through `available_filters`, and the window/quota drop models. + ### Docker exec Console (Vendor NOS) (`features/docker-exec-console.md`) Console for vendor NOS containers (SR Linux, XRd, …) whose CLI is a TUI off PID 1: runs the vendor CLI via the Docker exec API, plus `GNS3_SKIP_INIT`/`GNS3_INTERFACE_NAMES` boot knobs and SKIP_INIT volume persistence. diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index d2f7232e1..f9e1f74b0 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -175,7 +175,7 @@ flowchart LR What each stage drops: the reserved-range gate passes LACP, LLDP, 802.1X and STP per the port mask but never PAUSE/PFC (Link-local frames below); the clsact/netem stages drop or impair per the installed filters (Packet filters below). Whichever end hosts capture / markers, its AF_PACKET tap — the dashed sideband above — clones the frame at that anchor: a tap on gvA sees the frame on arrival, a tap on gvB only if the clsact chain let it through. -All dozen-plus GNS3 filter types ride these three kernel objects — nine of them (delay, packet_loss, corrupt, duplicate, rate, reorder, gemodel, seed, limit) are parameters of the single netem qdisc, `bpf` expressions are the cls_bpf filters, and frequency_drop / quota / window_drop map to three of the one eBPF program's four modes (evaluated nth → quota → window → flow; the flow-hash mode has no GNS3 filter type). The complete type-to-object mapping is the Packet filters table below. +All dozen-plus GNS3 filter types ride these three kernel objects — nine of them (delay, packet_loss, corrupt, duplicate, rate, reorder, gemodel, seed, limit) are parameters of the single netem qdisc, `bpf` expressions are the cls_bpf filters, and frequency_drop / quota / window_drop map to three of the one eBPF program's four modes (evaluated nth → quota → window → flow; the flow-hash mode has no GNS3 filter type). The complete type-to-object mapping and the per-frame evaluation order are documented in `packet-filters.md`. ## Datapath selection @@ -261,40 +261,7 @@ marker enable_kernel # off = installed but silent ## Packet filters -Impairment filters run **in the kernel** as tc netem qdiscs on the veth host ends (uBridge `tc netem set`, raw netlink — no `tc` binary needed): - -| GNS3 filter | kernel implementation | Notes | -|---|---|---| -| `delay [ms, jitter, dist]` | netem `delay X jitter Y [distribution D]` | jitter 0 omitted; dist ∈ uniform/normal/pareto/paretonormal (kernel-only parameter) | -| `packet_loss [%, correl]` | netem `loss P [correl C]` | per direction (see below); correlation is a kernel-only parameter | -| `corrupt [%]` | netem `corrupt P` | | -| `duplicate [%, correl]` | netem `dup P [correl C]` | kernel-only type | -| `rate ["512kbit"]` | netem `rate B` | tc-style integer+unit (bit/kbit/mbit/gbit/bps/kbps/mbps, ≤100gbit); kernel-only type | -| `reorder [%, correl, gap]` | netem `reorder P [correl C] [gap G]` | requires `delay`; kernel-only type | -| `gemodel [p, r, 1-h]` | netem `loss gemodel P R H` | Gilbert-Elliot bursty loss: p = good→bad transition, r = bad→good transition, 1-h = loss chance in the bad state (good loses nothing) — mean one-way loss p/(p+r)×(1-h); mutually exclusive with `packet_loss`; kernel-only type | -| `seed [u32]` | netem `seed N` | reproducible random draws; kernel-only type | -| `limit [pkts]` | netem `limit N` | queue depth above the 1000 default (rate+delay BDP); kernel-only type | -| `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | -| `frequency_drop [N]` | `tc nth_drop N` — the eBPF stateful classifier's exact every-Nth mode (clsact egress prio 1) | needs uBridge `ebpf=1`; -1 = drop everything → every 1st; per-direction counters (see below) | -| `quota [bytes, %]` | `tc quota_drop ` — after the byte quota is consumed, each further packet drops with the given chance (100 = hard cutoff) | kernel-only type; needs `ebpf=1`; per-direction byte counters | -| `window_drop [start, outage, %, period?, jitter?]` | `tc window_drop [ []]` — packets drop with the given chance inside `[start, start+outage)` | kernel-only type; needs `ebpf=1`; see the window semantics below | - -Semantics: - -* **Both endpoints** receive the filter dict and attach one qdisc to *their* veth host end. The qdisc's egress covers traffic entering that container, so every direction of the link is impaired exactly once — the same net effect as the relay, where both directions cross the single filtered bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the per-packet serialization time to the RTT. -* **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES optional attributes (rate, correlation, reorder, corrupt, gemodel, distribution: an absent attr keeps its previous value), so re-applying a filter set with a parameter *removed* would silently keep the old value. Every apply therefore starts with `tc reset` (also drops clsact and its bpf_drop filters — re-added right after in the same flow) followed by one `netem set` built from the current filters; no per-parameter diffing. An empty filter set is the reset alone (ENOENT-tolerated). -* **Extension gating.** The netem-extension keywords (rate, reorder, gemodel, dist, seed, limit — plus the correl suffixes, which gate on the `rate` token as the extension-build marker) probe `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. -* **kernel-only vs relay-available.** The extension types, `quota` and `window_drop` have no uBridge relay equivalent: `available_filters` hides them on relay links, setting one on a created relay link returns 409, and a project loaded with such filters on a link that cannot be kernel-wired drops them with a warning (invalid filters are dropped the same way at load). `frequency_drop` runs on both datapaths (relay userspace filter / eBPF every-Nth). -* **frequency_drop counting semantics differ per datapath.** The relay's single filtered bridge counts packets of BOTH directions through one counter; the kernel attaches one classifier per veth end, so each direction drops every Nth independently. For a round-trip measurement with every-Nth N: relay ≈ 1/N of frames (phase-locked), kernel = 1 − ((N−1)/N)² (e.g. N=3 → 55.6 % round-trip loss, measured 57 %). The kernel count is exact (atomic counter), unlike netem's stochastic loss. -* **Classifier order** on clsact egress: the eBPF stateful filter runs at prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the stateful modes never reaches the expression drops, and dropped packets never reach netem. Within the stateful program the modes evaluate in the fixed order nth → quota → window → flow. -* **window_drop semantics.** `start` is relative to the moment the filter is applied — and because reconcile re-sets the classifier on every apply (any filter change, node restart, link reset reloads the program after the netem reset), every such event **restarts the schedule**. 3 parameters = one single outage (`[start, start+outage)`), traffic passes before and after — the classic "link dies at T, comes back at T+outage". 4 parameters = recurring flap: `period ≥ outage` (validated), so the outage occupies `outage/period` of each cycle. 5 parameters = randomized flap: each cycle's outage and period are re-drawn uniformly in nominal ± `jitter` (integer-ms grid; jitter 0 draws nothing and equals the fixed schedule). Inside a window each packet drops with the given chance (100 = full outage, lower = degraded service during the window). Both endpoints run their own schedule, started ~simultaneously by the controller — a round trip survives only when **both** directions are outside their windows. Note `start = 0` (outage begins immediately) is a valid, active configuration — unlike other filters, a zero first parameter does not mean "disabled". A PUT re-sending an unchanged filters dict is normally a controller no-op, but one still carrying `window_drop` is reconciled anyway, so re-applying the same window re-arms a one-shot outage that has already passed (arm the filter immediately before generating traffic, or use a period for a repeatable event). -* **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + unit ≤ 100gbit. -* **Node restart** restores the qdisc from the NIO (like capture and markers); **link deletion** detaches it explicitly — the veth survives the link and an orphaned qdisc would keep impairing the next one. -* **Suspend** keeps the qdisc (carrier-driven loss); the first packet after resume takes one extra delay interval (known netem idle-baseline behaviour) and steady state is exact. -* **bpf reconcile = flush + re-add** (`tc bpf_drop flush`, then one add per line at priorities 10, 11, …). `tc reset` (link delete / filter clear) is the full restore in uBridge: filters → clsact → root qdisc, idempotent. -* **Observability caveat:** clsact drops happen before the AF_PACKET tap points — markers/captures on the same veth do not observe cls_bpf-dropped frames (on the relay, filter-vs-mark ordering determined visibility instead). Also note `tc qdisc show` cannot print a distribution's name (the kernel stores only the sampled table). -* **eBPF mode tokens.** A stateful mode is usable iff `tc capabilities` reports `ebpf=1` **and** lists the mode's token in `ebpf_modes` (uBridge `feature/tc-window`+). Builds predating the field keep the modes that shipped with it — everything but `window`, whose semantics were corrected exactly when the field was added: gating on `ebpf=1` alone would silently install the broken back-to-back-window behaviour on a pre-correction build. Requesting an unusable mode fails the apply with an upgrade error; the cleanup ("off") path only touches modes the build declares. -* **Capability reporting.** The compute exposes the probe through `GET /v3/compute/capabilities` as `ubridge_tc` (netem keyword list, `ebpf`/`cbpf` flags, and the usable `ebpf_modes` — legacy fallback applied), answered by a throwaway uBridge spawned as the server's own user and cached by binary identity — the runtime `ebpf` flag flips between users and kernels for the same binary, so a root-run probe would lie for a non-root server. The controller forwards it through `GET /v3/computes` and `available_filters` hides kernel-datapath filter types an endpoint compute reports it cannot run (both ends apply their own filters, so one incapable compute is enough). Computes that report nothing — older servers, failed probe — keep the full list; apply-time validation (409) stays the guard. +Impairment filters are documented once for every kernel-datapath node type in `packet-filters.md` — the 13 filter types ride three kernel objects on the anchor (one netem qdisc, cls_bpf match-drop classifiers, the eBPF stateful classifier; the frame-path diagram above shows where the chain sits), and the type-to-object mapping, per-frame evaluation order, reconcile behaviour, capability gating and window semantics all live there. `KernelDatapathMixin` keys everything on the anchor interface name, which is why this veth datapath and the TAP datapaths (QEMU, IOU, Dynamips) share the implementation unchanged. ## Link-local frames (LACP, LLDP, 802.1X, STP) diff --git a/docs/features/iol-docker-kernel-datapath.md b/docs/features/iol-docker-kernel-datapath.md index 26d89f8b7..2b83f5648 100644 --- a/docs/features/iol-docker-kernel-datapath.md +++ b/docs/features/iol-docker-kernel-datapath.md @@ -232,7 +232,7 @@ The attach above is `_attach_kernel_link`; its ensures are the frozen ensure-the ## Capture, markers, filters -Inherited from `KernelDatapathMixin` unchanged — everything is keyed on the anchor interface name, which is why a TAP and a veth host end are interchangeable there: `capture start_kernel` / `marker add_kernel` (AF_PACKET), one tc netem qdisc (the full netem surface plus the extensions), `bpf` as cls_bpf match-drop, `frequency_drop`/`quota`/`window_drop` as the eBPF stateful classifier — all capability-gated per compute exactly like the Docker veth datapath (the filter semantics live in `docker-kernel-datapath.md`). +Inherited from `KernelDatapathMixin` unchanged — everything is keyed on the anchor interface name, which is why a TAP and a veth host end are interchangeable there: `capture start_kernel` / `marker add_kernel` (AF_PACKET), one tc netem qdisc (the full netem surface plus the extensions), `bpf` as cls_bpf match-drop, `frequency_drop`/`quota`/`window_drop` as the eBPF stateful classifier — all capability-gated per compute exactly like the Docker veth datapath (the filter semantics live in `packet-filters.md`). Suspend semantics: anchor admin-down. The port bridge's TAP writes fail EIO (tolerated by the delivered uBridge hardening) and its reads fall silent — traffic stops both ways. The runner itself is unaware of the link state (netiomux has no carrier signal), exactly as on the relay datapath where suspend rode the synthetic frequency_drop filter instead. diff --git a/docs/features/packet-filters.md b/docs/features/packet-filters.md new file mode 100644 index 000000000..6c039d078 --- /dev/null +++ b/docs/features/packet-filters.md @@ -0,0 +1,109 @@ + + +> This documentation is AI-generated with reference to actual code and verified against real-kernel testing. AI can make mistakes — please verify against the source code when in doubt. + +# Packet Filters (link impairments) + +## Overview + +Per-link impairment filters — delay, packet loss, corruption, rate limiting, deterministic drops — configured through a link's `filters` dict and applied by **both link endpoints**, so every direction of a link is impaired exactly once. A `delay 100` link therefore measures ≈200 ms RTT; `packet_loss 30` measures ≈51 % round trip (1 − 0.7²). + +Every kernel-datapath node type shares one implementation: `KernelDatapathMixin` keys everything on the anchor interface name, which is why a Docker adapter's veth host end and a QEMU / IOU / Dynamips adapter's TAP are interchangeable. On the kernel datapath the filters run **in the kernel** as tc objects that uBridge installs over netlink (`tc netem set`, raw netlink — no `tc` binary needed): one netem qdisc, cls_bpf match-drop classifiers and one eBPF stateful classifier per anchor. Relay links run the uBridge userspace filters on the single relay bridge instead — five types there (`delay`, `packet_loss`, `corrupt`, `frequency_drop`, `bpf`); the rest are kernel-only. + +## Terms + +| Term | Meaning | +|---|---| +| anchor | the host-side interface a link attaches to — a Docker adapter's veth host end, a QEMU / IOU / Dynamips adapter's persistent TAP | +| tc / qdisc | traffic-control objects the kernel inserts into the anchor's egress path — stages frames pass through in flight, not external packet consumers; uBridge only installs them over netlink | +| `clsact` | the tc classifier carrier on the anchor; its egress chain is where the drop classifiers run | +| netem | the impairment qdisc behind clsact: delay, loss, rate, corrupt, duplicate, … | +| eBPF classifier | the stateful drop classifier at clsact egress prio 1 — modes every-Nth, quota, time window and flow (GNS3 exposes the first three as filter types) | +| `bpf_drop` | match-drop BPF expressions at clsact egress prio 10–99 | +| relay filter | a uBridge userspace filter on the relay bridge — both directions cross the single filtered bridge | + +## Evaluation order (kernel datapath) + +All 13 types in evaluation order for one frame with every filter set — the chain runs once per direction on the destination anchor's egress, the classifier chain first and the single netem qdisc behind it. A drop at any stage ends the frame: whatever the classifiers drop never reaches netem, nor the AF_PACKET tap points — captures and markers will not observe it: + +```mermaid +flowchart LR + subgraph CLS["clsact egress — the classifier chain"] + subgraph EB["eBPF stateful classifier — prio 1"] + M1{"frequency_drop (nth)"} + M2{"quota"} + M3{"window_drop (window)"} + end + BP{"bpf lines — prio 10-99"} + end + subgraph NET["netem root qdisc"] + DUP["duplicate — copy queued"] + LOS{"packet_loss / gemodel"} + COR["corrupt"] + TS["delay · reorder · rate"] + TFQ["delay queue — limit"] + SEED["seed"] + end + FR["frame from the link bridge"] + OUT["into the node"] + DROP["dropped"] + + FR --> M1 --> M2 --> M3 --> BP + BP --> DUP --> LOS --> COR --> TS --> TFQ --> OUT + M1 -->|"every Nth"| DROP + M2 -->|"cap reached"| DROP + M3 -->|"in window"| DROP + BP -->|"match"| DROP + LOS -->|"hit"| DROP + TFQ -->|"full"| DROP +``` + +Semantic model rather than kernel source order: the netem draws fire at enqueue (a `duplicate` hit queues a copy alongside the original, `corrupt` damages and passes), `delay`/`reorder`/`rate` fold into one time_to_send that the delay queue drains when due, `limit` is that queue's capacity — the `full` edge above — and `seed` is no stage but the PRNG behind every stochastic draw (duplicate, loss/gemodel, corrupt, jitter). The eBPF program's fourth mode (flow) has no GNS3 filter type, and the attached parameters (jitter, correlation, distribution) ride their types and add no stages. On the relay datapath there is no such chain — the five relay-supported types run as uBridge userspace filters on the single relay bridge, both directions through one filter list. + +## Filter types + +On the kernel datapath the 13 types ride three kernel objects on the anchor. The `tc …` verbs in the table are uBridge's own hypervisor command names — **not the iproute2 `tc` binary**: uBridge builds every netlink message itself (RTM_NEWQDISC for the netem qdisc, RTM_NEWTFILTER for the cls_bpf and eBPF classifiers), loads the eBPF program through raw `bpf()` syscalls with the instruction array embedded in the binary, embeds the jitter-distribution tables the `tc` CLI would read from disk, and uses libpcap only as a library to compile `bpf` expressions — the same compiler the relay filter uses. The `tc` and `ip` binaries appear only in uBridge's test suite, as kernel-state oracles: + +| GNS3 filter | kernel implementation | Notes | +|---|---|---| +| `delay [ms, jitter, dist]` | netem `delay X jitter Y [distribution D]` | jitter 0 omitted; dist ∈ uniform/normal/pareto/paretonormal (kernel-only parameter) | +| `packet_loss [%, correl]` | netem `loss P [correl C]` | per direction (see below); correlation is a kernel-only parameter | +| `corrupt [%]` | netem `corrupt P` | | +| `duplicate [%, correl]` | netem `dup P [correl C]` | kernel-only type | +| `rate ["512kbit"]` | netem `rate B` | tc-style integer+unit (bit/kbit/mbit/gbit/bps/kbps/mbps, ≤100gbit); kernel-only type | +| `reorder [%, correl, gap]` | netem `reorder P [correl C] [gap G]` | requires `delay`; kernel-only type | +| `gemodel [p, r, 1-h]` | netem `loss gemodel P R H` | Gilbert-Elliot bursty loss: p = good→bad transition, r = bad→good transition, 1-h = loss chance in the bad state (good loses nothing) — mean one-way loss p/(p+r)×(1-h); mutually exclusive with `packet_loss`; kernel-only type | +| `seed [u32]` | netem `seed N` | reproducible random draws; kernel-only type | +| `limit [pkts]` | netem `limit N` | queue depth above the 1000 default (rate+delay BDP); kernel-only type | +| `bpf` (one line per expression, OR) | `tc bpf_drop add ""` — cls_bpf on clsact egress + gact drop | needs uBridge `cbpf=1` (`tc capabilities`, probed once per uBridge process); a line that fails to compile on the compute is skipped with a warning, mirroring the relay | +| `frequency_drop [N]` | `tc nth_drop N` — the eBPF stateful classifier's exact every-Nth mode (clsact egress prio 1) | needs uBridge `ebpf=1`; -1 = drop everything → every 1st; per-direction counters (see below) | +| `quota [bytes, %]` | `tc quota_drop ` — after the byte quota is consumed, each further packet drops with the given chance (100 = hard cutoff) | kernel-only type; needs `ebpf=1`; per-direction byte counters | +| `window_drop [start, outage, %, period?, jitter?]` | `tc window_drop [ []]` — packets drop with the given chance inside `[start, start+outage)` | kernel-only type; needs `ebpf=1`; see the window semantics below | + +## Semantics + +* **Both endpoints** receive the filter dict and attach one qdisc to *their* anchor. The qdisc's egress covers traffic entering that node, so every direction of the link is impaired exactly once — the same net effect as the relay, where both directions cross the single filtered bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the per-packet serialization time to the RTT. +* **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES optional attributes (rate, correlation, reorder, corrupt, gemodel, distribution: an absent attr keeps its previous value), so re-applying a filter set with a parameter *removed* would silently keep the old value. Every apply therefore starts with `tc reset` (also drops clsact and its bpf_drop filters — re-added right after in the same flow) followed by one `netem set` built from the current filters; no per-parameter diffing. An empty filter set is the reset alone (ENOENT-tolerated). +* **Extension gating.** The netem-extension keywords (rate, reorder, gemodel, dist, seed, limit — plus the correl suffixes, which gate on the `rate` token as the extension-build marker) probe `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. +* **kernel-only vs relay-available.** The extension types, `quota` and `window_drop` have no uBridge relay equivalent: `available_filters` hides them on relay links, setting one on a created relay link returns 409, and a project loaded with such filters on a link that cannot be kernel-wired drops them with a warning (invalid filters are dropped the same way at load). `frequency_drop` runs on both datapaths (relay userspace filter / eBPF every-Nth). +* **frequency_drop counting semantics differ per datapath.** The relay's single filtered bridge counts packets of BOTH directions through one counter; the kernel attaches one classifier per anchor, so each direction drops every Nth independently. For a round-trip measurement with every-Nth N: relay ≈ 1/N of frames (phase-locked), kernel = 1 − ((N−1)/N)² (e.g. N=3 → 55.6 % round-trip loss, measured 57 %). The kernel count is exact (atomic counter), unlike netem's stochastic loss. +* **Classifier order** on clsact egress: the eBPF stateful filter runs at prio 1, `bpf_drop` expressions at 10–99 — a packet dropped by the stateful modes never reaches the expression drops, and dropped packets never reach netem. Within the stateful program the modes evaluate in the fixed order nth → quota → window → flow. +* **window_drop semantics.** `start` is relative to the moment the filter is applied — and because reconcile re-sets the classifier on every apply (any filter change, node restart, link reset reloads the program after the netem reset), every such event **restarts the schedule**. 3 parameters = one single outage (`[start, start+outage)`), traffic passes before and after — the classic "link dies at T, comes back at T+outage". 4 parameters = recurring flap: `period ≥ outage` (validated), so the outage occupies `outage/period` of each cycle. 5 parameters = randomized flap: each cycle's outage and period are re-drawn uniformly in nominal ± `jitter` (integer-ms grid; jitter 0 draws nothing and equals the fixed schedule). Inside a window each packet drops with the given chance (100 = full outage, lower = degraded service during the window). Both endpoints run their own schedule, started ~simultaneously by the controller — a round trip survives only when **both** directions are outside their windows. Note `start = 0` (outage begins immediately) is a valid, active configuration — unlike other filters, a zero first parameter does not mean "disabled". A PUT re-sending an unchanged filters dict is normally a controller no-op, but one still carrying `window_drop` is reconciled anyway, so re-applying the same window re-arms a one-shot outage that has already passed (arm the filter immediately before generating traffic, or use a period for a repeatable event). +* **Validation** mirrors the tc grammar: `reorder` requires `delay`, `gemodel` and `packet_loss` are mutually exclusive (both map to the netem loss keyword), `distribution` requires jitter > 0, rate must be integer + unit ≤ 100gbit. +* **Node restart** restores the qdisc from the NIO (like capture and markers); **link deletion** detaches it explicitly — the anchor survives the link and an orphaned qdisc would keep impairing the next one. +* **Suspend** keeps the qdisc (carrier-driven loss); the first packet after resume takes one extra delay interval (known netem idle-baseline behaviour) and steady state is exact. +* **bpf reconcile = flush + re-add** (`tc bpf_drop flush`, then one add per line at priorities 10, 11, …). `tc reset` (link delete / filter clear) is the full restore in uBridge: filters → clsact → root qdisc, idempotent. +* **Observability caveat:** clsact drops happen before the AF_PACKET tap points — markers/captures on the same anchor do not observe cls_bpf-dropped frames (on the relay, filter-vs-mark ordering determined visibility instead). Also note `tc qdisc show` cannot print a distribution's name (the kernel stores only the sampled table). +* **eBPF mode tokens.** A stateful mode is usable iff `tc capabilities` reports `ebpf=1` **and** lists the mode's token in `ebpf_modes` (uBridge `feature/tc-window`+). Builds predating the field keep the modes that shipped with it — everything but `window`, whose semantics were corrected exactly when the field was added: gating on `ebpf=1` alone would silently install the broken back-to-back-window behaviour on a pre-correction build. Requesting an unusable mode fails the apply with an upgrade error; the cleanup ("off") path only touches modes the build declares. +* **Capability reporting.** The compute exposes the probe through `GET /v3/compute/capabilities` as `ubridge_tc` (netem keyword list, `ebpf`/`cbpf` flags, and the usable `ebpf_modes` — legacy fallback applied), answered by a throwaway uBridge spawned as the server's own user and cached by binary identity — the runtime `ebpf` flag flips between users and kernels for the same binary, so a root-run probe would lie for a non-root server. The controller forwards it through `GET /v3/computes` and `available_filters` hides kernel-datapath filter types an endpoint compute reports it cannot run (both ends apply their own filters, so one incapable compute is enough). Computes that report nothing — older servers, failed probe — keep the full list; apply-time validation (409) stays the guard. + +## REST surface + +Filters travel on the link API: `PUT /v3/links/{link_id}` with a `filters` dict, validated by the controller (the dependency and exclusivity rules above return 409 on violation) and pushed to both endpoints on create, update and project reopen. Each link payload carries read-only `available_filters` — the per-type intersection of what both endpoint computes can run, per the capability reporting above. + +## Where the chain sits + +The per-node-type frame paths — where an anchor's egress chain sits in the whole journey from one node to another — are drawn in the datapath docs: `docker-kernel-datapath.md` (veth), `qemu-kernel-datapath.md` and `dynamips-kernel-datapath.md` (TAP), `iou-kernel-datapath.md`, `ethernet-switch-kernel-datapath.md` and `iol-docker-kernel-datapath.md`. The frozen uBridge contract behind the kernel objects (netlink encoding, `tc capabilities`, the full-restore `tc reset`) is `../design/ubridge-kernel-impairment-spec.md`. From 5be4f52a2fddb8780ad8843e7ab96c76a42f957b Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 7 Oct 2026 22:47:15 +0800 Subject: [PATCH 097/101] docs(features): describe how the 13 filter types combine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A "Combining filters" section for packet-filters.md: deterministic drops are an OR-gate (first kill wins — a clsact-dropped frame consumes no rate serialization, no limit capacity and no netem counter), netem draws are independent probabilities that multiply in one qdisc pass, and a round trip squares everything. The couplings that do exist: quota's byte count sees only frequency_drop survivors (ordered evaluation slows the burn), delay/reorder/rate fold into one time_to_send, limit x rate x delay is the tail-drop triangle to watch, and seed pins netem's draws but not the eBPF program's own PRNG. Closes with a fully-loaded 12-filter walkthrough of one frame through the gauntlet. --- docs/features/packet-filters.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/features/packet-filters.md b/docs/features/packet-filters.md index 6c039d078..39c68aa00 100644 --- a/docs/features/packet-filters.md +++ b/docs/features/packet-filters.md @@ -100,6 +100,18 @@ On the kernel datapath the 13 types ride three kernel objects on the anchor. The * **eBPF mode tokens.** A stateful mode is usable iff `tc capabilities` reports `ebpf=1` **and** lists the mode's token in `ebpf_modes` (uBridge `feature/tc-window`+). Builds predating the field keep the modes that shipped with it — everything but `window`, whose semantics were corrected exactly when the field was added: gating on `ebpf=1` alone would silently install the broken back-to-back-window behaviour on a pre-correction build. Requesting an unusable mode fails the apply with an upgrade error; the cleanup ("off") path only touches modes the build declares. * **Capability reporting.** The compute exposes the probe through `GET /v3/compute/capabilities` as `ubridge_tc` (netem keyword list, `ebpf`/`cbpf` flags, and the usable `ebpf_modes` — legacy fallback applied), answered by a throwaway uBridge spawned as the server's own user and cached by binary identity — the runtime `ebpf` flag flips between users and kernels for the same binary, so a root-run probe would lie for a non-root server. The controller forwards it through `GET /v3/computes` and `available_filters` hides kernel-datapath filter types an endpoint compute reports it cannot run (both ends apply their own filters, so one incapable compute is enough). Computes that report nothing — older servers, failed probe — keep the full list; apply-time validation (409) stays the guard. +## Combining filters + +The composition model in one line: **deterministic drops are an OR-gate** (first kill wins, the rest never run), **netem draws are independent probabilities that multiply**, and **a round trip squares everything** — the only genuine coupling inside one direction is `limit` backstopping the backlog that `rate` + `delay` create. + +Classifier-stage composition: a frame dropped at clsact never reaches netem, so it consumes no `rate` serialization time, no delay-queue capacity (`limit`), and no netem counter — and captures/markers will not see it (the observability caveat above). The stateful modes interact only through their counters: `quota`'s byte count sees just the `frequency_drop` survivors (evaluation is ordered), so every-Nth drops slow the quota burn by the nth pass rate; the `window_drop` schedule and the nth counter are independent of each other. A frame the eBPF program lets pass continues to the `bpf` lines — the program returns `TC_ACT_UNSPEC` on a pass, not `TC_ACT_OK`, so the prio chain continues — making the two classifier stages a chain, not an exclusion. + +Within netem, the nine types are **one qdisc pass, not nine stages**: duplicate, loss/gemodel and corrupt are independent draws (their probabilities multiply; a duplicated copy draws its own delay; corrupt damages and passes, leaving the receiver's checksum to do the dropping), while `delay`/`reorder`/`rate` fold into a single time_to_send. The coupling to watch is `limit` × `rate` × `delay`: rate serializes while delay holds, so the backlog grows toward the bandwidth-delay product — once it passes `limit`, the queue tail-drops and the link loses packets no filter asked to lose. Raise `limit` when combining a large `rate` with a large `delay`. `seed` pins only the netem draws (duplicate, loss, corrupt, jitter) — the eBPF program's percentage draws (quota, window) ride a separate PRNG seeded at program load, outside `seed`'s reach. + +Round-trip composition: both endpoints run their own instance on their own anchor (each direction impaired exactly once), so per-direction survival multiplies across drop sources and then squares for a round trip — `delay 100` → ≈200 ms RTT, `packet_loss 30` → ≈51 % round-trip loss, `frequency_drop 3` → 55.6 % (the per-type arithmetic in Semantics above); `window_drop` schedules start ~simultaneously on both ends, so a round trip survives only when **both** directions are outside their windows, and `quota` burns two independent byte counters. The combinations that cannot merge — `packet_loss` + `gemodel` (one netem loss slot), `reorder` without `delay`, `distribution` without jitter, `period < outage` — are rejected at validation (409), never silently merged. + +A fully-loaded example — `delay 100 jitter 10`, `packet_loss 10`, `corrupt 5`, `duplicate 10`, `rate 512kbit`, `reorder 25 gap 5`, `limit 5000`, `seed 42`, `bpf "icmp"`, `frequency_drop 3`, `quota 3000000 100`, `window_drop [1000, 1500, 100]` — sends one A→B frame through this gauntlet at B's anchor: it dies exactly on every 3rd send (nth), else it counts toward a 3 MB budget after which it is hard-cut (quota), else it drops when it lands inside the 1–2.5 s outage (window), else it drops when the expression matches (bpf); a survivor enters netem, where it may be duplicated (the copy draws its own delay), lost (10 %), or corrupted (5 %, passes damaged), and otherwise waits 100 ± 10 ms with 25 % reorder while 512 kbit/s serializes it inside a 5000-packet queue — every draw reproducible under `seed 42`. The B→A direction runs the same gauntlet independently on A's anchor. + ## REST surface Filters travel on the link API: `PUT /v3/links/{link_id}` with a `filters` dict, validated by the controller (the dependency and exclusivity rules above return 409 on violation) and pushed to both endpoints on create, update and project reopen. Each link payload carries read-only `available_filters` — the per-type intersection of what both endpoint computes can run, per the capability reporting above. From d640050d4b29afdafecf0ed86f4cf872ea390050 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Thu, 8 Oct 2026 19:42:18 +0800 Subject: [PATCH 098/101] docs(features): quantify the kernel netem correl bias that collapses small rates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel's correlated-random generator (get_crandom, sch_netem.c) truncates instead of wrapping, starving both edges of the stationary distribution: an event threshold below the correlation value is almost never reached. Measured with stock iproute2 on this kernel — uBridge's netlink encoding is identical: loss 1% + correl 25% drops nothing over 2000 pings (expected ~2%), 10%/25% measures ~1.8% one way, only the 50% symmetry point survives. duplicate and reorder correl share the same comparison structure and collapse the same way; gemodel is immune (Markov chain over plain uniform draws) and is the honest tool for bursty loss at a chosen mean rate. Documented in the packet-filters correlation-bias bullet, and surfaced in the two machine-facing descriptions (MCP link_update packet_loss note, controller filter schema) so agents and clients stop walking into the trap. --- docs/features/packet-filters.md | 3 ++- gns3server/agent/mcp/__init__.py | 5 ++++- gns3server/controller/link.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/features/packet-filters.md b/docs/features/packet-filters.md index 39c68aa00..e096dcfc0 100644 --- a/docs/features/packet-filters.md +++ b/docs/features/packet-filters.md @@ -70,7 +70,7 @@ On the kernel datapath the 13 types ride three kernel objects on the anchor. The | GNS3 filter | kernel implementation | Notes | |---|---|---| | `delay [ms, jitter, dist]` | netem `delay X jitter Y [distribution D]` | jitter 0 omitted; dist ∈ uniform/normal/pareto/paretonormal (kernel-only parameter) | -| `packet_loss [%, correl]` | netem `loss P [correl C]` | per direction (see below); correlation is a kernel-only parameter | +| `packet_loss [%, correl]` | netem `loss P [correl C]` | per direction (see below); correlation is a kernel-only parameter — and collapses small rates, see the correlation-bias bullet | | `corrupt [%]` | netem `corrupt P` | | | `duplicate [%, correl]` | netem `dup P [correl C]` | kernel-only type | | `rate ["512kbit"]` | netem `rate B` | tc-style integer+unit (bit/kbit/mbit/gbit/bps/kbps/mbps, ≤100gbit); kernel-only type | @@ -86,6 +86,7 @@ On the kernel datapath the 13 types ride three kernel objects on the anchor. The ## Semantics * **Both endpoints** receive the filter dict and attach one qdisc to *their* anchor. The qdisc's egress covers traffic entering that node, so every direction of the link is impaired exactly once — the same net effect as the relay, where both directions cross the single filtered bridge. A `delay 100` link measures ≈200 ms RTT; `packet_loss 30` measures ≈51 % round-trip (1 − 0.7²); `rate 512kbit` adds ≈2× the per-packet serialization time to the RTT. +* **Correlation collapses small rates (kernel netem defect, quantified).** netem's correlated draw mixes `answer = (value·(2³²−ρ) + last·ρ) >> 32` — truncated, never wrapped modulo 2³² — so the stationary distribution is starved at both edges: a draw almost never lands in the bottom ρ slice of the range. The effective rate of nominal P under correl C collapses toward zero whenever P ≲ C. Reproduced with stock iproute2 `tc` on this kernel (uBridge's netlink encoding is identical — GNS3 shows the same numbers): `1% 25%` → measured 0/1000 pings (theoretical ≈0.0002 %), `5% 25%` → ≈0.2 %, `10% 25%` → ≈1.8 % one-way (3.8 % round-trip measured), `25% 25%` → ≈16.7 %; nominal 50 % stays 50 % under every correl (the distribution's symmetry point). The same edge-starvation suppresses small `duplicate`/`reorder` rates under their correl and thins the tails of correlated jitter. `gemodel` is immune — a Markov chain over plain uniform draws — and is the honest tool for bursty loss at a chosen mean rate (p/(p+r)×(1-h)). Not fixable server- or uBridge-side without deviating from stock `tc` behavior; the kernel fix would be to wrap the mixing modulo 2³². * **Reconcile = reset + full re-apply.** The kernel's netem replace MERGES optional attributes (rate, correlation, reorder, corrupt, gemodel, distribution: an absent attr keeps its previous value), so re-applying a filter set with a parameter *removed* would silently keep the old value. Every apply therefore starts with `tc reset` (also drops clsact and its bpf_drop filters — re-added right after in the same flow) followed by one `netem set` built from the current filters; no per-parameter diffing. An empty filter set is the reset alone (ENOENT-tolerated). * **Extension gating.** The netem-extension keywords (rate, reorder, gemodel, dist, seed, limit — plus the correl suffixes, which gate on the `rate` token as the extension-build marker) probe `tc capabilities` once per uBridge process and require the tokens; a plain delay/loss/corrupt/dup filter never probes — an old uBridge serves the original surface untouched. * **kernel-only vs relay-available.** The extension types, `quota` and `window_drop` have no uBridge relay equivalent: `available_filters` hides them on relay links, setting one on a created relay link returns 409, and a project loaded with such filters on a link that cannot be kernel-wired drops them with a warning (invalid filters are dropped the same way at load). `frequency_drop` runs on both datapaths (relay userspace filter / eBPF every-Nth). diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index 00bef7e88..27cce38b9 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -808,7 +808,10 @@ async def link_update( * frequency_drop: [N] - Drop every Nth packet (N: -1 to 32767; -1 = drop everything). Exact on kernel links (eBPF counter) * packet_loss: [chance, correl?] - Loss percentage (0-100); correl (K) - makes consecutive losses dependent (bursty) + makes consecutive losses dependent (bursty). CAVEAT: the kernel netem + correl suppresses small rates — a chance below ~the correl value drops + almost nothing (1% + correl 25% ≈ zero loss); use gemodel for bursty + loss at a known mean rate * delay: [ms, jitter?, distribution?] - Latency 1-32767 ms, jitter 0-32767 ms; distribution (K) one of uniform|normal|pareto|paretonormal, needs jitter > 0 diff --git a/gns3server/controller/link.py b/gns3server/controller/link.py index 6d2fa355e..4db47a766 100644 --- a/gns3server/controller/link.py +++ b/gns3server/controller/link.py @@ -54,7 +54,9 @@ FILTERS = [ "type": "packet_loss", "name": "Packet loss", "description": "The percentage represents the chance for a packet to be lost. The optional correlation " - "(kernel-datapath links only) makes consecutive losses dependent, like a bursty real network", + "(kernel-datapath links only) makes consecutive losses dependent, like a bursty real network. " + "Kernel caveat: a chance at or below the correlation value collapses toward zero effective loss " + "(netem correlated-RNG bias) — use gemodel for bursty loss at a known mean rate", "parameters": [ {"name": "Chance", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, {"name": "Correlation", "minimum": 0, "maximum": 100, "type": "int", "unit": "%"}, From a9e5f859c8836ccbe2bf5b2f2318969125849e9c Mon Sep 17 00:00:00 2001 From: Jeremy Grossmann Date: Thu, 8 Oct 2026 14:49:12 +0200 Subject: [PATCH 099/101] Reorder import statements in test_nodes.py --- tests/api/routes/controller/test_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/routes/controller/test_nodes.py b/tests/api/routes/controller/test_nodes.py index 812780619..be05b9cf3 100644 --- a/tests/api/routes/controller/test_nodes.py +++ b/tests/api/routes/controller/test_nodes.py @@ -34,7 +34,7 @@ from pydantic import SecretStr from gns3server.api.routes.controller.nodes import vnc_console, ws_console from gns3server.config import Config from gns3server.controller.compute import Compute -from gns3server.controller.controller_error import ControllerError, ComputeError +from gns3server.controller.controller_error import ComputeError, ControllerError from gns3server.controller.node import Node from gns3server.controller.project import Project from gns3server.services import auth_service From 57286c4f09cf8a5ef5a45d04ce1c65eb1a6a9791 Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 8 Oct 2026 16:01:51 +0200 Subject: [PATCH 100/101] style: fix lint errors --- gns3server/api/routes/controller/links.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gns3server/api/routes/controller/links.py b/gns3server/api/routes/controller/links.py index 2376b6fec..89b2b8395 100644 --- a/gns3server/api/routes/controller/links.py +++ b/gns3server/api/routes/controller/links.py @@ -32,7 +32,11 @@ from fastapi.responses import FileResponse, StreamingResponse from gns3server import schemas from gns3server.agent.web_wireshark.manager import WebWiresharkManager from gns3server.controller import Controller -from gns3server.controller.controller_error import ControllerError, ControllerNotFoundError, controller_error_status_code +from gns3server.controller.controller_error import ( + ControllerError, + ControllerNotFoundError, + controller_error_status_code, +) from gns3server.controller.link import _UNSET, Link from gns3server.controller.project import Project from gns3server.db.repositories.rbac import RbacRepository @@ -204,7 +208,7 @@ async def create_links(project_id: UUID, links_create: List[schemas.LinkCreate]) ) return results - + @router.get( "/{link_id}/available_filters", response_model=List[schemas.LinkFilterDefinition], From 4195f3b7c16ca13192a563412c15a3fadbf10a61 Mon Sep 17 00:00:00 2001 From: grossmj Date: Fri, 9 Oct 2026 14:29:04 +0200 Subject: [PATCH 101/101] fix: remove incorrectly merged code --- gns3server/api/routes/controller/links.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/gns3server/api/routes/controller/links.py b/gns3server/api/routes/controller/links.py index 0acd09dab..22cb64f0e 100644 --- a/gns3server/api/routes/controller/links.py +++ b/gns3server/api/routes/controller/links.py @@ -151,13 +151,6 @@ async def _create_link(project: Project, link_create: schemas.LinkCreate) -> dic await link.update_suspend(link_data["suspend"]) if "show_filters_icon" in link_data: await link.update_show_filters_icon(link_data["show_filters_icon"]) - for node in link_data["nodes"]: - await link.add_node( - project.get_node(node["node_id"]), - node.get("adapter_number", 0), - node.get("port_number", 0), - label=node.get("label"), - ) endpoints = link_data["nodes"] attached_port = None for index, endpoint in enumerate(endpoints):