From 187201a08ca9726a4ff97ce1929fd2ab47928609 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Wed, 30 Sep 2026 18:56:57 +0800 Subject: [PATCH] feat(controller): offer QEMU the kernel datapath, gated per compute Kernel eligibility becomes "both endpoints on one compute, each able to anchor": Docker always (adapters are born as veth pairs), QEMU when that compute's uBridge can create the persistent TAPs its adapters anchor on. The question is asked per compute, because the link needs both ends anchored -- so a mixed or half-upgraded deployment degrades to the relay instead of failing an attach. probe_tap_support() creates and deletes one throwaway persistent TAP through a scratch uBridge, cached by binary identity like the tc report; /capabilities exposes `ubridge_tap` (absent = unknown, which keeps links on the relay). The compute route for a QEMU adapter NIO took UDPNIO only, so a kernel link was rejected with 422: its body and response now accept Union[UDPNIO, BridgeNIO] exactly like Docker's. Stale "Docker-to-Docker" wording in the config, link and MCP descriptions now talks about anchors, and docs/features/qemu-kernel-datapath.md documents the datapath -- verified end-to-end on a real instance: 32/32 kernel-datapath checks and 29/29 on the relay variant. --- docs/features/docker-kernel-datapath.md | 7 +- docs/features/qemu-kernel-datapath.md | 160 ++++++++++++++++++ gns3server/agent/mcp/__init__.py | 7 +- gns3server/api/routes/compute/capabilities.py | 7 +- gns3server/api/routes/compute/qemu_nodes.py | 15 +- gns3server/compute/ubridge/tc_probe.py | 71 ++++++++ gns3server/controller/compute.py | 1 + gns3server/controller/udp_link.py | 47 +++-- gns3server/schemas/config.py | 5 +- gns3server/schemas/controller/capabilities.py | 5 + gns3server/schemas/controller/links.py | 5 +- tests/api/routes/compute/test_capabilities.py | 25 +++ tests/compute/ubridge/test_tc_probe.py | 32 +++- tests/controller/test_compute.py | 2 +- tests/controller/test_kernel_datapath_link.py | 95 ++++++++++- 15 files changed, 444 insertions(+), 40 deletions(-) create mode 100644 docs/features/qemu-kernel-datapath.md diff --git a/docs/features/docker-kernel-datapath.md b/docs/features/docker-kernel-datapath.md index fad77439a..0be8facef 100644 --- a/docs/features/docker-kernel-datapath.md +++ b/docs/features/docker-kernel-datapath.md @@ -364,8 +364,11 @@ bridge add_nio_ethernet / add_nio_udp / start / stop / start_capture / stop_capt enable_kernel_datapath = True ``` -Intended to grow into the global datapath switch as QEMU / IOU / Dynamips -migrate to kernel bridges (QEMU: tap enslaved per-link bridge, zero relay). +Intended to grow into the global datapath switch as IOU / Dynamips migrate to +kernel bridges. **QEMU has landed** on the same pattern (persistent TAP anchor +enslaved into the per-link bridge, zero relay) — see +`docs/features/qemu-kernel-datapath.md`; the shared half of this document's +compute machinery now lives in `gns3server/compute/kernel_datapath.py`. ## Verification diff --git a/docs/features/qemu-kernel-datapath.md b/docs/features/qemu-kernel-datapath.md new file mode 100644 index 000000000..045b43dad --- /dev/null +++ b/docs/features/qemu-kernel-datapath.md @@ -0,0 +1,160 @@ + + +# QEMU kernel datapath + +QEMU adapters anchor on a **persistent TAP**, and links between two QEMU +nodes — or a QEMU node and a Docker container — on the same compute are wired +in the kernel (per-link Linux bridge) instead of the uBridge UDP relay. The +compute side is the same machinery Docker already uses: `KernelDatapathMixin` +(`gns3server/compute/kernel_datapath.py`) takes an *anchor interface* and does +not care whether it is a veth host end or a TAP. + +## Architecture + +Kernel link between two QEMU nodes on one compute: + +``` + QEMU A (fd) root namespace QEMU B (fd) + ┌──────────┐ -netdev ┌──────────────────┐ -netdev ┌──────────┐ + │ e1000 │◄───tap──────┤ gq{idA}e0p0 ────┼──► gns3{link_id_prefix} + └──────────┘ │ (bridge port) │ (bridge port) │ + │ per-link kernel │ gq{idB}e0p0 │ + │ bridge = "cable"│ │ + └──────────────────┘ +``` + +Relay link on the same anchor (cross-compute, mixed node types, or +`enable_kernel_datapath` off): + +``` + ┌──────────┐ tap ┌────────────────────────────┐ UDP ┌──────────┐ + │ QEMU A │◄────────►│ uBridge relay bridge │◄───────►│ peer │ + │ (fd) │ gq…e0p0 │ add_nio_ethernet ↔ add_ │ tunnel │ (any) │ + └──────────┘ │ nio_udp │ └──────────┘ + └────────────────────────────┘ +``` + +The relay attaches the TAP with `bridge add_nio_ethernet` (AF_PACKET, which +does not take the device over) — **never** `bridge add_nio_tap`, which would +claim the fd QEMU itself holds. + +## Why a TAP works like a veth host end + +| Event | Docker anchor (veth host end) | QEMU anchor (TAP) | +|---|---|---| +| Frames the node sends | arrive on the veth as `PACKET_HOST` | written to the tap fd, arrive as RX (`PACKET_HOST`) | +| Frames the peer sends | injected by uBridge, seen as `PACKET_OUTGOING` | injected by uBridge, seen as `PACKET_OUTGOING` | +| `tc` egress qdisc impairs | traffic entering the container | traffic entering the VM | +| Capture/marker anchor | the interface | the same interface | + +So every filter, capture and marker primitive carries over verbatim, and +`delay 100` on both anchors still measures ≈200 ms round trip while each +direction is impaired exactly once. + +## Datapath selection + +`UDPLink._kernel_datapath_eligible` needs both endpoints on one compute and +both able to anchor: + +* Docker always can (adapters are born as veth pairs); +* QEMU can when that compute's uBridge reports the `tap` module — asked per + compute through `/capabilities` (`ubridge_tap`, probed by creating and + deleting one throwaway TAP, cached by binary identity), because an old + uBridge leaves QEMU on the legacy socket-netdev datapath and cannot anchor + a kernel link. An unknown/failed probe keeps the link on the relay. + +Because the `-netdev` type is fixed when QEMU is launched, this is +all-or-nothing per VM: on the TAP datapath *every* link can attach to (or +detach from) a running VM, including switching a link between the two +datapaths (delete + re-create, or a project reopen). + +## Adapter lifecycle + +* **Birth (node start, before QEMU is launched)** — `_prepare_tap_datapath` + probes the tap module, then `_create_taps` sweeps any leftover (a persistent + TAP outlives a crash) and creates one TAP per adapter: + `tap create gq{node_id[:8]}e{adapter}p{port}` → `tap set_owner ` + (so the unprivileged QEMU process can open it) → `link set down` + (carrier off until a link attaches). QEMU then opens it as + `-netdev tap,id=gns3-{adapter},ifname=,script=no,downscript=no`. +* **Life (running)** — the TAP is never created or destroyed again. Link + create/delete/switch, suspend, capture and markers only change *what is + attached to it*. +* **Death (node stop)** — `_remove_taps` un-persists the TAPs (`tap delete`; + QEMU's open fd keeps each device alive until it exits) and deletes the + per-link kernel bridges the node still holds. Deleting a node never runs the + link-teardown path, so those bridges would otherwise stay behind as empty + orphans; a restart rebuilds them from the NIO. + +## Link operations + +| Operation | Kernel-datapath action | +|---|---| +| Create / attach | `brctl create` (EEXIST tolerated via `brctl show`) + `link set up` + `brctl addif ` + carrier up + markers + `tc netem set` (filters) | +| Delete / detach | marker teardown → `tc reset ` → carrier off → `brctl delif` → `brctl delete` (last endpoint wins; EBUSY/ENOENT suppressed). The TAP survives — the adapter keeps it for the next link | +| Reset (`POST /links/{id}/reset`) | delete + create (re-evaluates eligibility) | +| Suspend (`PUT /links/{id}` `{"suspend": true}`) | tap admin-down (writing to a down TAP fd fails with EIO, so the link is genuinely dead) + QMP `set_link gns3-N off` so the guest notices; resume restores both, and the netem qdisc survives the flap | + +## Capture, markers, filters + +Identical to Docker's kernel links, on the TAP: `capture start_kernel ` +(driven by the NIO type, not by a link flag), `marker add_kernel …` +(only the capture node's end), and one `tc netem set` per anchor so each +direction is impaired once. `frequency_drop`/`quota`/`window_drop` run as the +eBPF classifier on the anchor's clsact egress. + +## Legacy fallback + +A uBridge build without the `tap` module (or without `CAP_NET_ADMIN`) keeps +the previous datapath untouched: local UDP tunnels into uBridge, QEMU launched +with `-netdev socket`, and the node logs a warning that it cannot carry kernel +links. Such a node is never offered a kernel link, so nothing fails. + +## Verified + +End-to-end through a real server (isolated instance, two QEMU nodes with a +stub binary, the test playing the guest by holding the TAP fds and pushing +frames): **32/32 checks** on the kernel datapath and **29/29** on the relay +variant (`enable_kernel_datapath = false`). Kernel run highlights: + +* anchors created persistent and owned by the server user, QEMU's netdev is + the TAP, no local UDP tunnel left; `link_list` reports `kernel_datapath: true` + and one per-link bridge holds both taps; +* frames cross in both directions (~0.1 ms); `delay 100` ⇒ netem on both + anchors and a measured **100.3 ms** one-way; +* suspend ⇒ anchor DOWN, no traffic, resume ⇒ 100.3 ms again (filter kept); +* capture ⇒ a 252-byte pcap written by `capture start_kernel`; +* link delete ⇒ bridge gone, qdisc clean, anchor alive; re-create ⇒ 0.1 ms + with no residual impairment; node stop ⇒ no taps, no bridges left. + +Relay run the same script with the flag off: same lifecycle, filter measured +at 109.8 ms (applied by uBridge's bridge — no netem on the anchor, as +expected), no kernel bridge holding the tap. + +## Configuration + +```ini +[Server] +# Wire eligible links (both endpoints on one compute, both able to anchor) +# through kernel interfaces instead of the uBridge UDP relay. +enable_kernel_datapath = True +``` + +## Known caveat + +Anchors are host-side netdevs, so the kernel gives them an IPv6 link-local +address and emits MLD/DAD from them; those frames flood into the emulated +segment. The hardening requirement (anchors must be pure L2, no L3 identity) +is specified for uBridge in `docs/design/ubridge-l2-anchor-spec.md` — it +applies to QEMU TAPs and Docker veths alike. + +## Roadmap + +IOU (its fabric would need a kernel-side port in `iol_bridge`), Dynamips +(hypervisor-created taps, enslavable as they are), and the Ethernet switch / +cloud paths (their anchors are ubridge-owned by design) are still on the +relay; cross-compute kernel links need VXLAN/GENEVE encapsulation, which is +a separate project (it would serve Docker links the same way). diff --git a/gns3server/agent/mcp/__init__.py b/gns3server/agent/mcp/__init__.py index 1c5398233..232eb0193 100644 --- a/gns3server/agent/mcp/__init__.py +++ b/gns3server/agent/mcp/__init__.py @@ -801,9 +801,10 @@ async def link_update( Supported kwargs: - suspend: boolean - Suspend or resume the link - filters: dict - Packet filters (values MUST be positional arrays). Types - marked (K) run on kernel-datapath links only (Docker-to-Docker on one - compute) and are rejected with 409 on relay links; which types a given - link accepts is authoritatively answered by link_available_filters: + marked (K) run on kernel-datapath links only (both endpoints on one + compute with an anchor of their own) and are rejected with 409 on relay + links; which types a given link accepts is authoritatively answered by + link_available_filters: * frequency_drop: [N] - Drop every Nth packet (N: -1 to 32767; -1 = drop everything). Exact on kernel links (eBPF counter) * packet_loss: [chance, correl?] - Loss percentage (0-100); correl (K) diff --git a/gns3server/api/routes/compute/capabilities.py b/gns3server/api/routes/compute/capabilities.py index 055d0fb05..f2a51d702 100644 --- a/gns3server/api/routes/compute/capabilities.py +++ b/gns3server/api/routes/compute/capabilities.py @@ -25,7 +25,7 @@ from fastapi import APIRouter, Request from gns3server import schemas from gns3server.compute import MODULES -from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities +from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support from gns3server.utils.path import get_default_project_directory from gns3server.utils.tc_capabilities import usable_ebpf_modes from gns3server.version import __version__ @@ -52,6 +52,10 @@ async def get_capabilities(request: Request) -> dict: "cbpf": caps.get("cbpf") == "1", } + # Persistent TAP support: the anchor QEMU adapters need for the kernel + # datapath. None = unknown (probe failed or uBridge too old). + ubridge_tap = await probe_tap_support() + # record the controller hostname or IP address if request.client: request.app.state.controller_host = request.client.host @@ -64,4 +68,5 @@ async def get_capabilities(request: Request) -> dict: "disk_size": psutil.disk_usage(get_default_project_directory()).total, "node_types": node_types, "ubridge_tc": ubridge_tc, + "ubridge_tap": ubridge_tap, } diff --git a/gns3server/api/routes/compute/qemu_nodes.py b/gns3server/api/routes/compute/qemu_nodes.py index 17bcb3ca2..ca858d245 100644 --- a/gns3server/api/routes/compute/qemu_nodes.py +++ b/gns3server/api/routes/compute/qemu_nodes.py @@ -269,18 +269,19 @@ async def resume_qemu_node(node: QemuVM = Depends(dep_node)) -> None: @router.post( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def create_qemu_node_nio( *, adapter_number: int, port_number: int = Path(..., ge=0, le=0), - nio_data: schemas.UDPNIO, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], node: QemuVM = Depends(dep_node), -) -> schemas.UDPNIO: +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ - Add a NIO (Network Input/Output) to the node. + Add a NIO (Network Input/Output) to the node: a UDP relay NIO, or a + kernel-datapath bridge NIO the node enslaves its adapter TAP into. The port number on the Qemu node is always 0. """ @@ -292,16 +293,16 @@ async def create_qemu_node_nio( @router.put( "/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio", status_code=status.HTTP_201_CREATED, - response_model=schemas.UDPNIO, + response_model=Union[schemas.UDPNIO, schemas.BridgeNIO], dependencies=[Depends(compute_authentication)], ) async def update_qemu_node_nio( *, adapter_number: int, port_number: int = Path(..., ge=0, le=0), - nio_data: schemas.UDPNIO, + nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO], node: QemuVM = Depends(dep_node), -) -> schemas.UDPNIO: +) -> Union[schemas.UDPNIO, schemas.BridgeNIO]: """ Update a NIO (Network Input/Output) on the node. The port number on the Qemu node is always 0. diff --git a/gns3server/compute/ubridge/tc_probe.py b/gns3server/compute/ubridge/tc_probe.py index 33f2962b1..f6ba7a3ec 100644 --- a/gns3server/compute/ubridge/tc_probe.py +++ b/gns3server/compute/ubridge/tc_probe.py @@ -49,6 +49,8 @@ log = logging.getLogger(__name__) # binary identity -> parsed report (None = probed and unusable) _cache = {} +# binary identity -> whether the tap module works (None = unknown) +_tap_cache = {} _lock = asyncio.Lock() @@ -112,3 +114,72 @@ async def _probe(path, config, timeout): with contextlib.suppress(Exception): await hypervisor.stop() shutil.rmtree(working_dir, ignore_errors=True) + + +async def probe_tap_support(timeout: float = 15.0): + """ + Whether this host's uBridge can create persistent TAPs (the ``tap`` + module), probed the same way as the tc report — a throwaway uBridge, the + answer cached by binary identity. QEMU adapters anchor on such a TAP, so + this is what makes a QEMU node kernel-datapath eligible; an old build + answers "Unknown command", reported as None (unknown) and cached as such. + """ + + config = Config.instance() + path = shutil.which(config.settings.Server.ubridge_path) + if not path: + return None + try: + stat = os.stat(path) + key = (path, stat.st_mtime_ns, stat.st_size) + except OSError: + return None + + async with _lock: + if key in _tap_cache: + return _tap_cache[key] + supported = await _probe_tap(path, config, timeout) + _tap_cache[key] = supported + return supported + + +async def _probe_tap(path, config, timeout): + """ + Spawn a throwaway uBridge and try to create (then delete) one persistent + TAP. The probe device is named after a uuid so a concurrent probe on the + same host cannot collide; a delete that fails leaves it behind, which is + worth a warning but not a failure of the probe. + """ + + working_dir = tempfile.mkdtemp(prefix="gns3-tap-probe-") + hypervisor = Hypervisor( + None, + path, + working_dir, + config.settings.Server.ubridge_control_transport, + config.settings.Server.host, + str(uuid.uuid4()), + ) + name = f"gns3tap{uuid.uuid4().hex[:6]}" + try: + + async def ask(): + await hypervisor.start() + await hypervisor.connect() + await hypervisor.send(f"tap create {name}") + try: + await hypervisor.send(f"tap delete {name}") + except UbridgeError as e: + log.warning("Probe TAP %s could not be deleted: %s", name, e) + return True + + try: + return await asyncio.wait_for(ask(), timeout=timeout) + except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e: + # Missing module, old build, or no CAP_NET_ADMIN: unknown. + log.debug("uBridge tap-module probe failed: %s", e) + return None + finally: + with contextlib.suppress(Exception): + await hypervisor.stop() + shutil.rmtree(working_dir, ignore_errors=True) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index ef2593f66..3c470b1bb 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -100,6 +100,7 @@ class Compute: "disk_size": 0, "node_types": [], "ubridge_tc": None, + "ubridge_tap": None, } self.name = name # Cache of interfaces on remote host diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index e2e9602d7..9eca6924a 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -163,30 +163,47 @@ class UDPLink(Link): def _kernel_datapath_eligible(self, node1, node2): """ - Whether this link can be wired on the kernel datapath (veth pairs - enslaved into a per-link Linux bridge) instead of the uBridge UDP - relay. Every GNS3 filter type has a kernel equivalent — netem for - delay/packet_loss/corrupt and the netem extensions, cls_bpf for bpf, - the eBPF stateful classifier for frequency_drop — so filters no - longer disqualify the kernel path. Capture and markers are served by - uBridge's AF_PACKET modules on the veth host end. + Whether this link can be wired on the kernel datapath (the adapters' + host-side anchors enslaved into a per-link Linux bridge) instead of + the uBridge UDP relay. Every GNS3 filter type has a kernel equivalent + — netem for delay/packet_loss/corrupt and the netem extensions, + cls_bpf for bpf, the eBPF stateful classifier for frequency_drop — so + filters no longer disqualify the kernel path. Capture and markers are + served by uBridge's AF_PACKET modules on the anchor. - Docker adapters are born as veth pairs (unified interface), so the - datapath is a runtime decision — links attach to running containers - too: the compute side enslaves the veth host end (brctl) or attaches - the relay to it (add_nio_ethernet) without touching the container's - interfaces. + Every adapter owns an anchor, so the datapath is a runtime decision — + links attach to running nodes too: the compute side enslaves the + anchor (brctl) or attaches the relay to it (add_nio_ethernet) without + touching the interfaces the node itself uses. Both endpoints must be + on the same compute (there is no kernel link across hosts) and both + must be able to anchor: Docker adapters are born as veth pairs, + QEMU adapters as persistent TAPs, which need uBridge's tap module — + asked of that compute, since an old uBridge leaves QEMU on the relay + datapath and cannot carry a kernel link at all. """ if not Config.instance().settings.Server.enable_kernel_datapath: return False - if node1.node_type != "docker" or node2.node_type != "docker": - return False if node1.compute.id != node2.compute.id: return False if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2): return False - return True + return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2) + + @staticmethod + def _kernel_endpoint_ready(node): + """ + Whether a node can anchor a kernel link on its compute. + """ + + if node.node_type == "docker": + return True + if node.node_type != "qemu": + return False + capabilities = node.compute.capabilities or {} + # Strict True: an unreported capability (old uBridge, failed probe) + # keeps the link on the relay, where it always works. + return capabilities.get("ubridge_tap") is True async def _prepare(self): """ diff --git a/gns3server/schemas/config.py b/gns3server/schemas/config.py index fc83f39fd..e17084a0e 100644 --- a/gns3server/schemas/config.py +++ b/gns3server/schemas/config.py @@ -185,8 +185,9 @@ class ServerSettings(BaseModel): allow_raw_images: bool = Field(True, description="Allow raw images to be uploaded to the server") enable_kernel_datapath: bool = Field( True, - description="Wire eligible Docker-to-Docker links (same compute, no filters/markers) through " - "kernel veth/bridge interfaces instead of the uBridge UDP relay", + description="Wire eligible links (both endpoints on one compute, both able to anchor: Docker veth, " + "or QEMU when the compute's uBridge supports persistent TAPs) through kernel interfaces instead " + "of the uBridge UDP relay", ) auto_discover_images: bool = Field(True, description="Automatically discover images in the images directory") image_sync_interval: int = Field(900, ge=10, description="Seconds between automatic image inventory scans") diff --git a/gns3server/schemas/controller/capabilities.py b/gns3server/schemas/controller/capabilities.py index bd4b8832a..72786f755 100644 --- a/gns3server/schemas/controller/capabilities.py +++ b/gns3server/schemas/controller/capabilities.py @@ -50,3 +50,8 @@ class Capabilities(BaseModel): ubridge_tc: Optional[UbridgeTcCapabilities] = Field( None, description="uBridge tc-module capabilities; absent when the probe failed or uBridge has no tc module" ) + ubridge_tap: Optional[bool] = Field( + None, + description="Whether uBridge can create the persistent TAPs a QEMU adapter anchors on " + "(needed to wire QEMU links on the kernel datapath); absent when the probe failed", + ) diff --git a/gns3server/schemas/controller/links.py b/gns3server/schemas/controller/links.py index dccc33063..47c397855 100644 --- a/gns3server/schemas/controller/links.py +++ b/gns3server/schemas/controller/links.py @@ -83,8 +83,9 @@ class Link(LinkBase): link_type: Optional[LinkType] = None kernel_datapath: Optional[bool] = Field( None, - description="Read only property. True when this link is wired through kernel veth/bridge interfaces " - "(currently Docker-to-Docker on one compute) instead of the uBridge UDP relay", + description="Read only property. True when this link is wired through kernel interfaces " + "(veth or TAP anchors enslaved to a per-link Linux bridge, both endpoints on one compute) " + "instead of the uBridge UDP relay", ) capturing: Optional[bool] = Field(None, description="Read only property. True if a capture running on the link") capture_file_name: Optional[str] = Field( diff --git a/tests/api/routes/compute/test_capabilities.py b/tests/api/routes/compute/test_capabilities.py index f3216daea..4934ae6f2 100644 --- a/tests/api/routes/compute/test_capabilities.py +++ b/tests/api/routes/compute/test_capabilities.py @@ -28,6 +28,12 @@ from gns3server.version import __version__ pytestmark = pytest.mark.asyncio +async def _no_tap_probe(): + """The tap probe spawns uBridge — stub it out for payload assertions.""" + + return None + + class TestCapabilitiesRoutes: async def test_get(self, app: FastAPI, compute_client: AsyncClient, windows_platform, monkeypatch) -> None: @@ -35,6 +41,7 @@ class TestCapabilitiesRoutes: return None monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -59,6 +66,7 @@ class TestCapabilitiesRoutes: "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, "ubridge_tc": None, + "ubridge_tap": None, } async def test_get_on_gns3vm(self, app: FastAPI, compute_client: AsyncClient, on_gns3vm, monkeypatch) -> None: @@ -67,6 +75,7 @@ class TestCapabilitiesRoutes: return None monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", no_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json() == { @@ -91,6 +100,7 @@ class TestCapabilitiesRoutes: "memory": psutil.virtual_memory().total, "disk_size": psutil.disk_usage(get_default_project_directory()).total, "ubridge_tc": None, + "ubridge_tap": None, } async def test_get_reports_ubridge_tc(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: @@ -99,6 +109,7 @@ class TestCapabilitiesRoutes: return {"netem": "delay,rate", "ebpf": "1", "cbpf": "0", "ebpf_modes": "nth,quota,window,flow"} monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tc_capabilities", fake_probe) + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", _no_tap_probe) response = await compute_client.get(app.url_path_for("compute:get_capabilities")) assert response.status_code == status.HTTP_200_OK assert response.json()["ubridge_tc"] == { @@ -107,3 +118,17 @@ class TestCapabilitiesRoutes: "ebpf_modes": ["nth", "quota", "window", "flow"], "cbpf": False, } + + async def test_get_reports_ubridge_tap(self, app: FastAPI, compute_client: AsyncClient, monkeypatch) -> None: + """ + The tap capability is what makes a QEMU node kernel-eligible, so it + travels with the other compute capabilities. + """ + + async def fake_tap_probe(): + return True + + monkeypatch.setattr("gns3server.api.routes.compute.capabilities.probe_tap_support", fake_tap_probe) + response = await compute_client.get(app.url_path_for("compute:get_capabilities")) + assert response.status_code == status.HTTP_200_OK + assert response.json()["ubridge_tap"] is True diff --git a/tests/compute/ubridge/test_tc_probe.py b/tests/compute/ubridge/test_tc_probe.py index 6bb5178cd..3a7d2a640 100644 --- a/tests/compute/ubridge/test_tc_probe.py +++ b/tests/compute/ubridge/test_tc_probe.py @@ -22,7 +22,7 @@ payload: spawn a throwaway uBridge, ask once, cache by binary identity. import pytest from gns3server.compute.ubridge import tc_probe -from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities +from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support from gns3server.compute.ubridge.ubridge_error import UbridgeError pytestmark = pytest.mark.asyncio @@ -31,6 +31,8 @@ pytestmark = pytest.mark.asyncio class FakeHypervisor: """Records spawns; answers `tc capabilities` with a canned reply.""" + last = None + reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] error = None spawned = 0 @@ -38,6 +40,7 @@ class FakeHypervisor: def __init__(self, project, path, working_dir, transport, host, node_id): self.commands = [] + FakeHypervisor.last = self FakeHypervisor.spawned += 1 async def start(self): @@ -66,6 +69,8 @@ def probe_env(monkeypatch, tmp_path): FakeHypervisor.error = None FakeHypervisor.reply = ["netem=delay,rate;ebpf=1;cbpf=1;ebpf_modes=nth,quota,window,flow"] tc_probe._cache.clear() + tc_probe._tap_cache.clear() + FakeHypervisor.last = None # a real file on disk: the cache is keyed on its (path, mtime, size) binary = tmp_path / "ubridge" binary.write_bytes(b"") @@ -113,3 +118,28 @@ async def test_probe_empty_reply_means_unknown(): FakeHypervisor.reply = [] assert await probe_tc_capabilities() is None + + +async def test_probe_tap_support_creates_and_deletes_a_probe_tap(): + + assert await probe_tap_support() is True + assert FakeHypervisor.spawned == 1 + commands = FakeHypervisor.last.commands + assert commands[0].startswith("tap create ") + assert commands[1].startswith("tap delete ") + + +async def test_probe_tap_support_unknown_on_an_old_build(): + + FakeHypervisor.error = "202-Unknown command 'create'" + assert await probe_tap_support() is None + # cached like the tc report: no respawn on every /capabilities hit + assert await probe_tap_support() is None + assert FakeHypervisor.spawned == 1 + + +async def test_probe_tap_support_missing_binary(monkeypatch): + + monkeypatch.setattr(tc_probe.shutil, "which", lambda name: None) + assert await probe_tap_support() is None + assert FakeHypervisor.spawned == 0 diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index a6f41e2d3..1fce70eb1 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -381,7 +381,7 @@ async def test_json(compute): "disk_usage_percent": 0, "connected": True, "last_error": None, - "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None}, + "capabilities": {"version": "", "platform": "", "cpus": 0, "memory": 0, "disk_size": 0, "node_types": [], "ubridge_tc": None, "ubridge_tap": None}, } assert compute.asdict(topology_dump=True) == { "compute_id": "my_compute_id", diff --git a/tests/controller/test_kernel_datapath_link.py b/tests/controller/test_kernel_datapath_link.py index 1ce85ac30..e714994dd 100644 --- a/tests/controller/test_kernel_datapath_link.py +++ b/tests/controller/test_kernel_datapath_link.py @@ -90,18 +90,101 @@ async def test_kernel_datapath_eligible(project): assert link._kernel_datapath_eligible(node1, node2) is True -@pytest.mark.asyncio -async def test_kernel_datapath_not_eligible_for_other_node_types(project): - +def _tap_capable_compute(tap_support=True, compute_id="compute-1"): compute = MagicMock() - compute.id = "compute-1" - node1 = _node(project, compute, "docker1") - node2 = _node(project, compute, "qemu1", node_type="qemu") + compute.id = compute_id + compute.capabilities = {"ubridge_tap": tap_support} + return compute + + +async def _link(project, node1, node2): link = UDPLink(project) await link.add_node(node1, 0, 0, batch=True, dump=False) await link.add_node(node2, 0, 0, batch=True, dump=False) + return link + + +@pytest.mark.asyncio +async def test_kernel_datapath_not_eligible_for_other_node_types(project): + """ + A node type with no anchor of its own (IOU, VPCS, cloud, ...) keeps the + relay however capable the compute is — and a QEMU node on a compute whose + uBridge cannot create persistent TAPs does too, because it runs on the + legacy relay datapath there. + """ + + compute = _tap_capable_compute(tap_support=None) + node1 = _node(project, compute, "docker1") + node2 = _node(project, compute, "qemu1", node_type="qemu") + link = await _link(project, node1, node2) assert link._kernel_datapath_eligible(node1, node2) is False + compute = _tap_capable_compute() + node1 = _node(project, compute, "iou1", node_type="iou") + node2 = _node(project, compute, "docker1") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_kernel_datapath_eligible_for_qemu_with_tap_support(project): + """ + QEMU adapters anchor on a persistent TAP, so a compute whose uBridge has + the tap module gets kernel links too — including the mixed docker-to-qemu + case (both anchors end up in the same per-link kernel bridge). + """ + + compute = _tap_capable_compute() + node1 = _node(project, compute, "qemu1", node_type="qemu") + node2 = _node(project, compute, "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True + + node2 = _node(project, compute, "docker1") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is True + + +@pytest.mark.asyncio +async def test_kernel_datapath_capability_is_asked_per_compute(project): + """ + A QEMU node on a compute without the tap module keeps the link on the + relay even when its peer sits on a capable compute — the link needs both + endpoints anchored, so the capability is a two-end AND. + """ + + capable = _tap_capable_compute(compute_id="compute-1") + plain = _tap_capable_compute(tap_support=None, compute_id="compute-2") + node1 = _node(project, capable, "qemu1", node_type="qemu") + node2 = _node(project, plain, "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + # same compute, but the compute cannot anchor: still the relay + node2 = _node(project, _tap_capable_compute(tap_support=None), "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + assert link._kernel_datapath_eligible(node1, node2) is False + + +@pytest.mark.asyncio +async def test_prepare_qemu_kernel_link_emits_bridge_nios(project): + """ + The prepared link data is the same NIOBridge spec Docker links use — no + UDP ports, no peer resolution — so nothing else in the controller has to + know which node type the endpoint is. + """ + + compute = _tap_capable_compute() + node1 = _node(project, compute, "qemu1", node_type="qemu") + node2 = _node(project, compute, "qemu2", node_type="qemu") + link = await _link(project, node1, node2) + + await link._prepare() + + assert link.kernel_datapath is True + assert [d["type"] for d in link._link_data] == ["nio_bridge", "nio_bridge"] + assert link._link_data[0]["bridge"] == link._link_data[1]["bridge"] + @pytest.mark.asyncio async def test_kernel_datapath_not_eligible_across_computes(project):