mirror of
https://github.com/GNS3/gns3-server.git
synced 2026-11-03 05:51:15 +02:00
feat(controller): offer QEMU the kernel datapath, gated per compute
Kernel eligibility becomes "both endpoints on one compute, each able to anchor": Docker always (adapters are born as veth pairs), QEMU when that compute's uBridge can create the persistent TAPs its adapters anchor on. The question is asked per compute, because the link needs both ends anchored -- so a mixed or half-upgraded deployment degrades to the relay instead of failing an attach. probe_tap_support() creates and deletes one throwaway persistent TAP through a scratch uBridge, cached by binary identity like the tc report; /capabilities exposes `ubridge_tap` (absent = unknown, which keeps links on the relay). The compute route for a QEMU adapter NIO took UDPNIO only, so a kernel link was rejected with 422: its body and response now accept Union[UDPNIO, BridgeNIO] exactly like Docker's. Stale "Docker-to-Docker" wording in the config, link and MCP descriptions now talks about anchors, and docs/features/qemu-kernel-datapath.md documents the datapath -- verified end-to-end on a real instance: 32/32 kernel-datapath checks and 29/29 on the relay variant.
This commit is contained in:
15 files changed
+444
-40
No files matched your search
@@ -801,9 +801,10 @@ async def link_update(
|
||||
Supported kwargs:
|
||||
- suspend: boolean - Suspend or resume the link
|
||||
- filters: dict - Packet filters (values MUST be positional arrays). Types
|
||||
marked (K) run on kernel-datapath links only (Docker-to-Docker on one
|
||||
compute) and are rejected with 409 on relay links; which types a given
|
||||
link accepts is authoritatively answered by link_available_filters:
|
||||
marked (K) run on kernel-datapath links only (both endpoints on one
|
||||
compute with an anchor of their own) and are rejected with 409 on relay
|
||||
links; which types a given link accepts is authoritatively answered by
|
||||
link_available_filters:
|
||||
* frequency_drop: [N] - Drop every Nth packet (N: -1 to 32767; -1 = drop
|
||||
everything). Exact on kernel links (eBPF counter)
|
||||
* packet_loss: [chance, correl?] - Loss percentage (0-100); correl (K)
|
||||
|
||||
@@ -25,7 +25,7 @@ from fastapi import APIRouter, Request
|
||||
|
||||
from gns3server import schemas
|
||||
from gns3server.compute import MODULES
|
||||
from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities
|
||||
from gns3server.compute.ubridge.tc_probe import probe_tc_capabilities, probe_tap_support
|
||||
from gns3server.utils.path import get_default_project_directory
|
||||
from gns3server.utils.tc_capabilities import usable_ebpf_modes
|
||||
from gns3server.version import __version__
|
||||
@@ -52,6 +52,10 @@ async def get_capabilities(request: Request) -> dict:
|
||||
"cbpf": caps.get("cbpf") == "1",
|
||||
}
|
||||
|
||||
# Persistent TAP support: the anchor QEMU adapters need for the kernel
|
||||
# datapath. None = unknown (probe failed or uBridge too old).
|
||||
ubridge_tap = await probe_tap_support()
|
||||
|
||||
# record the controller hostname or IP address
|
||||
if request.client:
|
||||
request.app.state.controller_host = request.client.host
|
||||
@@ -64,4 +68,5 @@ async def get_capabilities(request: Request) -> dict:
|
||||
"disk_size": psutil.disk_usage(get_default_project_directory()).total,
|
||||
"node_types": node_types,
|
||||
"ubridge_tc": ubridge_tc,
|
||||
"ubridge_tap": ubridge_tap,
|
||||
}
|
||||
@@ -269,18 +269,19 @@ async def resume_qemu_node(node: QemuVM = Depends(dep_node)) -> None:
|
||||
@router.post(
|
||||
"/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
response_model=schemas.UDPNIO,
|
||||
response_model=Union[schemas.UDPNIO, schemas.BridgeNIO],
|
||||
dependencies=[Depends(compute_authentication)],
|
||||
)
|
||||
async def create_qemu_node_nio(
|
||||
*,
|
||||
adapter_number: int,
|
||||
port_number: int = Path(..., ge=0, le=0),
|
||||
nio_data: schemas.UDPNIO,
|
||||
nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO],
|
||||
node: QemuVM = Depends(dep_node),
|
||||
) -> schemas.UDPNIO:
|
||||
) -> Union[schemas.UDPNIO, schemas.BridgeNIO]:
|
||||
"""
|
||||
Add a NIO (Network Input/Output) to the node.
|
||||
Add a NIO (Network Input/Output) to the node: a UDP relay NIO, or a
|
||||
kernel-datapath bridge NIO the node enslaves its adapter TAP into.
|
||||
The port number on the Qemu node is always 0.
|
||||
"""
|
||||
|
||||
@@ -292,16 +293,16 @@ async def create_qemu_node_nio(
|
||||
@router.put(
|
||||
"/{node_id}/adapters/{adapter_number}/ports/{port_number}/nio",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
response_model=schemas.UDPNIO,
|
||||
response_model=Union[schemas.UDPNIO, schemas.BridgeNIO],
|
||||
dependencies=[Depends(compute_authentication)],
|
||||
)
|
||||
async def update_qemu_node_nio(
|
||||
*,
|
||||
adapter_number: int,
|
||||
port_number: int = Path(..., ge=0, le=0),
|
||||
nio_data: schemas.UDPNIO,
|
||||
nio_data: Union[schemas.UDPNIO, schemas.BridgeNIO],
|
||||
node: QemuVM = Depends(dep_node),
|
||||
) -> schemas.UDPNIO:
|
||||
) -> Union[schemas.UDPNIO, schemas.BridgeNIO]:
|
||||
"""
|
||||
Update a NIO (Network Input/Output) on the node.
|
||||
The port number on the Qemu node is always 0.
|
||||
|
||||
@@ -49,6 +49,8 @@ log = logging.getLogger(__name__)
|
||||
|
||||
# binary identity -> parsed report (None = probed and unusable)
|
||||
_cache = {}
|
||||
# binary identity -> whether the tap module works (None = unknown)
|
||||
_tap_cache = {}
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
@@ -112,3 +114,72 @@ async def _probe(path, config, timeout):
|
||||
with contextlib.suppress(Exception):
|
||||
await hypervisor.stop()
|
||||
shutil.rmtree(working_dir, ignore_errors=True)
|
||||
|
||||
|
||||
async def probe_tap_support(timeout: float = 15.0):
|
||||
"""
|
||||
Whether this host's uBridge can create persistent TAPs (the ``tap``
|
||||
module), probed the same way as the tc report — a throwaway uBridge, the
|
||||
answer cached by binary identity. QEMU adapters anchor on such a TAP, so
|
||||
this is what makes a QEMU node kernel-datapath eligible; an old build
|
||||
answers "Unknown command", reported as None (unknown) and cached as such.
|
||||
"""
|
||||
|
||||
config = Config.instance()
|
||||
path = shutil.which(config.settings.Server.ubridge_path)
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
key = (path, stat.st_mtime_ns, stat.st_size)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
async with _lock:
|
||||
if key in _tap_cache:
|
||||
return _tap_cache[key]
|
||||
supported = await _probe_tap(path, config, timeout)
|
||||
_tap_cache[key] = supported
|
||||
return supported
|
||||
|
||||
|
||||
async def _probe_tap(path, config, timeout):
|
||||
"""
|
||||
Spawn a throwaway uBridge and try to create (then delete) one persistent
|
||||
TAP. The probe device is named after a uuid so a concurrent probe on the
|
||||
same host cannot collide; a delete that fails leaves it behind, which is
|
||||
worth a warning but not a failure of the probe.
|
||||
"""
|
||||
|
||||
working_dir = tempfile.mkdtemp(prefix="gns3-tap-probe-")
|
||||
hypervisor = Hypervisor(
|
||||
None,
|
||||
path,
|
||||
working_dir,
|
||||
config.settings.Server.ubridge_control_transport,
|
||||
config.settings.Server.host,
|
||||
str(uuid.uuid4()),
|
||||
)
|
||||
name = f"gns3tap{uuid.uuid4().hex[:6]}"
|
||||
try:
|
||||
|
||||
async def ask():
|
||||
await hypervisor.start()
|
||||
await hypervisor.connect()
|
||||
await hypervisor.send(f"tap create {name}")
|
||||
try:
|
||||
await hypervisor.send(f"tap delete {name}")
|
||||
except UbridgeError as e:
|
||||
log.warning("Probe TAP %s could not be deleted: %s", name, e)
|
||||
return True
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(ask(), timeout=timeout)
|
||||
except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e:
|
||||
# Missing module, old build, or no CAP_NET_ADMIN: unknown.
|
||||
log.debug("uBridge tap-module probe failed: %s", e)
|
||||
return None
|
||||
finally:
|
||||
with contextlib.suppress(Exception):
|
||||
await hypervisor.stop()
|
||||
shutil.rmtree(working_dir, ignore_errors=True)
|
||||
@@ -100,6 +100,7 @@ class Compute:
|
||||
"disk_size": 0,
|
||||
"node_types": [],
|
||||
"ubridge_tc": None,
|
||||
"ubridge_tap": None,
|
||||
}
|
||||
self.name = name
|
||||
# Cache of interfaces on remote host
|
||||
|
||||
@@ -163,30 +163,47 @@ class UDPLink(Link):
|
||||
|
||||
def _kernel_datapath_eligible(self, node1, node2):
|
||||
"""
|
||||
Whether this link can be wired on the kernel datapath (veth pairs
|
||||
enslaved into a per-link Linux bridge) instead of the uBridge UDP
|
||||
relay. Every GNS3 filter type has a kernel equivalent — netem for
|
||||
delay/packet_loss/corrupt and the netem extensions, cls_bpf for bpf,
|
||||
the eBPF stateful classifier for frequency_drop — so filters no
|
||||
longer disqualify the kernel path. Capture and markers are served by
|
||||
uBridge's AF_PACKET modules on the veth host end.
|
||||
Whether this link can be wired on the kernel datapath (the adapters'
|
||||
host-side anchors enslaved into a per-link Linux bridge) instead of
|
||||
the uBridge UDP relay. Every GNS3 filter type has a kernel equivalent
|
||||
— netem for delay/packet_loss/corrupt and the netem extensions,
|
||||
cls_bpf for bpf, the eBPF stateful classifier for frequency_drop — so
|
||||
filters no longer disqualify the kernel path. Capture and markers are
|
||||
served by uBridge's AF_PACKET modules on the anchor.
|
||||
|
||||
Docker adapters are born as veth pairs (unified interface), so the
|
||||
datapath is a runtime decision — links attach to running containers
|
||||
too: the compute side enslaves the veth host end (brctl) or attaches
|
||||
the relay to it (add_nio_ethernet) without touching the container's
|
||||
interfaces.
|
||||
Every adapter owns an anchor, so the datapath is a runtime decision —
|
||||
links attach to running nodes too: the compute side enslaves the
|
||||
anchor (brctl) or attaches the relay to it (add_nio_ethernet) without
|
||||
touching the interfaces the node itself uses. Both endpoints must be
|
||||
on the same compute (there is no kernel link across hosts) and both
|
||||
must be able to anchor: Docker adapters are born as veth pairs,
|
||||
QEMU adapters as persistent TAPs, which need uBridge's tap module —
|
||||
asked of that compute, since an old uBridge leaves QEMU on the relay
|
||||
datapath and cannot carry a kernel link at all.
|
||||
"""
|
||||
|
||||
if not Config.instance().settings.Server.enable_kernel_datapath:
|
||||
return False
|
||||
if node1.node_type != "docker" or node2.node_type != "docker":
|
||||
return False
|
||||
if node1.compute.id != node2.compute.id:
|
||||
return False
|
||||
if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2):
|
||||
return False
|
||||
return True
|
||||
return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2)
|
||||
|
||||
@staticmethod
|
||||
def _kernel_endpoint_ready(node):
|
||||
"""
|
||||
Whether a node can anchor a kernel link on its compute.
|
||||
"""
|
||||
|
||||
if node.node_type == "docker":
|
||||
return True
|
||||
if node.node_type != "qemu":
|
||||
return False
|
||||
capabilities = node.compute.capabilities or {}
|
||||
# Strict True: an unreported capability (old uBridge, failed probe)
|
||||
# keeps the link on the relay, where it always works.
|
||||
return capabilities.get("ubridge_tap") is True
|
||||
|
||||
async def _prepare(self):
|
||||
"""
|
||||
|
||||
@@ -185,8 +185,9 @@ class ServerSettings(BaseModel):
|
||||
allow_raw_images: bool = Field(True, description="Allow raw images to be uploaded to the server")
|
||||
enable_kernel_datapath: bool = Field(
|
||||
True,
|
||||
description="Wire eligible Docker-to-Docker links (same compute, no filters/markers) through "
|
||||
"kernel veth/bridge interfaces instead of the uBridge UDP relay",
|
||||
description="Wire eligible links (both endpoints on one compute, both able to anchor: Docker veth, "
|
||||
"or QEMU when the compute's uBridge supports persistent TAPs) through kernel interfaces instead "
|
||||
"of the uBridge UDP relay",
|
||||
)
|
||||
auto_discover_images: bool = Field(True, description="Automatically discover images in the images directory")
|
||||
image_sync_interval: int = Field(900, ge=10, description="Seconds between automatic image inventory scans")
|
||||
|
||||
@@ -50,3 +50,8 @@ class Capabilities(BaseModel):
|
||||
ubridge_tc: Optional[UbridgeTcCapabilities] = Field(
|
||||
None, description="uBridge tc-module capabilities; absent when the probe failed or uBridge has no tc module"
|
||||
)
|
||||
ubridge_tap: Optional[bool] = Field(
|
||||
None,
|
||||
description="Whether uBridge can create the persistent TAPs a QEMU adapter anchors on "
|
||||
"(needed to wire QEMU links on the kernel datapath); absent when the probe failed",
|
||||
)
|
||||
@@ -83,8 +83,9 @@ class Link(LinkBase):
|
||||
link_type: Optional[LinkType] = None
|
||||
kernel_datapath: Optional[bool] = Field(
|
||||
None,
|
||||
description="Read only property. True when this link is wired through kernel veth/bridge interfaces "
|
||||
"(currently Docker-to-Docker on one compute) instead of the uBridge UDP relay",
|
||||
description="Read only property. True when this link is wired through kernel interfaces "
|
||||
"(veth or TAP anchors enslaved to a per-link Linux bridge, both endpoints on one compute) "
|
||||
"instead of the uBridge UDP relay",
|
||||
)
|
||||
capturing: Optional[bool] = Field(None, description="Read only property. True if a capture running on the link")
|
||||
capture_file_name: Optional[str] = Field(
|
||||
|
||||
Reference in new issue
Block a user