feat: cross-compute VXLAN kernel datapath

Cross-compute links can ride kernel VXLAN tunnels instead of the uBridge
UDP relay: each endpoint creates one VXLAN device (uBridge's rtnetlink
vxlan module) enslaved into the same per-link Linux bridge the same-host
datapath uses, so filters, markers, capture and suspend keep running on
the anchor unchanged. Opt-in (Server.enable_vxlan_cross_compute, default
off) because the tunnel caps the usable MTU at the underlay minus the
outer headers; the relay stays the default and the fallback.

The controller derives what the pair must agree on: one VNI per link
(upper VNI space, per-link random salt) and deterministic device names
(gvl{link}{node}) that make crashed-run leftovers claimable
(show -> delete -> create). Same-host compute pairs, switch endpoints
and serial/unix-socket containers stay on the relay.

A VNI the kernel refuses self-heals on every path it can surface:
interactive create retries a fresh salt then degrades to the relay;
project open re-dispatches refused batch entries the same way; node
start recognises the refused device, stops the half-started node,
re-wires both ends on a fresh VNI (or degrades the link) and retries
the start. The device MTU is derived from the lower device (dev=) and
the per-link bridge inherits it - no configuration. The capability
probe (ubridge_vxlan in /capabilities) runs one scratch create/delete
on a scratch port that never equals the configured dstport.

The e2e scenario runs two computes on one machine (a network namespace
joined by a veth pair as the fabric) and needs root.
This commit is contained in:
YueGuobin committed 2026-10-17 14:11:00 +08:00
1 parent 5514b2e929
commit 21b866aeed
24 files changed
+2107 -59

No files matched your search

@@ -30,6 +30,7 @@ from gns3server.compute.ubridge.tc_probe import (
probe_iol_tap_support,
probe_tap_support,
probe_tc_capabilities,
probe_vxlan_support,
)
from gns3server.compute.vpcs.vpcs_probe import probe_vpcs_tap
from gns3server.utils.path import get_default_project_directory
@@ -75,6 +76,11 @@ async def get_capabilities(request: Request) -> dict:
# kernel link. None = unknown (probe failed or binary missing).
vpcs_tap = await probe_vpcs_tap()
# VXLAN device creation (the vxlan module): the tunnel a cross-compute
# kernel link rides. None = unknown (probe failed, uBridge too old, or
# the kernel refuses vxlan devices here).
ubridge_vxlan = await probe_vxlan_support()
# record the controller hostname or IP address
if request.client:
request.app.state.controller_host = request.client.host
@@ -91,4 +97,5 @@ async def get_capabilities(request: Request) -> dict:
"ubridge_iol_tap": ubridge_iol_tap,
"ubridge_bridge_tap": ubridge_bridge_tap,
"vpcs_tap": vpcs_tap,
"ubridge_vxlan": ubridge_vxlan,
}
+27 -7
View File
@@ -35,6 +35,7 @@ from fastapi.encoders import jsonable_encoder
from fastapi.responses import FileResponse
from gns3server import schemas
from gns3server.compute.error import NodeError
from gns3server.compute.project import Project
from gns3server.compute.project_manager import ProjectManager
from gns3server.utils.path import is_safe_path
@@ -271,16 +272,35 @@ async def create_batch_nios(
# count (standard == 1, Dynamips == 2) and await the async variant.
sig = inspect.signature(node.manager.create_nio)
dynamips_style = len(sig.parameters) >= 2
failed = []
for entry in entries:
nio_settings = jsonable_encoder(entry.nio, exclude_unset=True)
if dynamips_style:
nio = await node.manager.create_nio(node, nio_settings)
else:
nio = node.manager.create_nio(nio_settings)
await _add_nio_binding(node, entry.adapter_number, entry.port_number, nio)
try:
if dynamips_style:
nio = await node.manager.create_nio(node, nio_settings)
else:
nio = node.manager.create_nio(nio_settings)
await _add_nio_binding(node, entry.adapter_number, entry.port_number, nio)
except NodeError:
# A VXLAN entry that cannot be wired (its VNI already
# registered on the host's shared socket, no route to the
# peer) must not fail the whole project open: report it so
# the controller can degrade that link to the uBridge relay.
# Every other NIO type keeps the fail-the-batch semantics.
if not nio_settings.get("vxlan"):
raise
failed.append(
{
"node_id": entry.node_id,
"adapter_number": entry.adapter_number,
"port_number": entry.port_number,
}
)
return failed
await asyncio.gather(*[_create_one_node(nid, ents) for nid, ents in per_node.items()])
return {"added": len(batch.nios)}
results = await asyncio.gather(*[_create_one_node(nid, ents) for nid, ents in per_node.items()])
failed = [item for node_failed in results for item in node_failed]
return {"added": len(batch.nios) - len(failed), "failed": failed}
@router.put(
+4 -1
View File
@@ -375,7 +375,10 @@ class BaseManager:
nio.markers = nio_settings.get("markers", {})
nio.suspend = nio_settings.get("suspend", False)
elif nio_settings["type"] == "nio_bridge":
nio = NIOBridge(nio_settings["bridge"])
# A vxlan sub-spec marks a cross-compute link: the node creates
# the VXLAN device and enslaves it into the same per-link bridge
# next to its anchor (see KernelDatapathMixin._kernel_attach).
nio = NIOBridge(nio_settings["bridge"], nio_settings.get("vxlan"))
nio.filters = nio_settings.get("filters", {})
# Markers ride the NIO and attach to the veth host interface via
# uBridge's AF_PACKET marker module (marker add_kernel).
+3 -1
View File
@@ -369,7 +369,9 @@ class Dynamips(BaseManager):
# Kernel-datapath NIO: the router binds the port's anchor TAP to
# a hypervisor nio_tap and enslaves it into this per-link bridge
# (Router._attach_kernel_nio). No hypervisor NIO is created here.
nio = NIOBridge(nio_settings["bridge"])
# A vxlan sub-spec (cross-compute link) rides along untouched —
# the router's attach path wires it like any kernel link.
nio = NIOBridge(nio_settings["bridge"], nio_settings.get("vxlan"))
nio.filters = nio_settings.get("filters", {})
nio.markers = nio_settings.get("markers", {})
nio.suspend = nio_settings.get("suspend", False)
+115
View File
@@ -48,6 +48,9 @@ import asyncio
import contextlib
import logging
import os
import socket
import psutil
from gns3server.compute.error import NodeError
from gns3server.compute.ubridge.ubridge_error import UbridgeError
@@ -264,6 +267,86 @@ class KernelDatapathMixin:
return not (isinstance(nio, NIOBridge) and nio.bridge is None)
async def _ensure_vxlan_device(self, spec):
"""
Create (or replace) the cross-compute VXLAN device this link end
tunnels through and return the device name. The device name is a
pure function of the link id, so a device found under it belongs to
this link: either a leftover from a run that crashed without
cleanup, or the previous instance of a node restart — deleted and
re-created either way, which makes wiring idempotent. The lower
device is resolved locally from the peer address; the kernel
derives the tunnel MTU from it, which is what keeps the per-link
bridge honest about what the underlay can carry.
A create failure with the name just cleared can only mean the VNI —
the kernel refuses a VNI already registered on the shared dstport
socket (EEXIST) — so it is surfaced as a wiring error and the
controller retries the link with a fresh VNI.
"""
name = spec["name"]
remote = spec["remote"]
try:
await self._ubridge_send(f'vxlan show "{name}"')
except UbridgeError:
# Not present (or uBridge cannot answer): the create below is
# the real gate either way.
pass
else:
await self._ubridge_send(f'vxlan delete "{name}"')
dev = await self._vxlan_lowerdev(remote)
try:
await self._ubridge_send(
f'vxlan create "{name}" {spec["vni"]} remote={remote} dstport={spec["dstport"]} dev={dev}'
)
except UbridgeError as e:
raise self._kernel_error(f"Could not create VXLAN device {name} to {remote} via {dev}: {e}")
return name
async def _vxlan_lowerdev(self, remote):
"""
The local interface the route to *remote* egresses through — the
``dev=`` a cross-compute VXLAN device needs so the kernel can derive
its MTU from the underlay (peer MTU - 50) and pick the outer source
address. A connected UDP socket makes the kernel pick the route
without sending anything; the chosen source address is matched back
to its interface.
"""
try:
info = socket.getaddrinfo(remote, 9, socket.AF_UNSPEC, socket.SOCK_DGRAM)
except socket.gaierror as e:
raise self._kernel_error(f"Cannot resolve VXLAN peer {remote}: {e}")
last_error = None
for af, socktype, proto, _, sa in info:
try:
with socket.socket(af, socktype, proto) as sock:
sock.connect(sa)
local_ip = sock.getsockname()[0]
except OSError as e:
last_error = e
continue
dev = self._interface_with_address(local_ip)
if dev is not None:
return dev
raise self._kernel_error(f"No local interface owns {local_ip}, the route to VXLAN peer {remote}")
raise self._kernel_error(f"No route to VXLAN peer {remote}: {last_error}")
@staticmethod
def _interface_with_address(address):
"""
The name of the interface owning *address* (scope suffix stripped
for link-local IPv6), or None.
"""
plain = address.split("%")[0]
for name, addrs in psutil.net_if_addrs().items():
for addr in addrs:
if addr.address.split("%")[0] == plain:
return name
return None
async def _kernel_attach(self, anchor, nio):
"""
Attach an anchor to the per-link kernel bridge carried by *nio*
@@ -276,6 +359,12 @@ class KernelDatapathMixin:
bridge first: an existing bridge is success instead of a create race
failure.
A NIO carrying a ``vxlan`` spec is a cross-compute link: this end's
VXLAN device is (re-)created first and enslaved into the same
bridge next to the anchor, so the kernel encapsulates/decapsulates
the frames while filters, markers, capture and carrier keep running
on the anchor exactly like a same-host kernel link.
A NIO whose ``bridge`` is None means the anchor is bridged
elsewhere — an Ethernet switch absorbed it into the switch's own
bridge — so the membership work is skipped here and only the link
@@ -287,6 +376,13 @@ class KernelDatapathMixin:
"""
if nio.bridge is not None:
vxlan_name = None
if nio.vxlan:
# Replace/refresh the device before the bridge work: a
# leftover device may still be enslaved to a leftover bridge
# (its deletion un-enrolls it), and the fresh device must be
# in place when the bridge's MTU auto-adjust runs.
vxlan_name = await self._ensure_vxlan_device(nio.vxlan)
try:
await self._ubridge_send(f'brctl create "{nio.bridge}"')
except UbridgeError:
@@ -294,6 +390,8 @@ class KernelDatapathMixin:
await self._ubridge_send(f'brctl show "{nio.bridge}"')
await self._ubridge_send(f'link set "{nio.bridge}" up')
await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"')
if vxlan_name is not None:
await self._ubridge_send(f'brctl addif "{nio.bridge}" "{vxlan_name}"')
if nio.capturing:
# Restore a capture that was active before a node restart
# (mirrors the relay path's start_capture).
@@ -373,6 +471,11 @@ class KernelDatapathMixin:
enslaved fails with EBUSY, so the last endpoint to remove its port
wins the deletion and the loser's failure is expected and suppressed.
A NIO carrying a ``vxlan`` spec is a cross-compute link: this end
also deletes its own tunnel device (the peer deletes its own on its
compute); the kernel's unregister path releases the bridge port, so
it goes before the bridge delete, same as the anchor.
A NIO whose ``bridge`` is None owns no bridge here: the anchor is
bridged by an Ethernet switch, which tears its own membership down
on its side; this end only releases the link state it applied
@@ -394,6 +497,9 @@ class KernelDatapathMixin:
with contextlib.suppress(UbridgeError):
await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"')
if nio.bridge is not None:
if nio.vxlan:
with contextlib.suppress(UbridgeError):
await self._ubridge_send(f'vxlan delete "{nio.vxlan["name"]}"')
with contextlib.suppress(UbridgeError):
await self._ubridge_send(f'brctl delete "{nio.bridge}"')
@@ -407,6 +513,12 @@ class KernelDatapathMixin:
the same contract as link deletion. A node restart re-creates the
bridge from the NIO in the attach path, so nothing is lost by
deleting it.
A cross-compute NIO's tunnel device is swept with its bridge: each
end owns its own device, and a device left behind would hold its
VNI on the host's shared socket until a reopen's attach path
replaces it by name (which it does — the sweep just frees it
sooner).
"""
if not self.ubridge:
@@ -416,6 +528,9 @@ class KernelDatapathMixin:
# bridge None = externally bridged (an Ethernet switch owns
# the bridge); never delete a bridge this node does not own.
if isinstance(nio, NIOBridge) and nio.bridge is not None:
if nio.vxlan:
with contextlib.suppress(UbridgeError):
await self._ubridge_send(f'vxlan delete "{nio.vxlan["name"]}"')
with contextlib.suppress(UbridgeError):
await self._ubridge_send(f'brctl delete "{nio.bridge}"')
+24 -1
View File
@@ -31,6 +31,13 @@ veth host end (marker add_kernel / capture start_kernel).
Ethernet switch absorbed it into the switch's own kernel bridge — and this
node applies only the link state the anchor carries (carrier, markers,
impairments), never bridge membership.
A NIO may additionally carry ``vxlan`` (a BridgeVxlan spec): the link
crosses computes through a VXLAN device this end creates (uBridge's
``vxlan`` module) and enslaves into the same per-link bridge next to the
anchor. The kernel encapsulates/decapsulates the frames; everything else —
filters, markers, capture, carrier, suspend — still runs on the anchor, so
the datapath behaves exactly like a same-host kernel link.
"""
from .nio import NIO
@@ -42,12 +49,16 @@ class NIOBridge(NIO):
:param bridge: name of the per-link Linux kernel bridge, or None when
the anchor is bridged by an Ethernet switch (see module docstring)
:param vxlan: cross-compute tunnel spec (dict: name/vni/remote/dstport)
this end creates and enslaves into the bridge, or None for a
same-host kernel link
"""
def __init__(self, bridge):
def __init__(self, bridge, vxlan=None):
super().__init__()
self._bridge = bridge
self._vxlan = vxlan
@property
def bridge(self):
@@ -60,6 +71,17 @@ class NIOBridge(NIO):
return self._bridge
@property
def vxlan(self):
"""
Returns the cross-compute VXLAN tunnel spec, or None when the link
is wired on this host alone.
:returns: dict (name/vni/remote/dstport) or None
"""
return self._vxlan
def __str__(self):
return "NIO bridge"
@@ -69,6 +91,7 @@ class NIOBridge(NIO):
return {
"type": "nio_bridge",
"bridge": self._bridge,
"vxlan": self._vxlan,
"suspend": self._suspended,
"filters": self._filters,
"markers": self._markers,
+93
View File
@@ -43,6 +43,7 @@ from typing import Optional
from gns3server.config import Config
from gns3server.utils.tc_capabilities import parse_tc_capabilities
from gns3server.utils.vxlan import VXLAN_VNI_MAX, VXLAN_VNI_MIN
from .hypervisor import Hypervisor
from .ubridge_error import UbridgeError
@@ -59,6 +60,8 @@ _tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
_iol_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
# binary identity -> whether the bridge module can release a named TAP NIO (None = unknown)
_bridge_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
# binary identity -> whether the vxlan module can create a device (None = unknown)
_vxlan_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
_lock = asyncio.Lock()
@@ -358,3 +361,93 @@ async def _probe_bridge_tap(path, config, timeout):
with contextlib.suppress(Exception):
await hypervisor.stop()
shutil.rmtree(working_dir, ignore_errors=True)
async def probe_vxlan_support(timeout: float = 15.0):
"""
Whether this host's uBridge can create a VXLAN device (the ``vxlan``
module) — the carrier a cross-compute kernel link tunnels through.
Probed on a scratch device with a scratch VNI on a scratch dstport
(never the deployment's configured port, so the probe cannot disturb
live tunnels; a device on the port would still be harmless — the kernel
shares one socket — but the probe has no business joining it) and
deleted again, cached by binary identity like the other probes. Any
failure — old build, missing module, no CAP_NET_ADMIN, an unloadable
kernel module — means unknown, reported as None and cached as such;
the controller reads unknown as "keep this link on the relay".
"""
config = Config.instance()
path = shutil.which(config.settings.Server.ubridge_path)
if not path:
return None
try:
stat = os.stat(path)
key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns)
except OSError:
return None
async with _lock:
if key in _vxlan_cache:
return _vxlan_cache[key]
supported = await _probe_vxlan(path, config, timeout)
_vxlan_cache[key] = supported
return supported
async def _probe_vxlan(path, config, timeout):
"""
Spawn a throwaway uBridge and run one ``vxlan create``/``delete`` cycle.
The scratch name (uuid suffix) cannot collide with a real link's
deterministic name; the scratch VNI and dstport are drawn from uuids so
two concurrent probes on one host (two servers, one uBridge user) cannot
fight over either. No ``dev=``: this probe answers "can this binary and
kernel create a VXLAN device at all" — resolving the real underlay
interface belongs to the link attach path, where an unroutable peer
means this particular link falls back to the relay, not that the
capability is absent.
"""
working_dir = tempfile.mkdtemp(prefix="gns3-vxlan-probe-")
hypervisor = Hypervisor(
None,
path,
working_dir,
config.settings.Server.ubridge_control_transport,
config.settings.Server.host,
str(uuid.uuid4()),
)
name = f"gvx{uuid.uuid4().hex[:8]}"
# VNI in the deployment's own derived range (utils.vxlan) so the probe
# exercises exactly the kernel path real links use.
vni = VXLAN_VNI_MIN + uuid.uuid4().int % (VXLAN_VNI_MAX - VXLAN_VNI_MIN + 1)
dstport = 20000 + uuid.uuid4().int % 10000
# The scratch socket must never be the configured dstport: on it the
# scratch VNI shares a demux table with the live links' devices, and an
# unlucky draw would make the probe report the capability unknown for
# the process's lifetime (the result is cached).
if dstport == config.settings.Server.vxlan_dstport:
dstport = 20000 if dstport == 29999 else dstport + 1
try:
async def ask():
await hypervisor.start()
await hypervisor.connect()
await hypervisor.send(f"vxlan create {name} {vni} remote=127.0.0.1 dstport={dstport}")
try:
await hypervisor.send(f"vxlan delete {name}")
except UbridgeError as e:
log.warning("Probe VXLAN %s could not be deleted: %s", name, e)
return True
try:
return await asyncio.wait_for(ask(), timeout=timeout)
except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e:
# Old build ("Unknown command"), missing module, or no
# CAP_NET_ADMIN: unknown.
log.debug("uBridge vxlan-module probe failed: %s", e)
return None
finally:
with contextlib.suppress(Exception):
await hypervisor.stop()
shutil.rmtree(working_dir, ignore_errors=True)
+76 -11
View File
@@ -20,18 +20,33 @@ import copy
import html
import logging
import os
import re
import uuid
from ..config import Config
from ..utils import int_to_macaddress, macaddress_to_int
from ..utils.application_id import is_iol_runner_environment
from ..utils.images import images_directories
from ..utils.vxlan import vxlan_device_name
from .controller_error import ComputeConflictError, ComputeError, ControllerError, ControllerTimeoutError
from .node_types import BUILTIN_NODE_TYPES
from .ports.port_factory import DynamipsPortFactory, PortFactory, StandardPortFactory
log = logging.getLogger(__name__)
# The compute's error when it could not create a VXLAN tunnel device — the
# wording is a same-repo contract with KernelDatapathMixin._kernel_error
# (only computes running this codebase carry VXLAN links: the capability
# gate keeps everything else on the relay), and the captured device name
# identifies the link (a pure function of link id + node id).
_VXLAN_CONFLICT_RE = re.compile(r"Could not create VXLAN device (?P<device>\S+)")
# Node-start self-heal rounds over a refused VXLAN VNI: the first re-wires
# the link on a fresh VNI, the second degrades it to the relay (whose NIOs
# cannot hit the conflict) — a start must not fail permanently because
# another link already registered the VNI on the shared dstport socket.
_VXLAN_START_HEALS = 2
# Image properties of a node, mapped by node type. The value is the image
# "type" used by the image manager (see utils.images). Docker images are not
@@ -955,18 +970,32 @@ class Node:
f"Cannot start node '{self._name}': {len(failed_links)} deferred link(s) "
"could not be restored. Please try again."
)
try:
# For IOU: we need to send the licence everytime we start a node
if self.node_type == "iou":
license_check = self._project.controller.iou_license.get("license_check", True)
iourc_content = self._project.controller.iou_license.get("iourc_content", None)
await self.post(
"/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
for heal in range(_VXLAN_START_HEALS + 1):
try:
await self._start_on_compute(data)
break
except asyncio.TimeoutError:
raise ControllerTimeoutError(f"Timeout when starting {self._name}")
except ComputeError as e:
conflicted = self._vxlan_conflicted_link(e)
if conflicted is None or heal == _VXLAN_START_HEALS:
raise
degrade = heal > 0 # a second conflict takes the relay
log.warning(
"Node '%s': start refused over a VXLAN VNI conflict on link %s — %s and retrying",
self._name,
conflicted.id,
"degrading the link to the uBridge relay" if degrade else "re-wiring the link on a fresh VNI",
)
else:
await self.post("/start", data=data, timeout=240)
except asyncio.TimeoutError:
raise ControllerTimeoutError(f"Timeout when starting {self._name}")
# The refused start leaves the node half-started (the
# emulator process runs, the wiring partial) — stop it
# first: a start on a running container early-returns
# without wiring anything.
await self.stop()
if await self._project._redispatch_vxlan_links([conflicted], downgrade=degrade) and not degrade:
# The fresh VNI was already refused before the start
# could even retry — degrade the pair to the relay now.
await self._project._redispatch_vxlan_links([conflicted], downgrade=True)
self._confirm_status("started")
# A started node may have brought kernel anchors into existence that
# a link needs to finish wiring (the Ethernet-switch fast path joins
@@ -974,6 +1003,42 @@ class Node:
for link in self._links:
await link.node_started(self)
async def _start_on_compute(self, data=None):
"""
The compute-side start request (IOU carries its licence along).
"""
# For IOU: we need to send the licence everytime we start a node
if self.node_type == "iou":
license_check = self._project.controller.iou_license.get("license_check", True)
iourc_content = self._project.controller.iou_license.get("iourc_content", None)
await self.post(
"/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
)
else:
await self.post("/start", data=data, timeout=240)
def _vxlan_conflicted_link(self, error):
"""
The link whose VXLAN device a failed node start could not create,
or None for every other failure.
The compute's error names the device it could not create and the
name is a pure function of (link id, node id), so it identifies the
link with no extra API surface. An unmatched message — another
failure, a future wording change — simply leaves the error to
surface unchanged, which is exactly the behaviour without the heal.
"""
match = _VXLAN_CONFLICT_RE.search(str(error))
if match is None:
return None
device = match.group("device")
for link in self._links:
if vxlan_device_name(link.id, self.id) == device and any(nio.get("vxlan") for nio in link._link_data):
return link
return None
async def stop(self, strict=False):
"""
Stop a node
+102 -9
View File
@@ -49,7 +49,7 @@ from .link import _UNSET
from .node import Node
from .snapshot import Snapshot
from .topology import load_topology, project_to_topology
from .udp_link import UDPLink
from .udp_link import _VXLAN_ATTEMPTS, UDPLink
log = logging.getLogger(__name__)
@@ -1021,6 +1021,84 @@ class Project:
entries = await link._prepare()
return (link, entries)
async def _dispatch_nio_batch(self, compute, nio_entries):
"""
POST one compute's NIO batch for this project, returning the entries
it refused (VXLAN entries a compute could not wire — see the batch
route; other NIO types fail the whole request).
"""
response = await compute.post(
f"/projects/{self._id}/nios/batch",
data={"nios": nio_entries},
timeout=300,
)
return response.json.get("failed") or []
async def _redispatch_vxlan_links(self, links, downgrade):
"""
Re-prepare *links* and dispatch their NIOs in a second per-compute
batch, returning the links the computes still refused.
With ``downgrade=False`` each link's VNI salt is bumped first — a
fresh VNI, since the refused one was already registered on the
host's shared dstport socket. With ``downgrade=True`` the links are
marked degraded to the uBridge relay (whose UDP NIOs cannot fail
this way) before re-preparing.
"""
for link in links:
if downgrade:
log.warning(
"Project '%s' [%s]: VXLAN wiring failed for link %s — falling back to the uBridge relay datapath",
self._name,
self._id,
link.id,
)
link._vxlan_downgraded = True
else:
link._vxlan_salt += 1
per_compute = {}
by_key = {}
for link in links:
for node, adapter_number, port_number, nio_data in await link._prepare():
by_key[(str(node.id), adapter_number, port_number)] = link
per_compute.setdefault(node.compute, []).append(
{
"node_id": node.id,
"adapter_number": adapter_number,
"port_number": port_number,
"nio": nio_data,
}
)
if not per_compute:
return []
results = await asyncio.gather(
*[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
)
failed = []
for item in (i for sub in results for i in sub):
link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
if link is not None and link not in failed:
failed.append(link)
return failed
async def _heal_refused_vxlan_links(self, pending):
"""
Drive the links a compute refused at open onto a working datapath:
fresh-VNI rounds first — the same candidate count an interactive
link creation tries — then, for whatever still fails, the
degradation to the uBridge relay (whose UDP NIOs cannot hit this
refusal).
"""
for _ in range(_VXLAN_ATTEMPTS - 1):
if not pending:
return
pending = await self._redispatch_vxlan_links(pending, downgrade=False)
if pending:
await self._redispatch_vxlan_links(pending, downgrade=True)
async def restore_deferred_links(self, node):
"""
Create on the computes the NIOs of the links that were deferred while
@@ -1994,15 +2072,30 @@ class Project:
}
)
async def _dispatch_batch(compute, nio_entries):
await compute.post(
f"/projects/{self._id}/nios/batch",
data={"nios": nio_entries},
timeout=300,
)
if per_compute:
await asyncio.gather(*[_dispatch_batch(c, n) for c, n in per_compute.items()])
results = await asyncio.gather(
*[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
)
failed = [item for sub in results for item in sub]
else:
failed = []
# A VXLAN entry a compute refused (its VNI already registered on
# the host's shared socket, no route to the peer) must not fail
# the whole open: the refused links are healed onto a working
# datapath (fresh VNIs, then the relay) — see
# _heal_refused_vxlan_links.
if failed:
by_key = {}
for link, entries in valid:
for node, adapter_number, port_number, nio_data in entries:
if nio_data.get("vxlan"):
by_key[(str(node.id), adapter_number, port_number)] = link
pending = []
for item in failed:
link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
if link is not None and link not in pending:
pending.append(link)
await self._heal_refused_vxlan_links(pending)
# Finalise every link: wire node/port back-references, mark created,
# notify clients, and apply project-level marker definitions.
+193 -12
View File
@@ -18,6 +18,7 @@
import asyncio
import logging
import uuid
from gns3server.config import Config
from gns3server.utils.application_id import is_iol_runner_environment
@@ -26,6 +27,7 @@ from gns3server.utils.packet_filter_validation import (
split_kernel_only_features,
validate_bpf_syntax,
)
from gns3server.utils.vxlan import derive_vni, vxlan_device_name
from .compute import ComputeError
from .controller_error import ControllerError, ControllerNotFoundError
@@ -52,6 +54,10 @@ _MARKER_CAPABLE_TYPES = frozenset(
log = logging.getLogger(__name__)
# VXLAN wiring attempts per link creation (the first plus the salt-bumped
# retries) before the link degrades to the uBridge relay.
_VXLAN_ATTEMPTS = 3
def _is_unix_socket_docker(node):
"""
@@ -92,6 +98,15 @@ class UDPLink(Link):
# of its endpoints has a missing image. The NIO creation is retried
# once the image is resolved.
self._deferred = False
# Cross-compute VXLAN wiring state: the salt is this link object's
# starting point in the VNI space — random per object, so a project
# reopen re-rolls every link's VNI and a link whose VNI collided
# with an unrelated device on a host is not permanently unlucky
# (the device *name*, which cleanup keys on, stays deterministic).
# _commit_vxlan bumps it for an in-flight retry; the downgrade flag
# marks a link degraded to the UDP relay for this object's lifetime.
self._vxlan_salt = uuid.uuid4().int
self._vxlan_downgraded = False
@property
def debug_link_data(self):
@@ -206,9 +221,13 @@ class UDPLink(Link):
Every adapter owns an anchor, so the datapath is a runtime decision —
links attach to running nodes too: the compute side enslaves the
anchor (brctl) or attaches the relay to it (add_nio_ethernet) without
touching the interfaces the node itself uses. Both endpoints must be
on the same compute (there is no kernel link across hosts) and both
must be able to anchor: Docker adapters are born as veth pairs, and
touching the interfaces the node itself uses. Both endpoints must sit
on computes that can carry their end: on one compute the ends meet in
a per-link Linux bridge; across computes each end additionally rides
its own VXLAN device (opt-in via enable_vxlan_cross_compute, uBridge's
vxlan module) enslaved into the same per-link bridge, so everything
below the bridge — filters, markers, capture, carrier — is identical
either way. Both must be able to anchor: Docker adapters are born as veth pairs, and
IOL runner containers — whose guest leg is unix sockets, not a veth —
anchor on persistent TAPs reached through the bridge module's
swappable TAP leg (its own capability pair, see
@@ -232,8 +251,21 @@ class UDPLink(Link):
if not Config.instance().settings.Server.enable_kernel_datapath:
return False
if node1.compute.id != node2.compute.id:
return False
cross_compute = node1.compute.id != node2.compute.id
if cross_compute:
# Cross-compute kernel links tunnel: each endpoint rides a VXLAN
# device into the same per-link bridge the same-host datapath
# uses. Opt-in (the tunnel caps the usable MTU at the underlay
# minus the outer headers, so it pays off on jumbo fabrics) and
# gated on both computes' uBridge being able to create the
# device; a link that cannot tunnel stays on the relay, where it
# always works. The downgrade flag is the runtime fallback: after
# the kernel kept refusing this link's VNI, the link stays on the
# relay for this object's lifetime.
if not Config.instance().settings.Server.enable_vxlan_cross_compute:
return False
if self._vxlan_downgraded:
return False
if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2):
return False
for side in self._nodes:
@@ -249,10 +281,17 @@ class UDPLink(Link):
# pair is the only shape a bridge-to-bridge cable can take).
switch = self._switch_endpoint()
if switch is not None:
if cross_compute:
# The fast path is a one-host shape (the peer's anchor is
# absorbed into the switch's bridge); a switch link across
# computes stays on the relay.
return False
if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
return True
peer = node2 if switch[0] == 0 else node1
return self._kernel_endpoint_ready(peer)
if cross_compute:
return self._vxlan_endpoint_ready(node1) and self._vxlan_endpoint_ready(node2)
return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2)
def _switch_endpoint(self):
@@ -305,6 +344,23 @@ class UDPLink(Link):
return capabilities.get("ubridge_tap") is True and capabilities.get("vpcs_tap") is True
return False
@staticmethod
def _vxlan_endpoint_ready(node):
"""
Whether a node can carry its end of a cross-compute VXLAN link: it
must be able to anchor (the same per-node-type capability the
same-host kernel datapath needs — its end of the tunnel is enslaved
into a per-link bridge like any kernel link's) and its compute's
uBridge must be able to create the VXLAN device the tunnel rides.
Strict True on both, like the other capability gates: an unreported
or failed probe keeps the link on the relay, where it always works.
"""
if not UDPLink._kernel_endpoint_ready(node):
return False
capabilities = node.compute.capabilities or {}
return capabilities.get("ubridge_vxlan") is True
async def _prepare(self):
"""
Local-only link setup: resolve peer addresses, reserve UDP ports and
@@ -330,10 +386,77 @@ class UDPLink(Link):
adapter_number2 = self._nodes[1]["adapter_number"]
port_number2 = self._nodes[1]["port_number"]
if self._kernel_datapath_eligible(node1, node2):
cross_compute = node1.compute.id != node2.compute.id
# A cross-compute link needs the compute pair's addresses on both
# datapaths (the VXLAN tunnel's outer endpoints, the relay's UDP
# peers): resolve once, and let the no-common-subnet failure stay a
# ControllerError either way — it is a deployment error, not a
# datapath decision. Equal addresses mean both computes share one
# host: the kernel shares one VXLAN socket per dstport, so a tunnel
# pair cannot live there, and such links keep the relay, where the
# two ends are told apart by their UDP ports.
hosts = None
if cross_compute:
try:
hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
except ValueError as e:
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
same_host = hosts is not None and hosts[0] == hosts[1]
if self._kernel_datapath_eligible(node1, node2) and not same_host:
node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True)
node1_markers, node2_markers = self._get_node_markers(node1, node2)
switch = self._switch_endpoint()
if hosts is not None:
# Cross-compute VXLAN: each end gets one VXLAN device
# carrying the link's VNI, enslaved into the per-link kernel
# bridge next to the node's anchor (see _kernel_attach on
# the compute side). The controller derives name and VNI —
# both ends must agree on the VNI and neither can ask the
# other — and the compute resolves the underlay interface
# locally from the peer address, so the kernel derives the
# device MTU (underlay - 50) and the bridge reports the
# honest "what this fabric can carry" value. A VNI the
# kernel refuses as already registered on the shared
# dstport socket surfaces as a commit failure; create()
# retries with a bumped salt and degrades to the relay when
# the retries run out.
node1_host, node2_host = hosts
dstport = Config.instance().settings.Server.vxlan_dstport
vni = derive_vni(self._id, self._vxlan_salt)
bridge_name = "gns3" + self._id.replace("-", "")[:11]
self._link_data = [
{
"type": "nio_bridge",
"bridge": bridge_name,
"vxlan": {
"name": vxlan_device_name(self._id, node1.id),
"vni": vni,
"remote": node2_host,
"dstport": dstport,
},
"filters": node1_filters,
"markers": node1_markers,
"suspend": self._suspended,
},
{
"type": "nio_bridge",
"bridge": bridge_name,
"vxlan": {
"name": vxlan_device_name(self._id, node2.id),
"vni": vni,
"remote": node1_host,
"dstport": dstport,
},
"filters": node2_filters,
"markers": node2_markers,
"suspend": self._suspended,
},
]
return [
(node1, adapter_number1, port_number1, self._link_data[0]),
(node2, adapter_number2, port_number2, self._link_data[1]),
]
if switch is not None:
if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
# Switch-to-switch cascade: one veth pair joins the two
@@ -449,11 +572,16 @@ class UDPLink(Link):
(node2, adapter_number2, port_number2, self._link_data[1]),
]
# Get an IP allowing communication between both host
try:
(node1_host, node2_host) = await node1.compute.get_ip_on_same_subnet(node2.compute)
except ValueError as e:
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
# Relay datapath: UDP tunnels between the computes' addresses —
# already resolved for a cross-compute pair, resolved here otherwise
# (a same-compute relay pair resolves to the host's own address on
# both sides, distinguished by their UDP ports).
if hosts is None:
try:
hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
except ValueError as e:
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
node1_host, node2_host = hosts
# Reserve a UDP port on both sides in parallel. Pre-allocated ports
# (used during batch project loading) are popped from memory; otherwise
@@ -537,17 +665,70 @@ class UDPLink(Link):
raise errors[0]
self._created = True
@staticmethod
def _has_vxlan(entries):
"""
Whether any prepared NIO spec of this link carries a VXLAN tunnel.
"""
return any(nio_data.get("vxlan") for _, _, _, nio_data in entries)
async def create(self):
"""
Create the link on the nodes (interactive path: prepare + commit).
"""
entries = await self._prepare()
await self._commit_nios(entries)
if self._has_vxlan(entries):
await self._commit_vxlan(entries)
else:
await self._commit_nios(entries)
# New links automatically inherit every active project-level marker
# definition so the user doesn't have to reconfigure.
await self._project.apply_defs_to_new_link(self)
async def _commit_vxlan(self, entries):
"""
Commit a cross-compute VXLAN link, retrying with a bumped VNI salt
on failure and degrading the link to the UDP relay when the retries
run out.
The kernel refuses a VNI already registered on the shared dstport
socket at device creation (EEXIST) — the one conflict a freshly
created, deterministically named device can hit: the other end's
device lives on another host, and a leftover under this end's name
is replaced by the attach path, not refused. Any other commit
failure (a stopped node, a missing underlay route) is retried too,
harmlessly — it fails the same way every attempt, and the final
relay attempt surfaces the real error if the link genuinely cannot
be created. VXLAN is opt-in and the relay always works, so a tunnel
that cannot be wired degrades instead of failing the link; the
degradation lasts for this link object's lifetime (a project reopen
retries VXLAN from a clean state).
"""
last_error = None
for attempt in range(_VXLAN_ATTEMPTS):
if attempt:
self._vxlan_salt += 1
entries = await self._prepare()
if not self._has_vxlan(entries):
# Eligibility changed under us (a capability disappeared):
# commit the relay entries as prepared.
await self._commit_nios(entries)
return
try:
await self._commit_nios(entries)
return
except (ComputeError, ControllerError) as e:
last_error = e
log.warning(
"Link %s: VXLAN wiring failed (%s) — falling back to the uBridge relay datapath", self._id, last_error
)
self._vxlan_downgraded = True
entries = await self._prepare()
await self._commit_nios(entries)
async def update(self):
"""
Update the link on the nodes
+25
View File
@@ -73,6 +73,26 @@ class BridgeNIOType(str, Enum):
bridge = "nio_bridge"
class BridgeVxlan(BaseModel):
"""
VXLAN tunnel spec carried by a cross-compute BridgeNIO. Each endpoint
creates one VXLAN device (uBridge ``vxlan create``) and enslaves it into
the link's per-link kernel bridge next to the node's anchor; the kernel
encapsulates/decapsulates the frames. The controller derives the name
and VNI (utils.vxlan) and resolves the peer address, so both endpoints
agree on the VNI without ever talking to each other.
The device's MTU is derived by the kernel from the underlay interface
the route to ``remote`` picks (MTU - 50 for the outer headers), which is
what the link's bridge then reports as "what this fabric can carry".
"""
name: str = Field(..., max_length=15, description="VXLAN device name on this endpoint's host (IFNAMSIZ-1)")
vni: int = Field(..., ge=1, le=16777215, description="VXLAN network identifier, identical on both endpoints")
remote: str = Field(..., description="Peer compute's address (unicast tunnel endpoint)")
dstport: int = Field(..., ge=1, le=65535, description="UDP destination port, identical on both endpoints")
class BridgeNIO(BaseModel):
"""
Kernel-datapath bridge Network Input/Output properties. The NIO tells the
@@ -99,6 +119,11 @@ class BridgeNIO(BaseModel):
description="Kernel bridge name e.g. gns3a1b2c3d4e5; None when an Ethernet switch "
"owns the anchor's bridge membership (absorbed link)",
)
vxlan: Optional[BridgeVxlan] = Field(
None,
description="Cross-compute tunnel: the VXLAN device this end creates and enslaves into the bridge "
"next to the anchor; absent for same-host kernel links",
)
suspend: Optional[bool] = Field(None, description="Suspend the NIO")
filters: Optional[dict] = Field(
None,
+18
View File
@@ -192,6 +192,24 @@ class ServerSettings(BaseModel):
"or QEMU when the compute's uBridge supports persistent TAPs) through kernel interfaces instead "
"of the uBridge UDP relay",
)
enable_vxlan_cross_compute: bool = Field(
False,
description="Wire eligible cross-compute links through kernel VXLAN tunnels (one device per endpoint, "
"uBridge's vxlan module) instead of the uBridge UDP relay. The kernel encapsulates/decapsulates, so "
"no user-space forwarding is involved; the price is MTU: a tunnel can only carry the underlay MTU "
"minus 50 bytes of outer headers, so enable this where the fabric between computes carries jumbo "
"frames. The relay stays the default and the fallback when a tunnel cannot be wired",
)
vxlan_dstport: int = Field(
14789,
ge=1,
le=65535,
description="UDP destination port every cross-compute VXLAN tunnel listens on. The kernel shares one "
"socket per port across all VXLAN devices, so all links of a deployment ride this one port — chosen "
"off the IANA 4789 and the legacy 8472 (other VXLAN software's ports: sharing a socket would merge "
"their demux table with ours) and above the ephemeral range. Change it only when it collides with "
"a service already listening on the computes",
)
auto_discover_images: bool = Field(True, description="Automatically discover images in the images directory")
image_sync_interval: int = Field(900, ge=10, description="Seconds between automatic image inventory scans")
report_errors: bool = Field(True, description="Automatically send crash reports to the GNS3 team")
@@ -73,3 +73,8 @@ class Capabilities(BaseModel):
"keeps the guest from auto-configuring the host side of the anchor TAP); absent when the "
"probe failed",
)
ubridge_vxlan: Optional[bool] = Field(
None,
description="Whether uBridge can create VXLAN devices (the vxlan module, kernel support included) — "
"the tunnel a cross-compute kernel link rides; absent when the probe failed",
)
+86
View File
@@ -0,0 +1,86 @@
#
# Copyright (C) 2026 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Identity of a cross-compute VXLAN link: the device name and VNI the
controller derives and ships in both endpoints' NIOs.
A cross-compute kernel link is carried by one VXLAN device per endpoint
(uBridge's ``vxlan`` module, rtnetlink), enslaved into the link's per-link
kernel bridge next to the node's anchor. Each device is local to its own
host, but the VNI is the kernel's demux key for received frames on the
deployment-wide shared dstport socket — both ends must agree on it while
neither can ask the other, so the controller is the single source for both
values.
* the device name is a pure function of the link id plus the endpoint's
node id: unique per (link, endpoint), within IFNAMSIZ, and outside the
``gns3``/anchor name spaces (``gvl`` prefix). Determinism is what makes
a crashed run's leftover device recognisable — the compute deletes a
device it finds under the name before re-creating it;
* the VNI is a pure function of the link id and a controller-side salt —
a random base per link object, bumped when a retry needs a fresh VNI —
drawn from the upper half of the VNI space so it cannot collide with the
low well-known VNIs other software uses (Flannel's 1, Docker overlay
networks from 256, ...).
The kernel refuses a second device registering a VNI already taken on the
same (dstport, socket) pair with EEXIST — a check that doubles as the
collision detector: the controller retries with a bumped salt and falls
back to the UDP relay when the space stays occupied (the relay needs no
VNI at all).
"""
import hashlib
# The VNI range derived links draw from: at or above 0x100000 (1M) to stay
# clear of the low VNIs other VXLAN software commonly registers, at or below
# 0xFFFFFE so every derived value is valid for the module's 1..16777215
# (0xFFFFFF) check with one value of headroom.
VXLAN_VNI_MIN = 0x100000
VXLAN_VNI_MAX = 0xFFFFFE
# IFNAMSIZ (16) - 1 for the terminator: "gvl" (3) + 8 (link) + 4 (node) = 15.
VXLAN_NAME_PREFIX = "gvl"
def derive_vni(link_id, salt=0):
"""
The VNI both endpoints of *link_id* tunnel through.
:param link_id: link UUID (or str)
:param salt: controller-side retry counter — bumped when the kernel
refuses the previous VNI as already registered on the shared
dstport socket; 0 for the first attempt
:returns: VNI in [VXLAN_VNI_MIN, VXLAN_VNI_MAX]
"""
digest = hashlib.sha256(f"{link_id}:{salt}".encode()).digest()
span = VXLAN_VNI_MAX - VXLAN_VNI_MIN + 1
return VXLAN_VNI_MIN + int.from_bytes(digest[:8], "big") % span
def vxlan_device_name(link_id, node_id):
"""
The VXLAN device name one endpoint of *link_id* creates on its own host,
deterministic from the link id plus the endpoint's node id (the pair
normally lives on two hosts, but a forced same-host pair must not name
both its devices identically).
"""
link = str(link_id).replace("-", "")[:8]
node = str(node_id).replace("-", "")[:4]
return f"{VXLAN_NAME_PREFIX}{link}{node}"