feat: cross-compute VXLAN kernel datapath

Cross-compute links can ride kernel VXLAN tunnels instead of the uBridge
UDP relay: each endpoint creates one VXLAN device (uBridge's rtnetlink
vxlan module) enslaved into the same per-link Linux bridge the same-host
datapath uses, so filters, markers, capture and suspend keep running on
the anchor unchanged. Opt-in (Server.enable_vxlan_cross_compute, default
off) because the tunnel caps the usable MTU at the underlay minus the
outer headers; the relay stays the default and the fallback.

The controller derives what the pair must agree on: one VNI per link
(upper VNI space, per-link random salt) and deterministic device names
(gvl{link}{node}) that make crashed-run leftovers claimable
(show -> delete -> create). Same-host compute pairs, switch endpoints
and serial/unix-socket containers stay on the relay.

A VNI the kernel refuses self-heals on every path it can surface:
interactive create retries a fresh salt then degrades to the relay;
project open re-dispatches refused batch entries the same way; node
start recognises the refused device, stops the half-started node,
re-wires both ends on a fresh VNI (or degrades the link) and retries
the start. The device MTU is derived from the lower device (dev=) and
the per-link bridge inherits it - no configuration. The capability
probe (ubridge_vxlan in /capabilities) runs one scratch create/delete
on a scratch port that never equals the configured dstport.

The e2e scenario runs two computes on one machine (a network namespace
joined by a veth pair as the fabric) and needs root.

gns3server/compute/base_manager.py

@@ -375,7 +375,10 @@ class BaseManager:
             nio.markers = nio_settings.get("markers", {})
             nio.suspend = nio_settings.get("suspend", False)
         elif nio_settings["type"] == "nio_bridge":
-            nio = NIOBridge(nio_settings["bridge"])
+            # A vxlan sub-spec marks a cross-compute link: the node creates
+            # the VXLAN device and enslaves it into the same per-link bridge
+            # next to its anchor (see KernelDatapathMixin._kernel_attach).
+            nio = NIOBridge(nio_settings["bridge"], nio_settings.get("vxlan"))
             nio.filters = nio_settings.get("filters", {})
             # Markers ride the NIO and attach to the veth host interface via
             # uBridge's AF_PACKET marker module (marker add_kernel).

gns3server/compute/dynamips/__init__.py

@@ -369,7 +369,9 @@ class Dynamips(BaseManager):
             # Kernel-datapath NIO: the router binds the port's anchor TAP to
             # a hypervisor nio_tap and enslaves it into this per-link bridge
             # (Router._attach_kernel_nio). No hypervisor NIO is created here.
-            nio = NIOBridge(nio_settings["bridge"])
+            # A vxlan sub-spec (cross-compute link) rides along untouched —
+            # the router's attach path wires it like any kernel link.
+            nio = NIOBridge(nio_settings["bridge"], nio_settings.get("vxlan"))
             nio.filters = nio_settings.get("filters", {})
             nio.markers = nio_settings.get("markers", {})
             nio.suspend = nio_settings.get("suspend", False)

gns3server/compute/kernel_datapath.py

@@ -48,6 +48,9 @@ import asyncio
 import contextlib
 import logging
 import os
+import socket
+
+import psutil
 
 from gns3server.compute.error import NodeError
 from gns3server.compute.ubridge.ubridge_error import UbridgeError
@@ -264,6 +267,86 @@ class KernelDatapathMixin:
 
         return not (isinstance(nio, NIOBridge) and nio.bridge is None)
 
+    async def _ensure_vxlan_device(self, spec):
+        """
+        Create (or replace) the cross-compute VXLAN device this link end
+        tunnels through and return the device name. The device name is a
+        pure function of the link id, so a device found under it belongs to
+        this link: either a leftover from a run that crashed without
+        cleanup, or the previous instance of a node restart — deleted and
+        re-created either way, which makes wiring idempotent. The lower
+        device is resolved locally from the peer address; the kernel
+        derives the tunnel MTU from it, which is what keeps the per-link
+        bridge honest about what the underlay can carry.
+
+        A create failure with the name just cleared can only mean the VNI —
+        the kernel refuses a VNI already registered on the shared dstport
+        socket (EEXIST) — so it is surfaced as a wiring error and the
+        controller retries the link with a fresh VNI.
+        """
+
+        name = spec["name"]
+        remote = spec["remote"]
+        try:
+            await self._ubridge_send(f'vxlan show "{name}"')
+        except UbridgeError:
+            # Not present (or uBridge cannot answer): the create below is
+            # the real gate either way.
+            pass
+        else:
+            await self._ubridge_send(f'vxlan delete "{name}"')
+        dev = await self._vxlan_lowerdev(remote)
+        try:
+            await self._ubridge_send(
+                f'vxlan create "{name}" {spec["vni"]} remote={remote} dstport={spec["dstport"]} dev={dev}'
+            )
+        except UbridgeError as e:
+            raise self._kernel_error(f"Could not create VXLAN device {name} to {remote} via {dev}: {e}")
+        return name
+
+    async def _vxlan_lowerdev(self, remote):
+        """
+        The local interface the route to *remote* egresses through — the
+        ``dev=`` a cross-compute VXLAN device needs so the kernel can derive
+        its MTU from the underlay (peer MTU - 50) and pick the outer source
+        address. A connected UDP socket makes the kernel pick the route
+        without sending anything; the chosen source address is matched back
+        to its interface.
+        """
+
+        try:
+            info = socket.getaddrinfo(remote, 9, socket.AF_UNSPEC, socket.SOCK_DGRAM)
+        except socket.gaierror as e:
+            raise self._kernel_error(f"Cannot resolve VXLAN peer {remote}: {e}")
+        last_error = None
+        for af, socktype, proto, _, sa in info:
+            try:
+                with socket.socket(af, socktype, proto) as sock:
+                    sock.connect(sa)
+                    local_ip = sock.getsockname()[0]
+            except OSError as e:
+                last_error = e
+                continue
+            dev = self._interface_with_address(local_ip)
+            if dev is not None:
+                return dev
+            raise self._kernel_error(f"No local interface owns {local_ip}, the route to VXLAN peer {remote}")
+        raise self._kernel_error(f"No route to VXLAN peer {remote}: {last_error}")
+
+    @staticmethod
+    def _interface_with_address(address):
+        """
+        The name of the interface owning *address* (scope suffix stripped
+        for link-local IPv6), or None.
+        """
+
+        plain = address.split("%")[0]
+        for name, addrs in psutil.net_if_addrs().items():
+            for addr in addrs:
+                if addr.address.split("%")[0] == plain:
+                    return name
+        return None
+
     async def _kernel_attach(self, anchor, nio):
         """
         Attach an anchor to the per-link kernel bridge carried by *nio*
@@ -276,6 +359,12 @@ class KernelDatapathMixin:
         bridge first: an existing bridge is success instead of a create race
         failure.
 
+        A NIO carrying a ``vxlan`` spec is a cross-compute link: this end's
+        VXLAN device is (re-)created first and enslaved into the same
+        bridge next to the anchor, so the kernel encapsulates/decapsulates
+        the frames while filters, markers, capture and carrier keep running
+        on the anchor exactly like a same-host kernel link.
+
         A NIO whose ``bridge`` is None means the anchor is bridged
         elsewhere — an Ethernet switch absorbed it into the switch's own
         bridge — so the membership work is skipped here and only the link
@@ -287,6 +376,13 @@ class KernelDatapathMixin:
         """
 
         if nio.bridge is not None:
+            vxlan_name = None
+            if nio.vxlan:
+                # Replace/refresh the device before the bridge work: a
+                # leftover device may still be enslaved to a leftover bridge
+                # (its deletion un-enrolls it), and the fresh device must be
+                # in place when the bridge's MTU auto-adjust runs.
+                vxlan_name = await self._ensure_vxlan_device(nio.vxlan)
             try:
                 await self._ubridge_send(f'brctl create "{nio.bridge}"')
             except UbridgeError:
@@ -294,6 +390,8 @@ class KernelDatapathMixin:
                 await self._ubridge_send(f'brctl show "{nio.bridge}"')
             await self._ubridge_send(f'link set "{nio.bridge}" up')
             await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"')
+            if vxlan_name is not None:
+                await self._ubridge_send(f'brctl addif "{nio.bridge}" "{vxlan_name}"')
         if nio.capturing:
             # Restore a capture that was active before a node restart
             # (mirrors the relay path's start_capture).
@@ -373,6 +471,11 @@ class KernelDatapathMixin:
         enslaved fails with EBUSY, so the last endpoint to remove its port
         wins the deletion and the loser's failure is expected and suppressed.
 
+        A NIO carrying a ``vxlan`` spec is a cross-compute link: this end
+        also deletes its own tunnel device (the peer deletes its own on its
+        compute); the kernel's unregister path releases the bridge port, so
+        it goes before the bridge delete, same as the anchor.
+
         A NIO whose ``bridge`` is None owns no bridge here: the anchor is
         bridged by an Ethernet switch, which tears its own membership down
         on its side; this end only releases the link state it applied
@@ -394,6 +497,9 @@ class KernelDatapathMixin:
                 with contextlib.suppress(UbridgeError):
                     await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"')
         if nio.bridge is not None:
+            if nio.vxlan:
+                with contextlib.suppress(UbridgeError):
+                    await self._ubridge_send(f'vxlan delete "{nio.vxlan["name"]}"')
             with contextlib.suppress(UbridgeError):
                 await self._ubridge_send(f'brctl delete "{nio.bridge}"')
 
@@ -407,6 +513,12 @@ class KernelDatapathMixin:
         the same contract as link deletion. A node restart re-creates the
         bridge from the NIO in the attach path, so nothing is lost by
         deleting it.
+
+        A cross-compute NIO's tunnel device is swept with its bridge: each
+        end owns its own device, and a device left behind would hold its
+        VNI on the host's shared socket until a reopen's attach path
+        replaces it by name (which it does — the sweep just frees it
+        sooner).
         """
 
         if not self.ubridge:
@@ -416,6 +528,9 @@ class KernelDatapathMixin:
                 # bridge None = externally bridged (an Ethernet switch owns
                 # the bridge); never delete a bridge this node does not own.
                 if isinstance(nio, NIOBridge) and nio.bridge is not None:
+                    if nio.vxlan:
+                        with contextlib.suppress(UbridgeError):
+                            await self._ubridge_send(f'vxlan delete "{nio.vxlan["name"]}"')
                     with contextlib.suppress(UbridgeError):
                         await self._ubridge_send(f'brctl delete "{nio.bridge}"')
 

gns3server/compute/nios/nio_bridge.py

@@ -31,6 +31,13 @@ veth host end (marker add_kernel / capture start_kernel).
 Ethernet switch absorbed it into the switch's own kernel bridge — and this
 node applies only the link state the anchor carries (carrier, markers,
 impairments), never bridge membership.
+
+A NIO may additionally carry ``vxlan`` (a BridgeVxlan spec): the link
+crosses computes through a VXLAN device this end creates (uBridge's
+``vxlan`` module) and enslaves into the same per-link bridge next to the
+anchor. The kernel encapsulates/decapsulates the frames; everything else —
+filters, markers, capture, carrier, suspend — still runs on the anchor, so
+the datapath behaves exactly like a same-host kernel link.
 """
 
 from .nio import NIO
@@ -42,12 +49,16 @@ class NIOBridge(NIO):
 
     :param bridge: name of the per-link Linux kernel bridge, or None when
         the anchor is bridged by an Ethernet switch (see module docstring)
+    :param vxlan: cross-compute tunnel spec (dict: name/vni/remote/dstport)
+        this end creates and enslaves into the bridge, or None for a
+        same-host kernel link
     """
 
-    def __init__(self, bridge):
+    def __init__(self, bridge, vxlan=None):
 
         super().__init__()
         self._bridge = bridge
+        self._vxlan = vxlan
 
     @property
     def bridge(self):
@@ -60,6 +71,17 @@ class NIOBridge(NIO):
 
         return self._bridge
 
+    @property
+    def vxlan(self):
+        """
+        Returns the cross-compute VXLAN tunnel spec, or None when the link
+        is wired on this host alone.
+
+        :returns: dict (name/vni/remote/dstport) or None
+        """
+
+        return self._vxlan
+
     def __str__(self):
 
         return "NIO bridge"
@@ -69,6 +91,7 @@ class NIOBridge(NIO):
         return {
             "type": "nio_bridge",
             "bridge": self._bridge,
+            "vxlan": self._vxlan,
             "suspend": self._suspended,
             "filters": self._filters,
             "markers": self._markers,

gns3server/compute/ubridge/tc_probe.py

@@ -43,6 +43,7 @@ from typing import Optional
 
 from gns3server.config import Config
 from gns3server.utils.tc_capabilities import parse_tc_capabilities
+from gns3server.utils.vxlan import VXLAN_VNI_MAX, VXLAN_VNI_MIN
 
 from .hypervisor import Hypervisor
 from .ubridge_error import UbridgeError
@@ -59,6 +60,8 @@ _tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
 _iol_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
 # binary identity -> whether the bridge module can release a named TAP NIO (None = unknown)
 _bridge_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
+# binary identity -> whether the vxlan module can create a device (None = unknown)
+_vxlan_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
 _lock = asyncio.Lock()
 
 
@@ -358,3 +361,93 @@ async def _probe_bridge_tap(path, config, timeout):
         with contextlib.suppress(Exception):
             await hypervisor.stop()
         shutil.rmtree(working_dir, ignore_errors=True)
+
+
+async def probe_vxlan_support(timeout: float = 15.0):
+    """
+    Whether this host's uBridge can create a VXLAN device (the ``vxlan``
+    module) — the carrier a cross-compute kernel link tunnels through.
+    Probed on a scratch device with a scratch VNI on a scratch dstport
+    (never the deployment's configured port, so the probe cannot disturb
+    live tunnels; a device on the port would still be harmless — the kernel
+    shares one socket — but the probe has no business joining it) and
+    deleted again, cached by binary identity like the other probes. Any
+    failure — old build, missing module, no CAP_NET_ADMIN, an unloadable
+    kernel module — means unknown, reported as None and cached as such;
+    the controller reads unknown as "keep this link on the relay".
+    """
+
+    config = Config.instance()
+    path = shutil.which(config.settings.Server.ubridge_path)
+    if not path:
+        return None
+    try:
+        stat = os.stat(path)
+        key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns)
+    except OSError:
+        return None
+
+    async with _lock:
+        if key in _vxlan_cache:
+            return _vxlan_cache[key]
+        supported = await _probe_vxlan(path, config, timeout)
+        _vxlan_cache[key] = supported
+        return supported
+
+
+async def _probe_vxlan(path, config, timeout):
+    """
+    Spawn a throwaway uBridge and run one ``vxlan create``/``delete`` cycle.
+    The scratch name (uuid suffix) cannot collide with a real link's
+    deterministic name; the scratch VNI and dstport are drawn from uuids so
+    two concurrent probes on one host (two servers, one uBridge user) cannot
+    fight over either. No ``dev=``: this probe answers "can this binary and
+    kernel create a VXLAN device at all" — resolving the real underlay
+    interface belongs to the link attach path, where an unroutable peer
+    means this particular link falls back to the relay, not that the
+    capability is absent.
+    """
+
+    working_dir = tempfile.mkdtemp(prefix="gns3-vxlan-probe-")
+    hypervisor = Hypervisor(
+        None,
+        path,
+        working_dir,
+        config.settings.Server.ubridge_control_transport,
+        config.settings.Server.host,
+        str(uuid.uuid4()),
+    )
+    name = f"gvx{uuid.uuid4().hex[:8]}"
+    # VNI in the deployment's own derived range (utils.vxlan) so the probe
+    # exercises exactly the kernel path real links use.
+    vni = VXLAN_VNI_MIN + uuid.uuid4().int % (VXLAN_VNI_MAX - VXLAN_VNI_MIN + 1)
+    dstport = 20000 + uuid.uuid4().int % 10000
+    # The scratch socket must never be the configured dstport: on it the
+    # scratch VNI shares a demux table with the live links' devices, and an
+    # unlucky draw would make the probe report the capability unknown for
+    # the process's lifetime (the result is cached).
+    if dstport == config.settings.Server.vxlan_dstport:
+        dstport = 20000 if dstport == 29999 else dstport + 1
+    try:
+
+        async def ask():
+            await hypervisor.start()
+            await hypervisor.connect()
+            await hypervisor.send(f"vxlan create {name} {vni} remote=127.0.0.1 dstport={dstport}")
+            try:
+                await hypervisor.send(f"vxlan delete {name}")
+            except UbridgeError as e:
+                log.warning("Probe VXLAN %s could not be deleted: %s", name, e)
+            return True
+
+        try:
+            return await asyncio.wait_for(ask(), timeout=timeout)
+        except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e:
+            # Old build ("Unknown command"), missing module, or no
+            # CAP_NET_ADMIN: unknown.
+            log.debug("uBridge vxlan-module probe failed: %s", e)
+            return None
+    finally:
+        with contextlib.suppress(Exception):
+            await hypervisor.stop()
+        shutil.rmtree(working_dir, ignore_errors=True)