feat: cross-compute VXLAN kernel datapath

Cross-compute links can ride kernel VXLAN tunnels instead of the uBridge
UDP relay: each endpoint creates one VXLAN device (uBridge's rtnetlink
vxlan module) enslaved into the same per-link Linux bridge the same-host
datapath uses, so filters, markers, capture and suspend keep running on
the anchor unchanged. Opt-in (Server.enable_vxlan_cross_compute, default
off) because the tunnel caps the usable MTU at the underlay minus the
outer headers; the relay stays the default and the fallback.

The controller derives what the pair must agree on: one VNI per link
(upper VNI space, per-link random salt) and deterministic device names
(gvl{link}{node}) that make crashed-run leftovers claimable
(show -> delete -> create). Same-host compute pairs, switch endpoints
and serial/unix-socket containers stay on the relay.

A VNI the kernel refuses self-heals on every path it can surface:
interactive create retries a fresh salt then degrades to the relay;
project open re-dispatches refused batch entries the same way; node
start recognises the refused device, stops the half-started node,
re-wires both ends on a fresh VNI (or degrades the link) and retries
the start. The device MTU is derived from the lower device (dev=) and
the per-link bridge inherits it - no configuration. The capability
probe (ubridge_vxlan in /capabilities) runs one scratch create/delete
on a scratch port that never equals the configured dstport.

The e2e scenario runs two computes on one machine (a network namespace
joined by a veth pair as the fabric) and needs root.

gns3server/controller/node.py

@@ -20,18 +20,33 @@ import copy
 import html
 import logging
 import os
+import re
 import uuid
 
 from ..config import Config
 from ..utils import int_to_macaddress, macaddress_to_int
 from ..utils.application_id import is_iol_runner_environment
 from ..utils.images import images_directories
+from ..utils.vxlan import vxlan_device_name
 from .controller_error import ComputeConflictError, ComputeError, ControllerError, ControllerTimeoutError
 from .node_types import BUILTIN_NODE_TYPES
 from .ports.port_factory import DynamipsPortFactory, PortFactory, StandardPortFactory
 
 log = logging.getLogger(__name__)
 
+# The compute's error when it could not create a VXLAN tunnel device — the
+# wording is a same-repo contract with KernelDatapathMixin._kernel_error
+# (only computes running this codebase carry VXLAN links: the capability
+# gate keeps everything else on the relay), and the captured device name
+# identifies the link (a pure function of link id + node id).
+_VXLAN_CONFLICT_RE = re.compile(r"Could not create VXLAN device (?P<device>\S+)")
+
+# Node-start self-heal rounds over a refused VXLAN VNI: the first re-wires
+# the link on a fresh VNI, the second degrades it to the relay (whose NIOs
+# cannot hit the conflict) — a start must not fail permanently because
+# another link already registered the VNI on the shared dstport socket.
+_VXLAN_START_HEALS = 2
+
 
 # Image properties of a node, mapped by node type. The value is the image
 # "type" used by the image manager (see utils.images). Docker images are not
@@ -955,18 +970,32 @@ class Node:
                     f"Cannot start node '{self._name}': {len(failed_links)} deferred link(s) "
                     "could not be restored. Please try again."
                 )
-        try:
-            # For IOU: we need to send the licence everytime we start a node
-            if self.node_type == "iou":
-                license_check = self._project.controller.iou_license.get("license_check", True)
-                iourc_content = self._project.controller.iou_license.get("iourc_content", None)
-                await self.post(
-                    "/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
+        for heal in range(_VXLAN_START_HEALS + 1):
+            try:
+                await self._start_on_compute(data)
+                break
+            except asyncio.TimeoutError:
+                raise ControllerTimeoutError(f"Timeout when starting {self._name}")
+            except ComputeError as e:
+                conflicted = self._vxlan_conflicted_link(e)
+                if conflicted is None or heal == _VXLAN_START_HEALS:
+                    raise
+                degrade = heal > 0  # a second conflict takes the relay
+                log.warning(
+                    "Node '%s': start refused over a VXLAN VNI conflict on link %s — %s and retrying",
+                    self._name,
+                    conflicted.id,
+                    "degrading the link to the uBridge relay" if degrade else "re-wiring the link on a fresh VNI",
                 )
-            else:
-                await self.post("/start", data=data, timeout=240)
-        except asyncio.TimeoutError:
-            raise ControllerTimeoutError(f"Timeout when starting {self._name}")
+                # The refused start leaves the node half-started (the
+                # emulator process runs, the wiring partial) — stop it
+                # first: a start on a running container early-returns
+                # without wiring anything.
+                await self.stop()
+                if await self._project._redispatch_vxlan_links([conflicted], downgrade=degrade) and not degrade:
+                    # The fresh VNI was already refused before the start
+                    # could even retry — degrade the pair to the relay now.
+                    await self._project._redispatch_vxlan_links([conflicted], downgrade=True)
         self._confirm_status("started")
         # A started node may have brought kernel anchors into existence that
         # a link needs to finish wiring (the Ethernet-switch fast path joins
@@ -974,6 +1003,42 @@ class Node:
         for link in self._links:
             await link.node_started(self)
 
+    async def _start_on_compute(self, data=None):
+        """
+        The compute-side start request (IOU carries its licence along).
+        """
+
+        # For IOU: we need to send the licence everytime we start a node
+        if self.node_type == "iou":
+            license_check = self._project.controller.iou_license.get("license_check", True)
+            iourc_content = self._project.controller.iou_license.get("iourc_content", None)
+            await self.post(
+                "/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
+            )
+        else:
+            await self.post("/start", data=data, timeout=240)
+
+    def _vxlan_conflicted_link(self, error):
+        """
+        The link whose VXLAN device a failed node start could not create,
+        or None for every other failure.
+
+        The compute's error names the device it could not create and the
+        name is a pure function of (link id, node id), so it identifies the
+        link with no extra API surface. An unmatched message — another
+        failure, a future wording change — simply leaves the error to
+        surface unchanged, which is exactly the behaviour without the heal.
+        """
+
+        match = _VXLAN_CONFLICT_RE.search(str(error))
+        if match is None:
+            return None
+        device = match.group("device")
+        for link in self._links:
+            if vxlan_device_name(link.id, self.id) == device and any(nio.get("vxlan") for nio in link._link_data):
+                return link
+        return None
+
     async def stop(self, strict=False):
         """
         Stop a node

gns3server/controller/project.py

@@ -49,7 +49,7 @@ from .link import _UNSET
 from .node import Node
 from .snapshot import Snapshot
 from .topology import load_topology, project_to_topology
-from .udp_link import UDPLink
+from .udp_link import _VXLAN_ATTEMPTS, UDPLink
 
 log = logging.getLogger(__name__)
 
@@ -1021,6 +1021,84 @@ class Project:
         entries = await link._prepare()
         return (link, entries)
 
+    async def _dispatch_nio_batch(self, compute, nio_entries):
+        """
+        POST one compute's NIO batch for this project, returning the entries
+        it refused (VXLAN entries a compute could not wire — see the batch
+        route; other NIO types fail the whole request).
+        """
+
+        response = await compute.post(
+            f"/projects/{self._id}/nios/batch",
+            data={"nios": nio_entries},
+            timeout=300,
+        )
+        return response.json.get("failed") or []
+
+    async def _redispatch_vxlan_links(self, links, downgrade):
+        """
+        Re-prepare *links* and dispatch their NIOs in a second per-compute
+        batch, returning the links the computes still refused.
+
+        With ``downgrade=False`` each link's VNI salt is bumped first — a
+        fresh VNI, since the refused one was already registered on the
+        host's shared dstport socket. With ``downgrade=True`` the links are
+        marked degraded to the uBridge relay (whose UDP NIOs cannot fail
+        this way) before re-preparing.
+        """
+
+        for link in links:
+            if downgrade:
+                log.warning(
+                    "Project '%s' [%s]: VXLAN wiring failed for link %s — falling back to the uBridge relay datapath",
+                    self._name,
+                    self._id,
+                    link.id,
+                )
+                link._vxlan_downgraded = True
+            else:
+                link._vxlan_salt += 1
+        per_compute = {}
+        by_key = {}
+        for link in links:
+            for node, adapter_number, port_number, nio_data in await link._prepare():
+                by_key[(str(node.id), adapter_number, port_number)] = link
+                per_compute.setdefault(node.compute, []).append(
+                    {
+                        "node_id": node.id,
+                        "adapter_number": adapter_number,
+                        "port_number": port_number,
+                        "nio": nio_data,
+                    }
+                )
+        if not per_compute:
+            return []
+        results = await asyncio.gather(
+            *[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
+        )
+        failed = []
+        for item in (i for sub in results for i in sub):
+            link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
+            if link is not None and link not in failed:
+                failed.append(link)
+        return failed
+
+    async def _heal_refused_vxlan_links(self, pending):
+        """
+        Drive the links a compute refused at open onto a working datapath:
+        fresh-VNI rounds first — the same candidate count an interactive
+        link creation tries — then, for whatever still fails, the
+        degradation to the uBridge relay (whose UDP NIOs cannot hit this
+        refusal).
+        """
+
+        for _ in range(_VXLAN_ATTEMPTS - 1):
+            if not pending:
+                return
+            pending = await self._redispatch_vxlan_links(pending, downgrade=False)
+        if pending:
+            await self._redispatch_vxlan_links(pending, downgrade=True)
+
     async def restore_deferred_links(self, node):
         """
         Create on the computes the NIOs of the links that were deferred while
@@ -1994,15 +2072,30 @@ class Project:
                         }
                     )
 
-            async def _dispatch_batch(compute, nio_entries):
-                await compute.post(
-                    f"/projects/{self._id}/nios/batch",
-                    data={"nios": nio_entries},
-                    timeout=300,
-                )
-
             if per_compute:
-                await asyncio.gather(*[_dispatch_batch(c, n) for c, n in per_compute.items()])
+                results = await asyncio.gather(
+                    *[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
+                )
+                failed = [item for sub in results for item in sub]
+            else:
+                failed = []
+            # A VXLAN entry a compute refused (its VNI already registered on
+            # the host's shared socket, no route to the peer) must not fail
+            # the whole open: the refused links are healed onto a working
+            # datapath (fresh VNIs, then the relay) — see
+            # _heal_refused_vxlan_links.
+            if failed:
+                by_key = {}
+                for link, entries in valid:
+                    for node, adapter_number, port_number, nio_data in entries:
+                        if nio_data.get("vxlan"):
+                            by_key[(str(node.id), adapter_number, port_number)] = link
+                pending = []
+                for item in failed:
+                    link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
+                    if link is not None and link not in pending:
+                        pending.append(link)
+                await self._heal_refused_vxlan_links(pending)
 
             # Finalise every link: wire node/port back-references, mark created,
             # notify clients, and apply project-level marker definitions.

gns3server/controller/udp_link.py

@@ -18,6 +18,7 @@
 
 import asyncio
 import logging
+import uuid
 
 from gns3server.config import Config
 from gns3server.utils.application_id import is_iol_runner_environment
@@ -26,6 +27,7 @@ from gns3server.utils.packet_filter_validation import (
     split_kernel_only_features,
     validate_bpf_syntax,
 )
+from gns3server.utils.vxlan import derive_vni, vxlan_device_name
 
 from .compute import ComputeError
 from .controller_error import ControllerError, ControllerNotFoundError
@@ -52,6 +54,10 @@ _MARKER_CAPABLE_TYPES = frozenset(
 
 log = logging.getLogger(__name__)
 
+# VXLAN wiring attempts per link creation (the first plus the salt-bumped
+# retries) before the link degrades to the uBridge relay.
+_VXLAN_ATTEMPTS = 3
+
 
 def _is_unix_socket_docker(node):
     """
@@ -92,6 +98,15 @@ class UDPLink(Link):
         # of its endpoints has a missing image. The NIO creation is retried
         # once the image is resolved.
         self._deferred = False
+        # Cross-compute VXLAN wiring state: the salt is this link object's
+        # starting point in the VNI space — random per object, so a project
+        # reopen re-rolls every link's VNI and a link whose VNI collided
+        # with an unrelated device on a host is not permanently unlucky
+        # (the device *name*, which cleanup keys on, stays deterministic).
+        # _commit_vxlan bumps it for an in-flight retry; the downgrade flag
+        # marks a link degraded to the UDP relay for this object's lifetime.
+        self._vxlan_salt = uuid.uuid4().int
+        self._vxlan_downgraded = False
 
     @property
     def debug_link_data(self):
@@ -206,9 +221,13 @@ class UDPLink(Link):
         Every adapter owns an anchor, so the datapath is a runtime decision —
         links attach to running nodes too: the compute side enslaves the
         anchor (brctl) or attaches the relay to it (add_nio_ethernet) without
-        touching the interfaces the node itself uses. Both endpoints must be
-        on the same compute (there is no kernel link across hosts) and both
-        must be able to anchor: Docker adapters are born as veth pairs, and
+        touching the interfaces the node itself uses. Both endpoints must sit
+        on computes that can carry their end: on one compute the ends meet in
+        a per-link Linux bridge; across computes each end additionally rides
+        its own VXLAN device (opt-in via enable_vxlan_cross_compute, uBridge's
+        vxlan module) enslaved into the same per-link bridge, so everything
+        below the bridge — filters, markers, capture, carrier — is identical
+        either way. Both must be able to anchor: Docker adapters are born as veth pairs, and
         IOL runner containers — whose guest leg is unix sockets, not a veth —
         anchor on persistent TAPs reached through the bridge module's
         swappable TAP leg (its own capability pair, see
@@ -232,8 +251,21 @@ class UDPLink(Link):
 
         if not Config.instance().settings.Server.enable_kernel_datapath:
             return False
-        if node1.compute.id != node2.compute.id:
-            return False
+        cross_compute = node1.compute.id != node2.compute.id
+        if cross_compute:
+            # Cross-compute kernel links tunnel: each endpoint rides a VXLAN
+            # device into the same per-link bridge the same-host datapath
+            # uses. Opt-in (the tunnel caps the usable MTU at the underlay
+            # minus the outer headers, so it pays off on jumbo fabrics) and
+            # gated on both computes' uBridge being able to create the
+            # device; a link that cannot tunnel stays on the relay, where it
+            # always works. The downgrade flag is the runtime fallback: after
+            # the kernel kept refusing this link's VNI, the link stays on the
+            # relay for this object's lifetime.
+            if not Config.instance().settings.Server.enable_vxlan_cross_compute:
+                return False
+            if self._vxlan_downgraded:
+                return False
         if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2):
             return False
         for side in self._nodes:
@@ -249,10 +281,17 @@ class UDPLink(Link):
         # pair is the only shape a bridge-to-bridge cable can take).
         switch = self._switch_endpoint()
         if switch is not None:
+            if cross_compute:
+                # The fast path is a one-host shape (the peer's anchor is
+                # absorbed into the switch's bridge); a switch link across
+                # computes stays on the relay.
+                return False
             if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
                 return True
             peer = node2 if switch[0] == 0 else node1
             return self._kernel_endpoint_ready(peer)
+        if cross_compute:
+            return self._vxlan_endpoint_ready(node1) and self._vxlan_endpoint_ready(node2)
         return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2)
 
     def _switch_endpoint(self):
@@ -305,6 +344,23 @@ class UDPLink(Link):
             return capabilities.get("ubridge_tap") is True and capabilities.get("vpcs_tap") is True
         return False
 
+    @staticmethod
+    def _vxlan_endpoint_ready(node):
+        """
+        Whether a node can carry its end of a cross-compute VXLAN link: it
+        must be able to anchor (the same per-node-type capability the
+        same-host kernel datapath needs — its end of the tunnel is enslaved
+        into a per-link bridge like any kernel link's) and its compute's
+        uBridge must be able to create the VXLAN device the tunnel rides.
+        Strict True on both, like the other capability gates: an unreported
+        or failed probe keeps the link on the relay, where it always works.
+        """
+
+        if not UDPLink._kernel_endpoint_ready(node):
+            return False
+        capabilities = node.compute.capabilities or {}
+        return capabilities.get("ubridge_vxlan") is True
+
     async def _prepare(self):
         """
         Local-only link setup: resolve peer addresses, reserve UDP ports and
@@ -330,10 +386,77 @@ class UDPLink(Link):
         adapter_number2 = self._nodes[1]["adapter_number"]
         port_number2 = self._nodes[1]["port_number"]
 
-        if self._kernel_datapath_eligible(node1, node2):
+        cross_compute = node1.compute.id != node2.compute.id
+        # A cross-compute link needs the compute pair's addresses on both
+        # datapaths (the VXLAN tunnel's outer endpoints, the relay's UDP
+        # peers): resolve once, and let the no-common-subnet failure stay a
+        # ControllerError either way — it is a deployment error, not a
+        # datapath decision. Equal addresses mean both computes share one
+        # host: the kernel shares one VXLAN socket per dstport, so a tunnel
+        # pair cannot live there, and such links keep the relay, where the
+        # two ends are told apart by their UDP ports.
+        hosts = None
+        if cross_compute:
+            try:
+                hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
+            except ValueError as e:
+                raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
+        same_host = hosts is not None and hosts[0] == hosts[1]
+
+        if self._kernel_datapath_eligible(node1, node2) and not same_host:
             node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True)
             node1_markers, node2_markers = self._get_node_markers(node1, node2)
             switch = self._switch_endpoint()
+            if hosts is not None:
+                # Cross-compute VXLAN: each end gets one VXLAN device
+                # carrying the link's VNI, enslaved into the per-link kernel
+                # bridge next to the node's anchor (see _kernel_attach on
+                # the compute side). The controller derives name and VNI —
+                # both ends must agree on the VNI and neither can ask the
+                # other — and the compute resolves the underlay interface
+                # locally from the peer address, so the kernel derives the
+                # device MTU (underlay - 50) and the bridge reports the
+                # honest "what this fabric can carry" value. A VNI the
+                # kernel refuses as already registered on the shared
+                # dstport socket surfaces as a commit failure; create()
+                # retries with a bumped salt and degrades to the relay when
+                # the retries run out.
+                node1_host, node2_host = hosts
+                dstport = Config.instance().settings.Server.vxlan_dstport
+                vni = derive_vni(self._id, self._vxlan_salt)
+                bridge_name = "gns3" + self._id.replace("-", "")[:11]
+                self._link_data = [
+                    {
+                        "type": "nio_bridge",
+                        "bridge": bridge_name,
+                        "vxlan": {
+                            "name": vxlan_device_name(self._id, node1.id),
+                            "vni": vni,
+                            "remote": node2_host,
+                            "dstport": dstport,
+                        },
+                        "filters": node1_filters,
+                        "markers": node1_markers,
+                        "suspend": self._suspended,
+                    },
+                    {
+                        "type": "nio_bridge",
+                        "bridge": bridge_name,
+                        "vxlan": {
+                            "name": vxlan_device_name(self._id, node2.id),
+                            "vni": vni,
+                            "remote": node1_host,
+                            "dstport": dstport,
+                        },
+                        "filters": node2_filters,
+                        "markers": node2_markers,
+                        "suspend": self._suspended,
+                    },
+                ]
+                return [
+                    (node1, adapter_number1, port_number1, self._link_data[0]),
+                    (node2, adapter_number2, port_number2, self._link_data[1]),
+                ]
             if switch is not None:
                 if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
                     # Switch-to-switch cascade: one veth pair joins the two
@@ -449,11 +572,16 @@ class UDPLink(Link):
                 (node2, adapter_number2, port_number2, self._link_data[1]),
             ]
 
-        # Get an IP allowing communication between both host
-        try:
-            (node1_host, node2_host) = await node1.compute.get_ip_on_same_subnet(node2.compute)
-        except ValueError as e:
-            raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
+        # Relay datapath: UDP tunnels between the computes' addresses —
+        # already resolved for a cross-compute pair, resolved here otherwise
+        # (a same-compute relay pair resolves to the host's own address on
+        # both sides, distinguished by their UDP ports).
+        if hosts is None:
+            try:
+                hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
+            except ValueError as e:
+                raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
+        node1_host, node2_host = hosts
 
         # Reserve a UDP port on both sides in parallel. Pre-allocated ports
         # (used during batch project loading) are popped from memory; otherwise
@@ -537,17 +665,70 @@ class UDPLink(Link):
             raise errors[0]
         self._created = True
 
+    @staticmethod
+    def _has_vxlan(entries):
+        """
+        Whether any prepared NIO spec of this link carries a VXLAN tunnel.
+        """
+
+        return any(nio_data.get("vxlan") for _, _, _, nio_data in entries)
+
     async def create(self):
         """
         Create the link on the nodes (interactive path: prepare + commit).
         """
 
         entries = await self._prepare()
-        await self._commit_nios(entries)
+        if self._has_vxlan(entries):
+            await self._commit_vxlan(entries)
+        else:
+            await self._commit_nios(entries)
         # New links automatically inherit every active project-level marker
         # definition so the user doesn't have to reconfigure.
         await self._project.apply_defs_to_new_link(self)
 
+    async def _commit_vxlan(self, entries):
+        """
+        Commit a cross-compute VXLAN link, retrying with a bumped VNI salt
+        on failure and degrading the link to the UDP relay when the retries
+        run out.
+
+        The kernel refuses a VNI already registered on the shared dstport
+        socket at device creation (EEXIST) — the one conflict a freshly
+        created, deterministically named device can hit: the other end's
+        device lives on another host, and a leftover under this end's name
+        is replaced by the attach path, not refused. Any other commit
+        failure (a stopped node, a missing underlay route) is retried too,
+        harmlessly — it fails the same way every attempt, and the final
+        relay attempt surfaces the real error if the link genuinely cannot
+        be created. VXLAN is opt-in and the relay always works, so a tunnel
+        that cannot be wired degrades instead of failing the link; the
+        degradation lasts for this link object's lifetime (a project reopen
+        retries VXLAN from a clean state).
+        """
+
+        last_error = None
+        for attempt in range(_VXLAN_ATTEMPTS):
+            if attempt:
+                self._vxlan_salt += 1
+                entries = await self._prepare()
+                if not self._has_vxlan(entries):
+                    # Eligibility changed under us (a capability disappeared):
+                    # commit the relay entries as prepared.
+                    await self._commit_nios(entries)
+                    return
+            try:
+                await self._commit_nios(entries)
+                return
+            except (ComputeError, ControllerError) as e:
+                last_error = e
+        log.warning(
+            "Link %s: VXLAN wiring failed (%s) — falling back to the uBridge relay datapath", self._id, last_error
+        )
+        self._vxlan_downgraded = True
+        entries = await self._prepare()
+        await self._commit_nios(entries)
+
     async def update(self):
         """
         Update the link on the nodes