mirror of
https://github.com/GNS3/gns3-server.git
synced 2026-11-05 06:41:13 +02:00
feat: cross-compute VXLAN kernel datapath
Cross-compute links can ride kernel VXLAN tunnels instead of the uBridge
UDP relay: each endpoint creates one VXLAN device (uBridge's rtnetlink
vxlan module) enslaved into the same per-link Linux bridge the same-host
datapath uses, so filters, markers, capture and suspend keep running on
the anchor unchanged. Opt-in (Server.enable_vxlan_cross_compute, default
off) because the tunnel caps the usable MTU at the underlay minus the
outer headers; the relay stays the default and the fallback.
The controller derives what the pair must agree on: one VNI per link
(upper VNI space, per-link random salt) and deterministic device names
(gvl{link}{node}) that make crashed-run leftovers claimable
(show -> delete -> create). Same-host compute pairs, switch endpoints
and serial/unix-socket containers stay on the relay.
A VNI the kernel refuses self-heals on every path it can surface:
interactive create retries a fresh salt then degrades to the relay;
project open re-dispatches refused batch entries the same way; node
start recognises the refused device, stops the half-started node,
re-wires both ends on a fresh VNI (or degrades the link) and retries
the start. The device MTU is derived from the lower device (dev=) and
the per-link bridge inherits it - no configuration. The capability
probe (ubridge_vxlan in /capabilities) runs one scratch create/delete
on a scratch port that never equals the configured dstport.
The e2e scenario runs two computes on one machine (a network namespace
joined by a veth pair as the fabric) and needs root.
This commit is contained in:
24 files changed
+2107
-59
No files matched your search
@@ -30,6 +30,7 @@ from gns3server.compute.ubridge.tc_probe import (
|
||||
probe_iol_tap_support,
|
||||
probe_tap_support,
|
||||
probe_tc_capabilities,
|
||||
probe_vxlan_support,
|
||||
)
|
||||
from gns3server.compute.vpcs.vpcs_probe import probe_vpcs_tap
|
||||
from gns3server.utils.path import get_default_project_directory
|
||||
@@ -75,6 +76,11 @@ async def get_capabilities(request: Request) -> dict:
|
||||
# kernel link. None = unknown (probe failed or binary missing).
|
||||
vpcs_tap = await probe_vpcs_tap()
|
||||
|
||||
# VXLAN device creation (the vxlan module): the tunnel a cross-compute
|
||||
# kernel link rides. None = unknown (probe failed, uBridge too old, or
|
||||
# the kernel refuses vxlan devices here).
|
||||
ubridge_vxlan = await probe_vxlan_support()
|
||||
|
||||
# record the controller hostname or IP address
|
||||
if request.client:
|
||||
request.app.state.controller_host = request.client.host
|
||||
@@ -91,4 +97,5 @@ async def get_capabilities(request: Request) -> dict:
|
||||
"ubridge_iol_tap": ubridge_iol_tap,
|
||||
"ubridge_bridge_tap": ubridge_bridge_tap,
|
||||
"vpcs_tap": vpcs_tap,
|
||||
"ubridge_vxlan": ubridge_vxlan,
|
||||
}
|
||||
@@ -35,6 +35,7 @@ from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from gns3server import schemas
|
||||
from gns3server.compute.error import NodeError
|
||||
from gns3server.compute.project import Project
|
||||
from gns3server.compute.project_manager import ProjectManager
|
||||
from gns3server.utils.path import is_safe_path
|
||||
@@ -271,16 +272,35 @@ async def create_batch_nios(
|
||||
# count (standard == 1, Dynamips == 2) and await the async variant.
|
||||
sig = inspect.signature(node.manager.create_nio)
|
||||
dynamips_style = len(sig.parameters) >= 2
|
||||
failed = []
|
||||
for entry in entries:
|
||||
nio_settings = jsonable_encoder(entry.nio, exclude_unset=True)
|
||||
if dynamips_style:
|
||||
nio = await node.manager.create_nio(node, nio_settings)
|
||||
else:
|
||||
nio = node.manager.create_nio(nio_settings)
|
||||
await _add_nio_binding(node, entry.adapter_number, entry.port_number, nio)
|
||||
try:
|
||||
if dynamips_style:
|
||||
nio = await node.manager.create_nio(node, nio_settings)
|
||||
else:
|
||||
nio = node.manager.create_nio(nio_settings)
|
||||
await _add_nio_binding(node, entry.adapter_number, entry.port_number, nio)
|
||||
except NodeError:
|
||||
# A VXLAN entry that cannot be wired (its VNI already
|
||||
# registered on the host's shared socket, no route to the
|
||||
# peer) must not fail the whole project open: report it so
|
||||
# the controller can degrade that link to the uBridge relay.
|
||||
# Every other NIO type keeps the fail-the-batch semantics.
|
||||
if not nio_settings.get("vxlan"):
|
||||
raise
|
||||
failed.append(
|
||||
{
|
||||
"node_id": entry.node_id,
|
||||
"adapter_number": entry.adapter_number,
|
||||
"port_number": entry.port_number,
|
||||
}
|
||||
)
|
||||
return failed
|
||||
|
||||
await asyncio.gather(*[_create_one_node(nid, ents) for nid, ents in per_node.items()])
|
||||
return {"added": len(batch.nios)}
|
||||
results = await asyncio.gather(*[_create_one_node(nid, ents) for nid, ents in per_node.items()])
|
||||
failed = [item for node_failed in results for item in node_failed]
|
||||
return {"added": len(batch.nios) - len(failed), "failed": failed}
|
||||
|
||||
|
||||
@router.put(
|
||||
|
||||
@@ -375,7 +375,10 @@ class BaseManager:
|
||||
nio.markers = nio_settings.get("markers", {})
|
||||
nio.suspend = nio_settings.get("suspend", False)
|
||||
elif nio_settings["type"] == "nio_bridge":
|
||||
nio = NIOBridge(nio_settings["bridge"])
|
||||
# A vxlan sub-spec marks a cross-compute link: the node creates
|
||||
# the VXLAN device and enslaves it into the same per-link bridge
|
||||
# next to its anchor (see KernelDatapathMixin._kernel_attach).
|
||||
nio = NIOBridge(nio_settings["bridge"], nio_settings.get("vxlan"))
|
||||
nio.filters = nio_settings.get("filters", {})
|
||||
# Markers ride the NIO and attach to the veth host interface via
|
||||
# uBridge's AF_PACKET marker module (marker add_kernel).
|
||||
|
||||
@@ -369,7 +369,9 @@ class Dynamips(BaseManager):
|
||||
# Kernel-datapath NIO: the router binds the port's anchor TAP to
|
||||
# a hypervisor nio_tap and enslaves it into this per-link bridge
|
||||
# (Router._attach_kernel_nio). No hypervisor NIO is created here.
|
||||
nio = NIOBridge(nio_settings["bridge"])
|
||||
# A vxlan sub-spec (cross-compute link) rides along untouched —
|
||||
# the router's attach path wires it like any kernel link.
|
||||
nio = NIOBridge(nio_settings["bridge"], nio_settings.get("vxlan"))
|
||||
nio.filters = nio_settings.get("filters", {})
|
||||
nio.markers = nio_settings.get("markers", {})
|
||||
nio.suspend = nio_settings.get("suspend", False)
|
||||
|
||||
@@ -48,6 +48,9 @@ import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
|
||||
import psutil
|
||||
|
||||
from gns3server.compute.error import NodeError
|
||||
from gns3server.compute.ubridge.ubridge_error import UbridgeError
|
||||
@@ -264,6 +267,86 @@ class KernelDatapathMixin:
|
||||
|
||||
return not (isinstance(nio, NIOBridge) and nio.bridge is None)
|
||||
|
||||
async def _ensure_vxlan_device(self, spec):
|
||||
"""
|
||||
Create (or replace) the cross-compute VXLAN device this link end
|
||||
tunnels through and return the device name. The device name is a
|
||||
pure function of the link id, so a device found under it belongs to
|
||||
this link: either a leftover from a run that crashed without
|
||||
cleanup, or the previous instance of a node restart — deleted and
|
||||
re-created either way, which makes wiring idempotent. The lower
|
||||
device is resolved locally from the peer address; the kernel
|
||||
derives the tunnel MTU from it, which is what keeps the per-link
|
||||
bridge honest about what the underlay can carry.
|
||||
|
||||
A create failure with the name just cleared can only mean the VNI —
|
||||
the kernel refuses a VNI already registered on the shared dstport
|
||||
socket (EEXIST) — so it is surfaced as a wiring error and the
|
||||
controller retries the link with a fresh VNI.
|
||||
"""
|
||||
|
||||
name = spec["name"]
|
||||
remote = spec["remote"]
|
||||
try:
|
||||
await self._ubridge_send(f'vxlan show "{name}"')
|
||||
except UbridgeError:
|
||||
# Not present (or uBridge cannot answer): the create below is
|
||||
# the real gate either way.
|
||||
pass
|
||||
else:
|
||||
await self._ubridge_send(f'vxlan delete "{name}"')
|
||||
dev = await self._vxlan_lowerdev(remote)
|
||||
try:
|
||||
await self._ubridge_send(
|
||||
f'vxlan create "{name}" {spec["vni"]} remote={remote} dstport={spec["dstport"]} dev={dev}'
|
||||
)
|
||||
except UbridgeError as e:
|
||||
raise self._kernel_error(f"Could not create VXLAN device {name} to {remote} via {dev}: {e}")
|
||||
return name
|
||||
|
||||
async def _vxlan_lowerdev(self, remote):
|
||||
"""
|
||||
The local interface the route to *remote* egresses through — the
|
||||
``dev=`` a cross-compute VXLAN device needs so the kernel can derive
|
||||
its MTU from the underlay (peer MTU - 50) and pick the outer source
|
||||
address. A connected UDP socket makes the kernel pick the route
|
||||
without sending anything; the chosen source address is matched back
|
||||
to its interface.
|
||||
"""
|
||||
|
||||
try:
|
||||
info = socket.getaddrinfo(remote, 9, socket.AF_UNSPEC, socket.SOCK_DGRAM)
|
||||
except socket.gaierror as e:
|
||||
raise self._kernel_error(f"Cannot resolve VXLAN peer {remote}: {e}")
|
||||
last_error = None
|
||||
for af, socktype, proto, _, sa in info:
|
||||
try:
|
||||
with socket.socket(af, socktype, proto) as sock:
|
||||
sock.connect(sa)
|
||||
local_ip = sock.getsockname()[0]
|
||||
except OSError as e:
|
||||
last_error = e
|
||||
continue
|
||||
dev = self._interface_with_address(local_ip)
|
||||
if dev is not None:
|
||||
return dev
|
||||
raise self._kernel_error(f"No local interface owns {local_ip}, the route to VXLAN peer {remote}")
|
||||
raise self._kernel_error(f"No route to VXLAN peer {remote}: {last_error}")
|
||||
|
||||
@staticmethod
|
||||
def _interface_with_address(address):
|
||||
"""
|
||||
The name of the interface owning *address* (scope suffix stripped
|
||||
for link-local IPv6), or None.
|
||||
"""
|
||||
|
||||
plain = address.split("%")[0]
|
||||
for name, addrs in psutil.net_if_addrs().items():
|
||||
for addr in addrs:
|
||||
if addr.address.split("%")[0] == plain:
|
||||
return name
|
||||
return None
|
||||
|
||||
async def _kernel_attach(self, anchor, nio):
|
||||
"""
|
||||
Attach an anchor to the per-link kernel bridge carried by *nio*
|
||||
@@ -276,6 +359,12 @@ class KernelDatapathMixin:
|
||||
bridge first: an existing bridge is success instead of a create race
|
||||
failure.
|
||||
|
||||
A NIO carrying a ``vxlan`` spec is a cross-compute link: this end's
|
||||
VXLAN device is (re-)created first and enslaved into the same
|
||||
bridge next to the anchor, so the kernel encapsulates/decapsulates
|
||||
the frames while filters, markers, capture and carrier keep running
|
||||
on the anchor exactly like a same-host kernel link.
|
||||
|
||||
A NIO whose ``bridge`` is None means the anchor is bridged
|
||||
elsewhere — an Ethernet switch absorbed it into the switch's own
|
||||
bridge — so the membership work is skipped here and only the link
|
||||
@@ -287,6 +376,13 @@ class KernelDatapathMixin:
|
||||
"""
|
||||
|
||||
if nio.bridge is not None:
|
||||
vxlan_name = None
|
||||
if nio.vxlan:
|
||||
# Replace/refresh the device before the bridge work: a
|
||||
# leftover device may still be enslaved to a leftover bridge
|
||||
# (its deletion un-enrolls it), and the fresh device must be
|
||||
# in place when the bridge's MTU auto-adjust runs.
|
||||
vxlan_name = await self._ensure_vxlan_device(nio.vxlan)
|
||||
try:
|
||||
await self._ubridge_send(f'brctl create "{nio.bridge}"')
|
||||
except UbridgeError:
|
||||
@@ -294,6 +390,8 @@ class KernelDatapathMixin:
|
||||
await self._ubridge_send(f'brctl show "{nio.bridge}"')
|
||||
await self._ubridge_send(f'link set "{nio.bridge}" up')
|
||||
await self._ubridge_send(f'brctl addif "{nio.bridge}" "{anchor}"')
|
||||
if vxlan_name is not None:
|
||||
await self._ubridge_send(f'brctl addif "{nio.bridge}" "{vxlan_name}"')
|
||||
if nio.capturing:
|
||||
# Restore a capture that was active before a node restart
|
||||
# (mirrors the relay path's start_capture).
|
||||
@@ -373,6 +471,11 @@ class KernelDatapathMixin:
|
||||
enslaved fails with EBUSY, so the last endpoint to remove its port
|
||||
wins the deletion and the loser's failure is expected and suppressed.
|
||||
|
||||
A NIO carrying a ``vxlan`` spec is a cross-compute link: this end
|
||||
also deletes its own tunnel device (the peer deletes its own on its
|
||||
compute); the kernel's unregister path releases the bridge port, so
|
||||
it goes before the bridge delete, same as the anchor.
|
||||
|
||||
A NIO whose ``bridge`` is None owns no bridge here: the anchor is
|
||||
bridged by an Ethernet switch, which tears its own membership down
|
||||
on its side; this end only releases the link state it applied
|
||||
@@ -394,6 +497,9 @@ class KernelDatapathMixin:
|
||||
with contextlib.suppress(UbridgeError):
|
||||
await self._ubridge_send(f'brctl delif "{nio.bridge}" "{host_ifc}"')
|
||||
if nio.bridge is not None:
|
||||
if nio.vxlan:
|
||||
with contextlib.suppress(UbridgeError):
|
||||
await self._ubridge_send(f'vxlan delete "{nio.vxlan["name"]}"')
|
||||
with contextlib.suppress(UbridgeError):
|
||||
await self._ubridge_send(f'brctl delete "{nio.bridge}"')
|
||||
|
||||
@@ -407,6 +513,12 @@ class KernelDatapathMixin:
|
||||
the same contract as link deletion. A node restart re-creates the
|
||||
bridge from the NIO in the attach path, so nothing is lost by
|
||||
deleting it.
|
||||
|
||||
A cross-compute NIO's tunnel device is swept with its bridge: each
|
||||
end owns its own device, and a device left behind would hold its
|
||||
VNI on the host's shared socket until a reopen's attach path
|
||||
replaces it by name (which it does — the sweep just frees it
|
||||
sooner).
|
||||
"""
|
||||
|
||||
if not self.ubridge:
|
||||
@@ -416,6 +528,9 @@ class KernelDatapathMixin:
|
||||
# bridge None = externally bridged (an Ethernet switch owns
|
||||
# the bridge); never delete a bridge this node does not own.
|
||||
if isinstance(nio, NIOBridge) and nio.bridge is not None:
|
||||
if nio.vxlan:
|
||||
with contextlib.suppress(UbridgeError):
|
||||
await self._ubridge_send(f'vxlan delete "{nio.vxlan["name"]}"')
|
||||
with contextlib.suppress(UbridgeError):
|
||||
await self._ubridge_send(f'brctl delete "{nio.bridge}"')
|
||||
|
||||
|
||||
@@ -31,6 +31,13 @@ veth host end (marker add_kernel / capture start_kernel).
|
||||
Ethernet switch absorbed it into the switch's own kernel bridge — and this
|
||||
node applies only the link state the anchor carries (carrier, markers,
|
||||
impairments), never bridge membership.
|
||||
|
||||
A NIO may additionally carry ``vxlan`` (a BridgeVxlan spec): the link
|
||||
crosses computes through a VXLAN device this end creates (uBridge's
|
||||
``vxlan`` module) and enslaves into the same per-link bridge next to the
|
||||
anchor. The kernel encapsulates/decapsulates the frames; everything else —
|
||||
filters, markers, capture, carrier, suspend — still runs on the anchor, so
|
||||
the datapath behaves exactly like a same-host kernel link.
|
||||
"""
|
||||
|
||||
from .nio import NIO
|
||||
@@ -42,12 +49,16 @@ class NIOBridge(NIO):
|
||||
|
||||
:param bridge: name of the per-link Linux kernel bridge, or None when
|
||||
the anchor is bridged by an Ethernet switch (see module docstring)
|
||||
:param vxlan: cross-compute tunnel spec (dict: name/vni/remote/dstport)
|
||||
this end creates and enslaves into the bridge, or None for a
|
||||
same-host kernel link
|
||||
"""
|
||||
|
||||
def __init__(self, bridge):
|
||||
def __init__(self, bridge, vxlan=None):
|
||||
|
||||
super().__init__()
|
||||
self._bridge = bridge
|
||||
self._vxlan = vxlan
|
||||
|
||||
@property
|
||||
def bridge(self):
|
||||
@@ -60,6 +71,17 @@ class NIOBridge(NIO):
|
||||
|
||||
return self._bridge
|
||||
|
||||
@property
|
||||
def vxlan(self):
|
||||
"""
|
||||
Returns the cross-compute VXLAN tunnel spec, or None when the link
|
||||
is wired on this host alone.
|
||||
|
||||
:returns: dict (name/vni/remote/dstport) or None
|
||||
"""
|
||||
|
||||
return self._vxlan
|
||||
|
||||
def __str__(self):
|
||||
|
||||
return "NIO bridge"
|
||||
@@ -69,6 +91,7 @@ class NIOBridge(NIO):
|
||||
return {
|
||||
"type": "nio_bridge",
|
||||
"bridge": self._bridge,
|
||||
"vxlan": self._vxlan,
|
||||
"suspend": self._suspended,
|
||||
"filters": self._filters,
|
||||
"markers": self._markers,
|
||||
|
||||
@@ -43,6 +43,7 @@ from typing import Optional
|
||||
|
||||
from gns3server.config import Config
|
||||
from gns3server.utils.tc_capabilities import parse_tc_capabilities
|
||||
from gns3server.utils.vxlan import VXLAN_VNI_MAX, VXLAN_VNI_MIN
|
||||
|
||||
from .hypervisor import Hypervisor
|
||||
from .ubridge_error import UbridgeError
|
||||
@@ -59,6 +60,8 @@ _tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
|
||||
_iol_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
|
||||
# binary identity -> whether the bridge module can release a named TAP NIO (None = unknown)
|
||||
_bridge_tap_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
|
||||
# binary identity -> whether the vxlan module can create a device (None = unknown)
|
||||
_vxlan_cache: dict[tuple[str, int, int, int], Optional[bool]] = {}
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
@@ -358,3 +361,93 @@ async def _probe_bridge_tap(path, config, timeout):
|
||||
with contextlib.suppress(Exception):
|
||||
await hypervisor.stop()
|
||||
shutil.rmtree(working_dir, ignore_errors=True)
|
||||
|
||||
|
||||
async def probe_vxlan_support(timeout: float = 15.0):
|
||||
"""
|
||||
Whether this host's uBridge can create a VXLAN device (the ``vxlan``
|
||||
module) — the carrier a cross-compute kernel link tunnels through.
|
||||
Probed on a scratch device with a scratch VNI on a scratch dstport
|
||||
(never the deployment's configured port, so the probe cannot disturb
|
||||
live tunnels; a device on the port would still be harmless — the kernel
|
||||
shares one socket — but the probe has no business joining it) and
|
||||
deleted again, cached by binary identity like the other probes. Any
|
||||
failure — old build, missing module, no CAP_NET_ADMIN, an unloadable
|
||||
kernel module — means unknown, reported as None and cached as such;
|
||||
the controller reads unknown as "keep this link on the relay".
|
||||
"""
|
||||
|
||||
config = Config.instance()
|
||||
path = shutil.which(config.settings.Server.ubridge_path)
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
key = (path, stat.st_mtime_ns, stat.st_size, stat.st_ctime_ns)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
async with _lock:
|
||||
if key in _vxlan_cache:
|
||||
return _vxlan_cache[key]
|
||||
supported = await _probe_vxlan(path, config, timeout)
|
||||
_vxlan_cache[key] = supported
|
||||
return supported
|
||||
|
||||
|
||||
async def _probe_vxlan(path, config, timeout):
|
||||
"""
|
||||
Spawn a throwaway uBridge and run one ``vxlan create``/``delete`` cycle.
|
||||
The scratch name (uuid suffix) cannot collide with a real link's
|
||||
deterministic name; the scratch VNI and dstport are drawn from uuids so
|
||||
two concurrent probes on one host (two servers, one uBridge user) cannot
|
||||
fight over either. No ``dev=``: this probe answers "can this binary and
|
||||
kernel create a VXLAN device at all" — resolving the real underlay
|
||||
interface belongs to the link attach path, where an unroutable peer
|
||||
means this particular link falls back to the relay, not that the
|
||||
capability is absent.
|
||||
"""
|
||||
|
||||
working_dir = tempfile.mkdtemp(prefix="gns3-vxlan-probe-")
|
||||
hypervisor = Hypervisor(
|
||||
None,
|
||||
path,
|
||||
working_dir,
|
||||
config.settings.Server.ubridge_control_transport,
|
||||
config.settings.Server.host,
|
||||
str(uuid.uuid4()),
|
||||
)
|
||||
name = f"gvx{uuid.uuid4().hex[:8]}"
|
||||
# VNI in the deployment's own derived range (utils.vxlan) so the probe
|
||||
# exercises exactly the kernel path real links use.
|
||||
vni = VXLAN_VNI_MIN + uuid.uuid4().int % (VXLAN_VNI_MAX - VXLAN_VNI_MIN + 1)
|
||||
dstport = 20000 + uuid.uuid4().int % 10000
|
||||
# The scratch socket must never be the configured dstport: on it the
|
||||
# scratch VNI shares a demux table with the live links' devices, and an
|
||||
# unlucky draw would make the probe report the capability unknown for
|
||||
# the process's lifetime (the result is cached).
|
||||
if dstport == config.settings.Server.vxlan_dstport:
|
||||
dstport = 20000 if dstport == 29999 else dstport + 1
|
||||
try:
|
||||
|
||||
async def ask():
|
||||
await hypervisor.start()
|
||||
await hypervisor.connect()
|
||||
await hypervisor.send(f"vxlan create {name} {vni} remote=127.0.0.1 dstport={dstport}")
|
||||
try:
|
||||
await hypervisor.send(f"vxlan delete {name}")
|
||||
except UbridgeError as e:
|
||||
log.warning("Probe VXLAN %s could not be deleted: %s", name, e)
|
||||
return True
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(ask(), timeout=timeout)
|
||||
except (UbridgeError, OSError, asyncio.TimeoutError, ValueError) as e:
|
||||
# Old build ("Unknown command"), missing module, or no
|
||||
# CAP_NET_ADMIN: unknown.
|
||||
log.debug("uBridge vxlan-module probe failed: %s", e)
|
||||
return None
|
||||
finally:
|
||||
with contextlib.suppress(Exception):
|
||||
await hypervisor.stop()
|
||||
shutil.rmtree(working_dir, ignore_errors=True)
|
||||
@@ -20,18 +20,33 @@ import copy
|
||||
import html
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from ..config import Config
|
||||
from ..utils import int_to_macaddress, macaddress_to_int
|
||||
from ..utils.application_id import is_iol_runner_environment
|
||||
from ..utils.images import images_directories
|
||||
from ..utils.vxlan import vxlan_device_name
|
||||
from .controller_error import ComputeConflictError, ComputeError, ControllerError, ControllerTimeoutError
|
||||
from .node_types import BUILTIN_NODE_TYPES
|
||||
from .ports.port_factory import DynamipsPortFactory, PortFactory, StandardPortFactory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# The compute's error when it could not create a VXLAN tunnel device — the
|
||||
# wording is a same-repo contract with KernelDatapathMixin._kernel_error
|
||||
# (only computes running this codebase carry VXLAN links: the capability
|
||||
# gate keeps everything else on the relay), and the captured device name
|
||||
# identifies the link (a pure function of link id + node id).
|
||||
_VXLAN_CONFLICT_RE = re.compile(r"Could not create VXLAN device (?P<device>\S+)")
|
||||
|
||||
# Node-start self-heal rounds over a refused VXLAN VNI: the first re-wires
|
||||
# the link on a fresh VNI, the second degrades it to the relay (whose NIOs
|
||||
# cannot hit the conflict) — a start must not fail permanently because
|
||||
# another link already registered the VNI on the shared dstport socket.
|
||||
_VXLAN_START_HEALS = 2
|
||||
|
||||
|
||||
# Image properties of a node, mapped by node type. The value is the image
|
||||
# "type" used by the image manager (see utils.images). Docker images are not
|
||||
@@ -955,18 +970,32 @@ class Node:
|
||||
f"Cannot start node '{self._name}': {len(failed_links)} deferred link(s) "
|
||||
"could not be restored. Please try again."
|
||||
)
|
||||
try:
|
||||
# For IOU: we need to send the licence everytime we start a node
|
||||
if self.node_type == "iou":
|
||||
license_check = self._project.controller.iou_license.get("license_check", True)
|
||||
iourc_content = self._project.controller.iou_license.get("iourc_content", None)
|
||||
await self.post(
|
||||
"/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
|
||||
for heal in range(_VXLAN_START_HEALS + 1):
|
||||
try:
|
||||
await self._start_on_compute(data)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
raise ControllerTimeoutError(f"Timeout when starting {self._name}")
|
||||
except ComputeError as e:
|
||||
conflicted = self._vxlan_conflicted_link(e)
|
||||
if conflicted is None or heal == _VXLAN_START_HEALS:
|
||||
raise
|
||||
degrade = heal > 0 # a second conflict takes the relay
|
||||
log.warning(
|
||||
"Node '%s': start refused over a VXLAN VNI conflict on link %s — %s and retrying",
|
||||
self._name,
|
||||
conflicted.id,
|
||||
"degrading the link to the uBridge relay" if degrade else "re-wiring the link on a fresh VNI",
|
||||
)
|
||||
else:
|
||||
await self.post("/start", data=data, timeout=240)
|
||||
except asyncio.TimeoutError:
|
||||
raise ControllerTimeoutError(f"Timeout when starting {self._name}")
|
||||
# The refused start leaves the node half-started (the
|
||||
# emulator process runs, the wiring partial) — stop it
|
||||
# first: a start on a running container early-returns
|
||||
# without wiring anything.
|
||||
await self.stop()
|
||||
if await self._project._redispatch_vxlan_links([conflicted], downgrade=degrade) and not degrade:
|
||||
# The fresh VNI was already refused before the start
|
||||
# could even retry — degrade the pair to the relay now.
|
||||
await self._project._redispatch_vxlan_links([conflicted], downgrade=True)
|
||||
self._confirm_status("started")
|
||||
# A started node may have brought kernel anchors into existence that
|
||||
# a link needs to finish wiring (the Ethernet-switch fast path joins
|
||||
@@ -974,6 +1003,42 @@ class Node:
|
||||
for link in self._links:
|
||||
await link.node_started(self)
|
||||
|
||||
async def _start_on_compute(self, data=None):
|
||||
"""
|
||||
The compute-side start request (IOU carries its licence along).
|
||||
"""
|
||||
|
||||
# For IOU: we need to send the licence everytime we start a node
|
||||
if self.node_type == "iou":
|
||||
license_check = self._project.controller.iou_license.get("license_check", True)
|
||||
iourc_content = self._project.controller.iou_license.get("iourc_content", None)
|
||||
await self.post(
|
||||
"/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
|
||||
)
|
||||
else:
|
||||
await self.post("/start", data=data, timeout=240)
|
||||
|
||||
def _vxlan_conflicted_link(self, error):
|
||||
"""
|
||||
The link whose VXLAN device a failed node start could not create,
|
||||
or None for every other failure.
|
||||
|
||||
The compute's error names the device it could not create and the
|
||||
name is a pure function of (link id, node id), so it identifies the
|
||||
link with no extra API surface. An unmatched message — another
|
||||
failure, a future wording change — simply leaves the error to
|
||||
surface unchanged, which is exactly the behaviour without the heal.
|
||||
"""
|
||||
|
||||
match = _VXLAN_CONFLICT_RE.search(str(error))
|
||||
if match is None:
|
||||
return None
|
||||
device = match.group("device")
|
||||
for link in self._links:
|
||||
if vxlan_device_name(link.id, self.id) == device and any(nio.get("vxlan") for nio in link._link_data):
|
||||
return link
|
||||
return None
|
||||
|
||||
async def stop(self, strict=False):
|
||||
"""
|
||||
Stop a node
|
||||
|
||||
@@ -49,7 +49,7 @@ from .link import _UNSET
|
||||
from .node import Node
|
||||
from .snapshot import Snapshot
|
||||
from .topology import load_topology, project_to_topology
|
||||
from .udp_link import UDPLink
|
||||
from .udp_link import _VXLAN_ATTEMPTS, UDPLink
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -1021,6 +1021,84 @@ class Project:
|
||||
entries = await link._prepare()
|
||||
return (link, entries)
|
||||
|
||||
async def _dispatch_nio_batch(self, compute, nio_entries):
|
||||
"""
|
||||
POST one compute's NIO batch for this project, returning the entries
|
||||
it refused (VXLAN entries a compute could not wire — see the batch
|
||||
route; other NIO types fail the whole request).
|
||||
"""
|
||||
|
||||
response = await compute.post(
|
||||
f"/projects/{self._id}/nios/batch",
|
||||
data={"nios": nio_entries},
|
||||
timeout=300,
|
||||
)
|
||||
return response.json.get("failed") or []
|
||||
|
||||
async def _redispatch_vxlan_links(self, links, downgrade):
|
||||
"""
|
||||
Re-prepare *links* and dispatch their NIOs in a second per-compute
|
||||
batch, returning the links the computes still refused.
|
||||
|
||||
With ``downgrade=False`` each link's VNI salt is bumped first — a
|
||||
fresh VNI, since the refused one was already registered on the
|
||||
host's shared dstport socket. With ``downgrade=True`` the links are
|
||||
marked degraded to the uBridge relay (whose UDP NIOs cannot fail
|
||||
this way) before re-preparing.
|
||||
"""
|
||||
|
||||
for link in links:
|
||||
if downgrade:
|
||||
log.warning(
|
||||
"Project '%s' [%s]: VXLAN wiring failed for link %s — falling back to the uBridge relay datapath",
|
||||
self._name,
|
||||
self._id,
|
||||
link.id,
|
||||
)
|
||||
link._vxlan_downgraded = True
|
||||
else:
|
||||
link._vxlan_salt += 1
|
||||
per_compute = {}
|
||||
by_key = {}
|
||||
for link in links:
|
||||
for node, adapter_number, port_number, nio_data in await link._prepare():
|
||||
by_key[(str(node.id), adapter_number, port_number)] = link
|
||||
per_compute.setdefault(node.compute, []).append(
|
||||
{
|
||||
"node_id": node.id,
|
||||
"adapter_number": adapter_number,
|
||||
"port_number": port_number,
|
||||
"nio": nio_data,
|
||||
}
|
||||
)
|
||||
if not per_compute:
|
||||
return []
|
||||
results = await asyncio.gather(
|
||||
*[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
|
||||
)
|
||||
failed = []
|
||||
for item in (i for sub in results for i in sub):
|
||||
link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
|
||||
if link is not None and link not in failed:
|
||||
failed.append(link)
|
||||
return failed
|
||||
|
||||
async def _heal_refused_vxlan_links(self, pending):
|
||||
"""
|
||||
Drive the links a compute refused at open onto a working datapath:
|
||||
fresh-VNI rounds first — the same candidate count an interactive
|
||||
link creation tries — then, for whatever still fails, the
|
||||
degradation to the uBridge relay (whose UDP NIOs cannot hit this
|
||||
refusal).
|
||||
"""
|
||||
|
||||
for _ in range(_VXLAN_ATTEMPTS - 1):
|
||||
if not pending:
|
||||
return
|
||||
pending = await self._redispatch_vxlan_links(pending, downgrade=False)
|
||||
if pending:
|
||||
await self._redispatch_vxlan_links(pending, downgrade=True)
|
||||
|
||||
async def restore_deferred_links(self, node):
|
||||
"""
|
||||
Create on the computes the NIOs of the links that were deferred while
|
||||
@@ -1994,15 +2072,30 @@ class Project:
|
||||
}
|
||||
)
|
||||
|
||||
async def _dispatch_batch(compute, nio_entries):
|
||||
await compute.post(
|
||||
f"/projects/{self._id}/nios/batch",
|
||||
data={"nios": nio_entries},
|
||||
timeout=300,
|
||||
)
|
||||
|
||||
if per_compute:
|
||||
await asyncio.gather(*[_dispatch_batch(c, n) for c, n in per_compute.items()])
|
||||
results = await asyncio.gather(
|
||||
*[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
|
||||
)
|
||||
failed = [item for sub in results for item in sub]
|
||||
else:
|
||||
failed = []
|
||||
# A VXLAN entry a compute refused (its VNI already registered on
|
||||
# the host's shared socket, no route to the peer) must not fail
|
||||
# the whole open: the refused links are healed onto a working
|
||||
# datapath (fresh VNIs, then the relay) — see
|
||||
# _heal_refused_vxlan_links.
|
||||
if failed:
|
||||
by_key = {}
|
||||
for link, entries in valid:
|
||||
for node, adapter_number, port_number, nio_data in entries:
|
||||
if nio_data.get("vxlan"):
|
||||
by_key[(str(node.id), adapter_number, port_number)] = link
|
||||
pending = []
|
||||
for item in failed:
|
||||
link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
|
||||
if link is not None and link not in pending:
|
||||
pending.append(link)
|
||||
await self._heal_refused_vxlan_links(pending)
|
||||
|
||||
# Finalise every link: wire node/port back-references, mark created,
|
||||
# notify clients, and apply project-level marker definitions.
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from gns3server.config import Config
|
||||
from gns3server.utils.application_id import is_iol_runner_environment
|
||||
@@ -26,6 +27,7 @@ from gns3server.utils.packet_filter_validation import (
|
||||
split_kernel_only_features,
|
||||
validate_bpf_syntax,
|
||||
)
|
||||
from gns3server.utils.vxlan import derive_vni, vxlan_device_name
|
||||
|
||||
from .compute import ComputeError
|
||||
from .controller_error import ControllerError, ControllerNotFoundError
|
||||
@@ -52,6 +54,10 @@ _MARKER_CAPABLE_TYPES = frozenset(
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# VXLAN wiring attempts per link creation (the first plus the salt-bumped
|
||||
# retries) before the link degrades to the uBridge relay.
|
||||
_VXLAN_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _is_unix_socket_docker(node):
|
||||
"""
|
||||
@@ -92,6 +98,15 @@ class UDPLink(Link):
|
||||
# of its endpoints has a missing image. The NIO creation is retried
|
||||
# once the image is resolved.
|
||||
self._deferred = False
|
||||
# Cross-compute VXLAN wiring state: the salt is this link object's
|
||||
# starting point in the VNI space — random per object, so a project
|
||||
# reopen re-rolls every link's VNI and a link whose VNI collided
|
||||
# with an unrelated device on a host is not permanently unlucky
|
||||
# (the device *name*, which cleanup keys on, stays deterministic).
|
||||
# _commit_vxlan bumps it for an in-flight retry; the downgrade flag
|
||||
# marks a link degraded to the UDP relay for this object's lifetime.
|
||||
self._vxlan_salt = uuid.uuid4().int
|
||||
self._vxlan_downgraded = False
|
||||
|
||||
@property
|
||||
def debug_link_data(self):
|
||||
@@ -206,9 +221,13 @@ class UDPLink(Link):
|
||||
Every adapter owns an anchor, so the datapath is a runtime decision —
|
||||
links attach to running nodes too: the compute side enslaves the
|
||||
anchor (brctl) or attaches the relay to it (add_nio_ethernet) without
|
||||
touching the interfaces the node itself uses. Both endpoints must be
|
||||
on the same compute (there is no kernel link across hosts) and both
|
||||
must be able to anchor: Docker adapters are born as veth pairs, and
|
||||
touching the interfaces the node itself uses. Both endpoints must sit
|
||||
on computes that can carry their end: on one compute the ends meet in
|
||||
a per-link Linux bridge; across computes each end additionally rides
|
||||
its own VXLAN device (opt-in via enable_vxlan_cross_compute, uBridge's
|
||||
vxlan module) enslaved into the same per-link bridge, so everything
|
||||
below the bridge — filters, markers, capture, carrier — is identical
|
||||
either way. Both must be able to anchor: Docker adapters are born as veth pairs, and
|
||||
IOL runner containers — whose guest leg is unix sockets, not a veth —
|
||||
anchor on persistent TAPs reached through the bridge module's
|
||||
swappable TAP leg (its own capability pair, see
|
||||
@@ -232,8 +251,21 @@ class UDPLink(Link):
|
||||
|
||||
if not Config.instance().settings.Server.enable_kernel_datapath:
|
||||
return False
|
||||
if node1.compute.id != node2.compute.id:
|
||||
return False
|
||||
cross_compute = node1.compute.id != node2.compute.id
|
||||
if cross_compute:
|
||||
# Cross-compute kernel links tunnel: each endpoint rides a VXLAN
|
||||
# device into the same per-link bridge the same-host datapath
|
||||
# uses. Opt-in (the tunnel caps the usable MTU at the underlay
|
||||
# minus the outer headers, so it pays off on jumbo fabrics) and
|
||||
# gated on both computes' uBridge being able to create the
|
||||
# device; a link that cannot tunnel stays on the relay, where it
|
||||
# always works. The downgrade flag is the runtime fallback: after
|
||||
# the kernel kept refusing this link's VNI, the link stays on the
|
||||
# relay for this object's lifetime.
|
||||
if not Config.instance().settings.Server.enable_vxlan_cross_compute:
|
||||
return False
|
||||
if self._vxlan_downgraded:
|
||||
return False
|
||||
if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2):
|
||||
return False
|
||||
for side in self._nodes:
|
||||
@@ -249,10 +281,17 @@ class UDPLink(Link):
|
||||
# pair is the only shape a bridge-to-bridge cable can take).
|
||||
switch = self._switch_endpoint()
|
||||
if switch is not None:
|
||||
if cross_compute:
|
||||
# The fast path is a one-host shape (the peer's anchor is
|
||||
# absorbed into the switch's bridge); a switch link across
|
||||
# computes stays on the relay.
|
||||
return False
|
||||
if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
|
||||
return True
|
||||
peer = node2 if switch[0] == 0 else node1
|
||||
return self._kernel_endpoint_ready(peer)
|
||||
if cross_compute:
|
||||
return self._vxlan_endpoint_ready(node1) and self._vxlan_endpoint_ready(node2)
|
||||
return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2)
|
||||
|
||||
def _switch_endpoint(self):
|
||||
@@ -305,6 +344,23 @@ class UDPLink(Link):
|
||||
return capabilities.get("ubridge_tap") is True and capabilities.get("vpcs_tap") is True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _vxlan_endpoint_ready(node):
|
||||
"""
|
||||
Whether a node can carry its end of a cross-compute VXLAN link: it
|
||||
must be able to anchor (the same per-node-type capability the
|
||||
same-host kernel datapath needs — its end of the tunnel is enslaved
|
||||
into a per-link bridge like any kernel link's) and its compute's
|
||||
uBridge must be able to create the VXLAN device the tunnel rides.
|
||||
Strict True on both, like the other capability gates: an unreported
|
||||
or failed probe keeps the link on the relay, where it always works.
|
||||
"""
|
||||
|
||||
if not UDPLink._kernel_endpoint_ready(node):
|
||||
return False
|
||||
capabilities = node.compute.capabilities or {}
|
||||
return capabilities.get("ubridge_vxlan") is True
|
||||
|
||||
async def _prepare(self):
|
||||
"""
|
||||
Local-only link setup: resolve peer addresses, reserve UDP ports and
|
||||
@@ -330,10 +386,77 @@ class UDPLink(Link):
|
||||
adapter_number2 = self._nodes[1]["adapter_number"]
|
||||
port_number2 = self._nodes[1]["port_number"]
|
||||
|
||||
if self._kernel_datapath_eligible(node1, node2):
|
||||
cross_compute = node1.compute.id != node2.compute.id
|
||||
# A cross-compute link needs the compute pair's addresses on both
|
||||
# datapaths (the VXLAN tunnel's outer endpoints, the relay's UDP
|
||||
# peers): resolve once, and let the no-common-subnet failure stay a
|
||||
# ControllerError either way — it is a deployment error, not a
|
||||
# datapath decision. Equal addresses mean both computes share one
|
||||
# host: the kernel shares one VXLAN socket per dstport, so a tunnel
|
||||
# pair cannot live there, and such links keep the relay, where the
|
||||
# two ends are told apart by their UDP ports.
|
||||
hosts = None
|
||||
if cross_compute:
|
||||
try:
|
||||
hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
|
||||
except ValueError as e:
|
||||
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
|
||||
same_host = hosts is not None and hosts[0] == hosts[1]
|
||||
|
||||
if self._kernel_datapath_eligible(node1, node2) and not same_host:
|
||||
node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True)
|
||||
node1_markers, node2_markers = self._get_node_markers(node1, node2)
|
||||
switch = self._switch_endpoint()
|
||||
if hosts is not None:
|
||||
# Cross-compute VXLAN: each end gets one VXLAN device
|
||||
# carrying the link's VNI, enslaved into the per-link kernel
|
||||
# bridge next to the node's anchor (see _kernel_attach on
|
||||
# the compute side). The controller derives name and VNI —
|
||||
# both ends must agree on the VNI and neither can ask the
|
||||
# other — and the compute resolves the underlay interface
|
||||
# locally from the peer address, so the kernel derives the
|
||||
# device MTU (underlay - 50) and the bridge reports the
|
||||
# honest "what this fabric can carry" value. A VNI the
|
||||
# kernel refuses as already registered on the shared
|
||||
# dstport socket surfaces as a commit failure; create()
|
||||
# retries with a bumped salt and degrades to the relay when
|
||||
# the retries run out.
|
||||
node1_host, node2_host = hosts
|
||||
dstport = Config.instance().settings.Server.vxlan_dstport
|
||||
vni = derive_vni(self._id, self._vxlan_salt)
|
||||
bridge_name = "gns3" + self._id.replace("-", "")[:11]
|
||||
self._link_data = [
|
||||
{
|
||||
"type": "nio_bridge",
|
||||
"bridge": bridge_name,
|
||||
"vxlan": {
|
||||
"name": vxlan_device_name(self._id, node1.id),
|
||||
"vni": vni,
|
||||
"remote": node2_host,
|
||||
"dstport": dstport,
|
||||
},
|
||||
"filters": node1_filters,
|
||||
"markers": node1_markers,
|
||||
"suspend": self._suspended,
|
||||
},
|
||||
{
|
||||
"type": "nio_bridge",
|
||||
"bridge": bridge_name,
|
||||
"vxlan": {
|
||||
"name": vxlan_device_name(self._id, node2.id),
|
||||
"vni": vni,
|
||||
"remote": node1_host,
|
||||
"dstport": dstport,
|
||||
},
|
||||
"filters": node2_filters,
|
||||
"markers": node2_markers,
|
||||
"suspend": self._suspended,
|
||||
},
|
||||
]
|
||||
return [
|
||||
(node1, adapter_number1, port_number1, self._link_data[0]),
|
||||
(node2, adapter_number2, port_number2, self._link_data[1]),
|
||||
]
|
||||
if switch is not None:
|
||||
if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
|
||||
# Switch-to-switch cascade: one veth pair joins the two
|
||||
@@ -449,11 +572,16 @@ class UDPLink(Link):
|
||||
(node2, adapter_number2, port_number2, self._link_data[1]),
|
||||
]
|
||||
|
||||
# Get an IP allowing communication between both host
|
||||
try:
|
||||
(node1_host, node2_host) = await node1.compute.get_ip_on_same_subnet(node2.compute)
|
||||
except ValueError as e:
|
||||
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
|
||||
# Relay datapath: UDP tunnels between the computes' addresses —
|
||||
# already resolved for a cross-compute pair, resolved here otherwise
|
||||
# (a same-compute relay pair resolves to the host's own address on
|
||||
# both sides, distinguished by their UDP ports).
|
||||
if hosts is None:
|
||||
try:
|
||||
hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
|
||||
except ValueError as e:
|
||||
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
|
||||
node1_host, node2_host = hosts
|
||||
|
||||
# Reserve a UDP port on both sides in parallel. Pre-allocated ports
|
||||
# (used during batch project loading) are popped from memory; otherwise
|
||||
@@ -537,17 +665,70 @@ class UDPLink(Link):
|
||||
raise errors[0]
|
||||
self._created = True
|
||||
|
||||
@staticmethod
|
||||
def _has_vxlan(entries):
|
||||
"""
|
||||
Whether any prepared NIO spec of this link carries a VXLAN tunnel.
|
||||
"""
|
||||
|
||||
return any(nio_data.get("vxlan") for _, _, _, nio_data in entries)
|
||||
|
||||
async def create(self):
|
||||
"""
|
||||
Create the link on the nodes (interactive path: prepare + commit).
|
||||
"""
|
||||
|
||||
entries = await self._prepare()
|
||||
await self._commit_nios(entries)
|
||||
if self._has_vxlan(entries):
|
||||
await self._commit_vxlan(entries)
|
||||
else:
|
||||
await self._commit_nios(entries)
|
||||
# New links automatically inherit every active project-level marker
|
||||
# definition so the user doesn't have to reconfigure.
|
||||
await self._project.apply_defs_to_new_link(self)
|
||||
|
||||
async def _commit_vxlan(self, entries):
|
||||
"""
|
||||
Commit a cross-compute VXLAN link, retrying with a bumped VNI salt
|
||||
on failure and degrading the link to the UDP relay when the retries
|
||||
run out.
|
||||
|
||||
The kernel refuses a VNI already registered on the shared dstport
|
||||
socket at device creation (EEXIST) — the one conflict a freshly
|
||||
created, deterministically named device can hit: the other end's
|
||||
device lives on another host, and a leftover under this end's name
|
||||
is replaced by the attach path, not refused. Any other commit
|
||||
failure (a stopped node, a missing underlay route) is retried too,
|
||||
harmlessly — it fails the same way every attempt, and the final
|
||||
relay attempt surfaces the real error if the link genuinely cannot
|
||||
be created. VXLAN is opt-in and the relay always works, so a tunnel
|
||||
that cannot be wired degrades instead of failing the link; the
|
||||
degradation lasts for this link object's lifetime (a project reopen
|
||||
retries VXLAN from a clean state).
|
||||
"""
|
||||
|
||||
last_error = None
|
||||
for attempt in range(_VXLAN_ATTEMPTS):
|
||||
if attempt:
|
||||
self._vxlan_salt += 1
|
||||
entries = await self._prepare()
|
||||
if not self._has_vxlan(entries):
|
||||
# Eligibility changed under us (a capability disappeared):
|
||||
# commit the relay entries as prepared.
|
||||
await self._commit_nios(entries)
|
||||
return
|
||||
try:
|
||||
await self._commit_nios(entries)
|
||||
return
|
||||
except (ComputeError, ControllerError) as e:
|
||||
last_error = e
|
||||
log.warning(
|
||||
"Link %s: VXLAN wiring failed (%s) — falling back to the uBridge relay datapath", self._id, last_error
|
||||
)
|
||||
self._vxlan_downgraded = True
|
||||
entries = await self._prepare()
|
||||
await self._commit_nios(entries)
|
||||
|
||||
async def update(self):
|
||||
"""
|
||||
Update the link on the nodes
|
||||
|
||||
@@ -73,6 +73,26 @@ class BridgeNIOType(str, Enum):
|
||||
bridge = "nio_bridge"
|
||||
|
||||
|
||||
class BridgeVxlan(BaseModel):
|
||||
"""
|
||||
VXLAN tunnel spec carried by a cross-compute BridgeNIO. Each endpoint
|
||||
creates one VXLAN device (uBridge ``vxlan create``) and enslaves it into
|
||||
the link's per-link kernel bridge next to the node's anchor; the kernel
|
||||
encapsulates/decapsulates the frames. The controller derives the name
|
||||
and VNI (utils.vxlan) and resolves the peer address, so both endpoints
|
||||
agree on the VNI without ever talking to each other.
|
||||
|
||||
The device's MTU is derived by the kernel from the underlay interface
|
||||
the route to ``remote`` picks (MTU - 50 for the outer headers), which is
|
||||
what the link's bridge then reports as "what this fabric can carry".
|
||||
"""
|
||||
|
||||
name: str = Field(..., max_length=15, description="VXLAN device name on this endpoint's host (IFNAMSIZ-1)")
|
||||
vni: int = Field(..., ge=1, le=16777215, description="VXLAN network identifier, identical on both endpoints")
|
||||
remote: str = Field(..., description="Peer compute's address (unicast tunnel endpoint)")
|
||||
dstport: int = Field(..., ge=1, le=65535, description="UDP destination port, identical on both endpoints")
|
||||
|
||||
|
||||
class BridgeNIO(BaseModel):
|
||||
"""
|
||||
Kernel-datapath bridge Network Input/Output properties. The NIO tells the
|
||||
@@ -99,6 +119,11 @@ class BridgeNIO(BaseModel):
|
||||
description="Kernel bridge name e.g. gns3a1b2c3d4e5; None when an Ethernet switch "
|
||||
"owns the anchor's bridge membership (absorbed link)",
|
||||
)
|
||||
vxlan: Optional[BridgeVxlan] = Field(
|
||||
None,
|
||||
description="Cross-compute tunnel: the VXLAN device this end creates and enslaves into the bridge "
|
||||
"next to the anchor; absent for same-host kernel links",
|
||||
)
|
||||
suspend: Optional[bool] = Field(None, description="Suspend the NIO")
|
||||
filters: Optional[dict] = Field(
|
||||
None,
|
||||
|
||||
@@ -192,6 +192,24 @@ class ServerSettings(BaseModel):
|
||||
"or QEMU when the compute's uBridge supports persistent TAPs) through kernel interfaces instead "
|
||||
"of the uBridge UDP relay",
|
||||
)
|
||||
enable_vxlan_cross_compute: bool = Field(
|
||||
False,
|
||||
description="Wire eligible cross-compute links through kernel VXLAN tunnels (one device per endpoint, "
|
||||
"uBridge's vxlan module) instead of the uBridge UDP relay. The kernel encapsulates/decapsulates, so "
|
||||
"no user-space forwarding is involved; the price is MTU: a tunnel can only carry the underlay MTU "
|
||||
"minus 50 bytes of outer headers, so enable this where the fabric between computes carries jumbo "
|
||||
"frames. The relay stays the default and the fallback when a tunnel cannot be wired",
|
||||
)
|
||||
vxlan_dstport: int = Field(
|
||||
14789,
|
||||
ge=1,
|
||||
le=65535,
|
||||
description="UDP destination port every cross-compute VXLAN tunnel listens on. The kernel shares one "
|
||||
"socket per port across all VXLAN devices, so all links of a deployment ride this one port — chosen "
|
||||
"off the IANA 4789 and the legacy 8472 (other VXLAN software's ports: sharing a socket would merge "
|
||||
"their demux table with ours) and above the ephemeral range. Change it only when it collides with "
|
||||
"a service already listening on the computes",
|
||||
)
|
||||
auto_discover_images: bool = Field(True, description="Automatically discover images in the images directory")
|
||||
image_sync_interval: int = Field(900, ge=10, description="Seconds between automatic image inventory scans")
|
||||
report_errors: bool = Field(True, description="Automatically send crash reports to the GNS3 team")
|
||||
|
||||
@@ -73,3 +73,8 @@ class Capabilities(BaseModel):
|
||||
"keeps the guest from auto-configuring the host side of the anchor TAP); absent when the "
|
||||
"probe failed",
|
||||
)
|
||||
ubridge_vxlan: Optional[bool] = Field(
|
||||
None,
|
||||
description="Whether uBridge can create VXLAN devices (the vxlan module, kernel support included) — "
|
||||
"the tunnel a cross-compute kernel link rides; absent when the probe failed",
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
#
|
||||
# Copyright (C) 2026 GNS3 Technologies Inc.
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Identity of a cross-compute VXLAN link: the device name and VNI the
|
||||
controller derives and ships in both endpoints' NIOs.
|
||||
|
||||
A cross-compute kernel link is carried by one VXLAN device per endpoint
|
||||
(uBridge's ``vxlan`` module, rtnetlink), enslaved into the link's per-link
|
||||
kernel bridge next to the node's anchor. Each device is local to its own
|
||||
host, but the VNI is the kernel's demux key for received frames on the
|
||||
deployment-wide shared dstport socket — both ends must agree on it while
|
||||
neither can ask the other, so the controller is the single source for both
|
||||
values.
|
||||
|
||||
* the device name is a pure function of the link id plus the endpoint's
|
||||
node id: unique per (link, endpoint), within IFNAMSIZ, and outside the
|
||||
``gns3``/anchor name spaces (``gvl`` prefix). Determinism is what makes
|
||||
a crashed run's leftover device recognisable — the compute deletes a
|
||||
device it finds under the name before re-creating it;
|
||||
* the VNI is a pure function of the link id and a controller-side salt —
|
||||
a random base per link object, bumped when a retry needs a fresh VNI —
|
||||
drawn from the upper half of the VNI space so it cannot collide with the
|
||||
low well-known VNIs other software uses (Flannel's 1, Docker overlay
|
||||
networks from 256, ...).
|
||||
|
||||
The kernel refuses a second device registering a VNI already taken on the
|
||||
same (dstport, socket) pair with EEXIST — a check that doubles as the
|
||||
collision detector: the controller retries with a bumped salt and falls
|
||||
back to the UDP relay when the space stays occupied (the relay needs no
|
||||
VNI at all).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
# The VNI range derived links draw from: at or above 0x100000 (1M) to stay
|
||||
# clear of the low VNIs other VXLAN software commonly registers, at or below
|
||||
# 0xFFFFFE so every derived value is valid for the module's 1..16777215
|
||||
# (0xFFFFFF) check with one value of headroom.
|
||||
VXLAN_VNI_MIN = 0x100000
|
||||
VXLAN_VNI_MAX = 0xFFFFFE
|
||||
|
||||
# IFNAMSIZ (16) - 1 for the terminator: "gvl" (3) + 8 (link) + 4 (node) = 15.
|
||||
VXLAN_NAME_PREFIX = "gvl"
|
||||
|
||||
|
||||
def derive_vni(link_id, salt=0):
|
||||
"""
|
||||
The VNI both endpoints of *link_id* tunnel through.
|
||||
|
||||
:param link_id: link UUID (or str)
|
||||
:param salt: controller-side retry counter — bumped when the kernel
|
||||
refuses the previous VNI as already registered on the shared
|
||||
dstport socket; 0 for the first attempt
|
||||
:returns: VNI in [VXLAN_VNI_MIN, VXLAN_VNI_MAX]
|
||||
"""
|
||||
|
||||
digest = hashlib.sha256(f"{link_id}:{salt}".encode()).digest()
|
||||
span = VXLAN_VNI_MAX - VXLAN_VNI_MIN + 1
|
||||
return VXLAN_VNI_MIN + int.from_bytes(digest[:8], "big") % span
|
||||
|
||||
|
||||
def vxlan_device_name(link_id, node_id):
|
||||
"""
|
||||
The VXLAN device name one endpoint of *link_id* creates on its own host,
|
||||
deterministic from the link id plus the endpoint's node id (the pair
|
||||
normally lives on two hosts, but a forced same-host pair must not name
|
||||
both its devices identically).
|
||||
"""
|
||||
|
||||
link = str(link_id).replace("-", "")[:8]
|
||||
node = str(node_id).replace("-", "")[:4]
|
||||
return f"{VXLAN_NAME_PREFIX}{link}{node}"
|
||||
Reference in new issue
Block a user