feat: cross-compute VXLAN kernel datapath

Cross-compute links can ride kernel VXLAN tunnels instead of the uBridge
UDP relay: each endpoint creates one VXLAN device (uBridge's rtnetlink
vxlan module) enslaved into the same per-link Linux bridge the same-host
datapath uses, so filters, markers, capture and suspend keep running on
the anchor unchanged. Opt-in (Server.enable_vxlan_cross_compute, default
off) because the tunnel caps the usable MTU at the underlay minus the
outer headers; the relay stays the default and the fallback.

The controller derives what the pair must agree on: one VNI per link
(upper VNI space, per-link random salt) and deterministic device names
(gvl{link}{node}) that make crashed-run leftovers claimable
(show -> delete -> create). Same-host compute pairs, switch endpoints
and serial/unix-socket containers stay on the relay.

A VNI the kernel refuses self-heals on every path it can surface:
interactive create retries a fresh salt then degrades to the relay;
project open re-dispatches refused batch entries the same way; node
start recognises the refused device, stops the half-started node,
re-wires both ends on a fresh VNI (or degrades the link) and retries
the start. The device MTU is derived from the lower device (dev=) and
the per-link bridge inherits it - no configuration. The capability
probe (ubridge_vxlan in /capabilities) runs one scratch create/delete
on a scratch port that never equals the configured dstport.

The e2e scenario runs two computes on one machine (a network namespace
joined by a veth pair as the fabric) and needs root.
This commit is contained in:
YueGuobin committed 2026-10-17 14:11:00 +08:00
1 parent 5514b2e929
commit 21b866aeed
24 files changed
+2107 -59

No files matched your search

+102 -9
View File
@@ -49,7 +49,7 @@ from .link import _UNSET
from .node import Node
from .snapshot import Snapshot
from .topology import load_topology, project_to_topology
from .udp_link import UDPLink
from .udp_link import _VXLAN_ATTEMPTS, UDPLink
log = logging.getLogger(__name__)
@@ -1021,6 +1021,84 @@ class Project:
entries = await link._prepare()
return (link, entries)
async def _dispatch_nio_batch(self, compute, nio_entries):
"""
POST one compute's NIO batch for this project, returning the entries
it refused (VXLAN entries a compute could not wire — see the batch
route; other NIO types fail the whole request).
"""
response = await compute.post(
f"/projects/{self._id}/nios/batch",
data={"nios": nio_entries},
timeout=300,
)
return response.json.get("failed") or []
async def _redispatch_vxlan_links(self, links, downgrade):
"""
Re-prepare *links* and dispatch their NIOs in a second per-compute
batch, returning the links the computes still refused.
With ``downgrade=False`` each link's VNI salt is bumped first — a
fresh VNI, since the refused one was already registered on the
host's shared dstport socket. With ``downgrade=True`` the links are
marked degraded to the uBridge relay (whose UDP NIOs cannot fail
this way) before re-preparing.
"""
for link in links:
if downgrade:
log.warning(
"Project '%s' [%s]: VXLAN wiring failed for link %s — falling back to the uBridge relay datapath",
self._name,
self._id,
link.id,
)
link._vxlan_downgraded = True
else:
link._vxlan_salt += 1
per_compute = {}
by_key = {}
for link in links:
for node, adapter_number, port_number, nio_data in await link._prepare():
by_key[(str(node.id), adapter_number, port_number)] = link
per_compute.setdefault(node.compute, []).append(
{
"node_id": node.id,
"adapter_number": adapter_number,
"port_number": port_number,
"nio": nio_data,
}
)
if not per_compute:
return []
results = await asyncio.gather(
*[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
)
failed = []
for item in (i for sub in results for i in sub):
link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
if link is not None and link not in failed:
failed.append(link)
return failed
async def _heal_refused_vxlan_links(self, pending):
"""
Drive the links a compute refused at open onto a working datapath:
fresh-VNI rounds first — the same candidate count an interactive
link creation tries — then, for whatever still fails, the
degradation to the uBridge relay (whose UDP NIOs cannot hit this
refusal).
"""
for _ in range(_VXLAN_ATTEMPTS - 1):
if not pending:
return
pending = await self._redispatch_vxlan_links(pending, downgrade=False)
if pending:
await self._redispatch_vxlan_links(pending, downgrade=True)
async def restore_deferred_links(self, node):
"""
Create on the computes the NIOs of the links that were deferred while
@@ -1994,15 +2072,30 @@ class Project:
}
)
async def _dispatch_batch(compute, nio_entries):
await compute.post(
f"/projects/{self._id}/nios/batch",
data={"nios": nio_entries},
timeout=300,
)
if per_compute:
await asyncio.gather(*[_dispatch_batch(c, n) for c, n in per_compute.items()])
results = await asyncio.gather(
*[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
)
failed = [item for sub in results for item in sub]
else:
failed = []
# A VXLAN entry a compute refused (its VNI already registered on
# the host's shared socket, no route to the peer) must not fail
# the whole open: the refused links are healed onto a working
# datapath (fresh VNIs, then the relay) — see
# _heal_refused_vxlan_links.
if failed:
by_key = {}
for link, entries in valid:
for node, adapter_number, port_number, nio_data in entries:
if nio_data.get("vxlan"):
by_key[(str(node.id), adapter_number, port_number)] = link
pending = []
for item in failed:
link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
if link is not None and link not in pending:
pending.append(link)
await self._heal_refused_vxlan_links(pending)
# Finalise every link: wire node/port back-references, mark created,
# notify clients, and apply project-level marker definitions.