mirror of
https://github.com/GNS3/gns3-server.git
synced 2026-11-19 12:45:19 +02:00
feat: cross-compute VXLAN kernel datapath
Cross-compute links can ride kernel VXLAN tunnels instead of the uBridge
UDP relay: each endpoint creates one VXLAN device (uBridge's rtnetlink
vxlan module) enslaved into the same per-link Linux bridge the same-host
datapath uses, so filters, markers, capture and suspend keep running on
the anchor unchanged. Opt-in (Server.enable_vxlan_cross_compute, default
off) because the tunnel caps the usable MTU at the underlay minus the
outer headers; the relay stays the default and the fallback.
The controller derives what the pair must agree on: one VNI per link
(upper VNI space, per-link random salt) and deterministic device names
(gvl{link}{node}) that make crashed-run leftovers claimable
(show -> delete -> create). Same-host compute pairs, switch endpoints
and serial/unix-socket containers stay on the relay.
A VNI the kernel refuses self-heals on every path it can surface:
interactive create retries a fresh salt then degrades to the relay;
project open re-dispatches refused batch entries the same way; node
start recognises the refused device, stops the half-started node,
re-wires both ends on a fresh VNI (or degrades the link) and retries
the start. The device MTU is derived from the lower device (dev=) and
the per-link bridge inherits it - no configuration. The capability
probe (ubridge_vxlan in /capabilities) runs one scratch create/delete
on a scratch port that never equals the configured dstport.
The e2e scenario runs two computes on one machine (a network namespace
joined by a veth pair as the fabric) and needs root.
This commit is contained in:
24 files changed
+2107
-59
No files matched your search
@@ -20,18 +20,21 @@ Controller-side kernel-datapath link selection: eligibility of Docker-to-
|
||||
Docker links, NIO emission and the filters/markers guards.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from gns3server.config import Config
|
||||
from gns3server.controller.controller_error import ControllerError
|
||||
from gns3server.controller.compute import ComputeError
|
||||
from gns3server.controller.controller_error import ComputeConflictError, ControllerError
|
||||
from gns3server.controller.link import FILTERS
|
||||
from gns3server.controller.node import Node
|
||||
from gns3server.controller.ports.ethernet_port import EthernetPort
|
||||
from gns3server.controller.ports.serial_port import SerialPort
|
||||
from gns3server.controller.udp_link import UDPLink
|
||||
from gns3server.utils.kernel_anchor import kernel_anchor_name, kernel_cascade_names
|
||||
from gns3server.utils.vxlan import vxlan_device_name
|
||||
from tests.utils import AsyncioMagicMock
|
||||
|
||||
|
||||
@@ -1174,3 +1177,350 @@ async def test_node_started_repushes_both_cascade_ends(project):
|
||||
|
||||
sw1.put.assert_called_once_with("/adapters/0/ports/0/nio", data=entries[0][3], timeout=120)
|
||||
sw2.put.assert_called_once_with("/adapters/0/ports/0/nio", data=entries[1][3], timeout=120)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-compute VXLAN links
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _cross_compute_enabled():
|
||||
settings = Config.instance().settings.Server
|
||||
original = settings.enable_vxlan_cross_compute
|
||||
settings.enable_vxlan_cross_compute = True
|
||||
try:
|
||||
yield settings
|
||||
finally:
|
||||
settings.enable_vxlan_cross_compute = original
|
||||
|
||||
|
||||
def _vxlan_compute(compute_id, vxlan=True):
|
||||
compute = MagicMock()
|
||||
compute.id = compute_id
|
||||
compute.capabilities = {"ubridge_vxlan": vxlan}
|
||||
return compute
|
||||
|
||||
|
||||
async def _cross_compute_link(project, vxlan1=True, vxlan2=True, same_host=False):
|
||||
"""A docker↔docker link whose endpoints live on two computes, with the
|
||||
peer resolution (and the relay fallback's port reservation) mocked."""
|
||||
|
||||
compute1 = _vxlan_compute("compute-1", vxlan1)
|
||||
compute2 = _vxlan_compute("compute-2", vxlan2)
|
||||
|
||||
async def subnet_callback(other):
|
||||
return ("192.168.1.1", "192.168.1.1" if same_host else "192.168.1.2")
|
||||
|
||||
compute1.get_ip_on_same_subnet.side_effect = subnet_callback
|
||||
|
||||
async def post_callback(path, data=None, **kwargs):
|
||||
response = MagicMock()
|
||||
response.json = {"udp_port": 1024}
|
||||
return response
|
||||
|
||||
compute1.post.side_effect = post_callback
|
||||
compute2.post.side_effect = post_callback
|
||||
node1 = _node(project, compute1, "docker1")
|
||||
node2 = _node(project, compute2, "docker2")
|
||||
link = UDPLink(project)
|
||||
await link.add_node(node1, 0, 0, batch=True, dump=False)
|
||||
await link.add_node(node2, 0, 0, batch=True, dump=False)
|
||||
return link, node1, node2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vxlan_cross_compute_disabled_by_config(project):
|
||||
"""
|
||||
Cross-compute links stay on the relay until the deployment opts in —
|
||||
the VXLAN tunnel caps the usable MTU at the underlay minus the outer
|
||||
headers, so it is opt-in.
|
||||
"""
|
||||
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
assert link._kernel_datapath_eligible(node1, node2) is False
|
||||
|
||||
with _cross_compute_enabled():
|
||||
assert link._kernel_datapath_eligible(node1, node2) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vxlan_cross_compute_requires_the_vxlan_capability(project):
|
||||
"""
|
||||
Both computes' uBridge must be able to create the tunnel device; an
|
||||
unreported/failed probe (None) or an old build (False) keeps the link
|
||||
on the relay, exactly like the other capability gates.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
for vxlan1, vxlan2 in ((False, True), (True, False), (True, None), (None, None)):
|
||||
link, node1, node2 = await _cross_compute_link(project, vxlan1=vxlan1, vxlan2=vxlan2)
|
||||
assert link._kernel_datapath_eligible(node1, node2) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vxlan_cross_compute_switch_stays_on_relay(project):
|
||||
"""
|
||||
The switch fast path absorbs the peer's anchor into the switch's own
|
||||
bridge — a one-host shape; a switch link across computes keeps the
|
||||
relay.
|
||||
"""
|
||||
|
||||
switch = _node(project, _vxlan_compute("compute-1"), "switch1", node_type="ethernet_switch")
|
||||
peer = _node(project, _vxlan_compute("compute-2"), "docker2")
|
||||
link = await _link(project, switch, peer)
|
||||
with _cross_compute_enabled():
|
||||
assert link._kernel_datapath_eligible(switch, peer) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_vxlan_link_emits_tunnel_specs(project):
|
||||
"""
|
||||
Both ends get one nio_bridge sharing the per-link bridge name, each
|
||||
carrying the same VNI/dstport (the pair must agree, and neither end can
|
||||
ask the other) and the peer's address, with per-endpoint deterministic
|
||||
device names.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
entries = await link._prepare()
|
||||
|
||||
assert len(entries) == 2
|
||||
expected_bridge = "gns3" + link.id.replace("-", "")[:11]
|
||||
dstport = Config.instance().settings.Server.vxlan_dstport
|
||||
specs = [entry[3] for entry in entries]
|
||||
assert all(spec["type"] == "nio_bridge" for spec in specs)
|
||||
assert all(spec["bridge"] == expected_bridge for spec in specs)
|
||||
vxlan1, vxlan2 = specs[0]["vxlan"], specs[1]["vxlan"]
|
||||
assert vxlan1["vni"] == vxlan2["vni"]
|
||||
assert vxlan1["dstport"] == vxlan2["dstport"] == dstport
|
||||
assert vxlan1["remote"] == "192.168.1.2"
|
||||
assert vxlan2["remote"] == "192.168.1.1"
|
||||
assert vxlan1["name"] == vxlan_device_name(link.id, node1.id)
|
||||
assert vxlan2["name"] == vxlan_device_name(link.id, node2.id)
|
||||
assert vxlan1["name"] != vxlan2["name"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_vxlan_same_host_pair_falls_back_to_relay(project):
|
||||
"""
|
||||
Equal resolved addresses mean both computes share one host: the kernel
|
||||
shares one VXLAN socket per dstport, so a tunnel pair cannot live there
|
||||
— such links keep the relay, where the ends differ by UDP port.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _cross_compute_link(project, same_host=True)
|
||||
entries = await link._prepare()
|
||||
|
||||
assert all(entry[3]["type"] == "nio_udp" for entry in entries)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_commit_vxlan_retries_with_a_fresh_vni(project):
|
||||
"""
|
||||
A refused commit (reactively: the VNI was already registered on the
|
||||
host's shared socket) re-prepares the link with a bumped salt — a fresh
|
||||
VNI, identical on both ends — and recommits.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
vnis = []
|
||||
|
||||
async def commit(entries):
|
||||
vnis.append(entries[0][3]["vxlan"]["vni"])
|
||||
if len(vnis) == 1:
|
||||
raise ComputeError("Could not create vxlan gvl...: File exists")
|
||||
|
||||
with patch.object(link, "_commit_nios", commit):
|
||||
await link.create()
|
||||
|
||||
assert len(vnis) == 2
|
||||
assert vnis[0] != vnis[1]
|
||||
assert link._vxlan_downgraded is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_commit_vxlan_degrades_to_relay_after_retries(project):
|
||||
"""
|
||||
When every VXLAN attempt fails, the link degrades to the UDP relay —
|
||||
the opt-in datapath must never make a link uncreatable.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
commits = []
|
||||
|
||||
async def commit(entries):
|
||||
commits.append(entries)
|
||||
if any(entry[3].get("vxlan") for entry in entries):
|
||||
raise ComputeError("Could not create vxlan gvl...: File exists")
|
||||
|
||||
with patch.object(link, "_commit_nios", commit):
|
||||
await link.create()
|
||||
|
||||
assert link._vxlan_downgraded is True
|
||||
assert len(commits) == 4 # the three VXLAN attempts, then the relay
|
||||
assert all(entry[3].get("vxlan") for entry in commits[0])
|
||||
assert all(entry[3]["type"] == "nio_udp" for entry in commits[-1])
|
||||
|
||||
|
||||
async def _prepared_vxlan_link(project):
|
||||
"""A cross-compute link with its tunnel NIOs prepared (the shape a node
|
||||
start replays) — call inside ``_cross_compute_enabled`` so the heal's
|
||||
re-preparation stays on the tunnel too. _bind_members mirrors the
|
||||
project-open finalize (the node's link back-reference is what lets the
|
||||
heal find the conflicted link)."""
|
||||
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
await link._prepare()
|
||||
link._bind_members()
|
||||
link._created = True
|
||||
return link, node1, node2
|
||||
|
||||
|
||||
def _vni_conflict(node, link):
|
||||
"""The compute's 409 when the kernel refuses the link's VNI on the
|
||||
shared dstport socket — the wording KernelDatapathMixin emits."""
|
||||
|
||||
device = vxlan_device_name(link.id, node.id)
|
||||
return ComputeConflictError(
|
||||
"http://compute", {"message": f"Could not create VXLAN device {device} to 192.168.1.2 via eth0: File exists"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_node_start_self_heals_a_vni_conflict(project):
|
||||
"""
|
||||
A link wired while its nodes were stopped hits its VNI collision at
|
||||
node start (the kernel's refusal is the first time anyone asks). The
|
||||
start stops the half-started node, re-wires the link on a fresh VNI —
|
||||
both ends, through the per-compute batch — and retries.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _prepared_vxlan_link(project)
|
||||
salt = link._vxlan_salt
|
||||
starts = []
|
||||
|
||||
async def post_callback(path, data=None, **kwargs):
|
||||
if path.endswith("/start"):
|
||||
starts.append(path)
|
||||
if len(starts) == 1:
|
||||
raise _vni_conflict(node1, link)
|
||||
response = MagicMock()
|
||||
response.json = {"udp_port": 1024}
|
||||
return response
|
||||
|
||||
node1.compute.post.side_effect = post_callback
|
||||
await node1.start()
|
||||
|
||||
assert len(starts) == 2
|
||||
assert link._vxlan_downgraded is False
|
||||
assert link._vxlan_salt == salt + 1
|
||||
# the half-started node was stopped before the retry (a start on a
|
||||
# running container early-returns without wiring)
|
||||
assert any(call.args[0].endswith("/stop") for call in node1.compute.post.call_args_list)
|
||||
# the fresh-VNI NIOs were re-dispatched to both computes
|
||||
for compute in (node1.compute, node2.compute):
|
||||
batches = [c for c in compute.post.call_args_list if str(c.args[0]).endswith("/nios/batch")]
|
||||
assert len(batches) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_node_start_degrades_the_link_after_a_second_conflict(project):
|
||||
"""
|
||||
A conflict that survives the fresh VNI degrades the link to the relay
|
||||
— the start must succeed no later than that round, the same guarantee
|
||||
create() and the project-open batch path give.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _prepared_vxlan_link(project)
|
||||
starts = []
|
||||
|
||||
async def post_callback(path, data=None, **kwargs):
|
||||
if path.endswith("/start"):
|
||||
starts.append(path)
|
||||
if len(starts) <= 2:
|
||||
raise _vni_conflict(node1, link)
|
||||
response = MagicMock()
|
||||
response.json = {"udp_port": 1024}
|
||||
return response
|
||||
|
||||
node1.compute.post.side_effect = post_callback
|
||||
await node1.start()
|
||||
|
||||
assert len(starts) == 3
|
||||
assert link._vxlan_downgraded is True
|
||||
assert all(nio["type"] == "nio_udp" for nio in link._link_data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_node_start_does_not_retry_other_failures(project):
|
||||
"""
|
||||
Only the VXLAN device conflict heals: any other start failure surfaces
|
||||
unchanged, with a single attempt and no stop/redispatch side effects.
|
||||
"""
|
||||
|
||||
with _cross_compute_enabled():
|
||||
link, node1, node2 = await _prepared_vxlan_link(project)
|
||||
salt = link._vxlan_salt
|
||||
|
||||
async def post_callback(path, data=None, **kwargs):
|
||||
if path.endswith("/start"):
|
||||
raise ComputeError("DockerError: image not found")
|
||||
response = MagicMock()
|
||||
response.json = {}
|
||||
return response
|
||||
|
||||
node1.compute.post.side_effect = post_callback
|
||||
with pytest.raises(ComputeError):
|
||||
await node1.start()
|
||||
|
||||
starts = [c for c in node1.compute.post.call_args_list if str(c.args[0]).endswith("/start")]
|
||||
assert len(starts) == 1
|
||||
assert not any(call.args[0].endswith("/stop") for call in node1.compute.post.call_args_list)
|
||||
assert link._vxlan_salt == salt
|
||||
assert link._vxlan_downgraded is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_heal_tries_the_full_vni_candidate_count(project):
|
||||
"""
|
||||
The project-open heal gives a refused link the same VNI candidate count
|
||||
an interactive creation gets (fresh-salt rounds), degrading to the
|
||||
relay only after they run out.
|
||||
"""
|
||||
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
rounds = []
|
||||
|
||||
async def redispatch(links, downgrade):
|
||||
rounds.append(downgrade)
|
||||
return links if not downgrade else []
|
||||
|
||||
with patch.object(project, "_redispatch_vxlan_links", redispatch):
|
||||
await project._heal_refused_vxlan_links([link])
|
||||
assert rounds == [False, False, True]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_heal_stops_at_the_first_successful_vni(project):
|
||||
"""
|
||||
A link healed by the first fresh-VNI round needs no further rounds —
|
||||
no pointless second redispatch, no relay degradation.
|
||||
"""
|
||||
|
||||
link, node1, node2 = await _cross_compute_link(project)
|
||||
rounds = []
|
||||
|
||||
async def redispatch(links, downgrade):
|
||||
rounds.append(downgrade)
|
||||
return []
|
||||
|
||||
with patch.object(project, "_redispatch_vxlan_links", redispatch):
|
||||
await project._heal_refused_vxlan_links([link])
|
||||
assert rounds == [False]
|
||||
Reference in new issue
Block a user