Merge branch '3.1' into 3.1

gns3server/controller/export_project.py

@@ -207,6 +207,7 @@ async def _patch_project_file(
 
     # image files that we need to include in the exported archive
     images = []
+    portable_references = []
 
     try:
         with open(path) as f:
@@ -248,32 +249,54 @@ async def _patch_project_file(
                         if value is None or value.strip() == "":
                             continue
 
-                        if not keep_compute_ids:  # If we keep the original compute we can keep the image path
+                        if not keep_compute_ids:
                             node["properties"][prop] = os.path.basename(value)
 
-                        if include_images is True:
-                            images.append({"compute_id": compute_id, "image": value, "image_type": node["node_type"]})
+                        if include_images:
+                            image = {"compute_id": compute_id, "image": value, "image_type": node["node_type"]}
+                            images.append(image)
+                            if not keep_compute_ids:
+                                portable_references.append((node["properties"], prop, image))
 
         if not keep_compute_ids:
             topology["topology"][
                 "computes"
             ] = []  # Strip compute information because could contain secret info like password
 
-    local_images = {i["image"] for i in images if i["compute_id"] == "local"}
+    destinations = _image_export_names(images)
+    for properties, prop, image in portable_references:
+        properties[prop] = destinations[(image["compute_id"], image["image_type"], image["image"])]
 
-    for image in local_images:
-        _export_local_image(image, zstream)
-
-    remote_images = {(i["compute_id"], i["image_type"], i["image"]) for i in images if i["compute_id"] != "local"}
-
-    for compute_id, image_type, image in remote_images:
-        await _export_remote_images(project, compute_id, image_type, image, zstream, temporary_dir)
+    for (compute_id, image_type, image), export_name in destinations.items():
+        if compute_id == "local":
+            _export_local_image(image, zstream, export_name, image_type)
+        else:
+            await _export_remote_images(project, compute_id, image_type, image, zstream, temporary_dir, export_name)
 
     zstream.writestr("project.gns3", json.dumps(topology, indent=4, sort_keys=True).encode())
     return images
 
 
-def _export_local_image(image, zstream):
+def _image_export_names(images):
+    """Allocate distinct portable names without changing ordinary image names."""
+    destinations = {}
+    used = set()
+    for image in images:
+        key = (image["compute_id"], image["image_type"], image["image"])
+        if key in destinations:
+            continue
+        filename = os.path.basename(image["image"])
+        stem, extension = os.path.splitext(filename)
+        index = 1
+        while (image["image_type"], filename) in used:
+            index += 1
+            filename = f"{stem}-{index}{extension}"
+        used.add((image["image_type"], filename))
+        destinations[key] = filename
+    return destinations
+
+
+def _export_local_image(image, zstream, export_name=None, image_type=None):
     """
     Exports a local image to the zip file.
 
@@ -284,6 +307,8 @@ def _export_local_image(image, zstream):
     from ..compute import MODULES
 
     for module in MODULES:
+        if image_type and getattr(module, "_NODE_TYPE", None) != image_type:
+            continue
         try:
             images_directory = module.instance().get_images_directory()
         except NotImplementedError:
@@ -297,13 +322,15 @@ def _export_local_image(image, zstream):
             path = os.path.join(images_directory, image)
 
         if os.path.exists(path):
-            arcname = os.path.join("images", directory, os.path.basename(image))
+            arcname = os.path.join("images", directory, export_name or os.path.basename(image))
             _patch_mtime(path)
             zstream.write(path, arcname)
             return
 
 
-async def _export_remote_images(project, compute_id, image_type, image, project_zipfile, temporary_dir):
+async def _export_remote_images(
+    project, compute_id, image_type, image, project_zipfile, temporary_dir, export_name=None
+):
     """
     Export specific image from remote compute.
     """
@@ -333,5 +360,6 @@ async def _export_remote_images(project, compute_id, image_type, image, project_
                 break
             await f.write(data)
     response.close()
-    arcname = os.path.join("images", image_type, image)
+    directory = "QEMU" if image_type == "qemu" else image_type
+    arcname = os.path.join("images", directory, export_name or image)
     project_zipfile.write(temp_path, arcname=arcname, compress_type=zipfile.ZIP_DEFLATED)

gns3server/controller/marker_replay.py

@@ -51,9 +51,17 @@ merge sorts by ``(ts, source file, frame number)`` and index structures must
 never use ts alone as a dict key, or same-microsecond frames silently
 overwrite each other. The canonical ts strings travel to clients verbatim
 and must be round-tripped verbatim.
+
+Capture files live on the compute hosting the capture node — only a local
+marker's pcap sits in the controller's project directory. A remote marker's
+pcap is fetched from its compute over HTTP on every acquire into a bounded
+cache (paused marker pcaps hold matched packets only, so they stay small);
+the previous copy is kept when the bytes are identical so resident sharkd
+sessions (validated by ``(mtime, size)``) survive unchanged refetches.
 """
 
 import asyncio
+import filecmp
 import json
 import logging
 import os
@@ -63,7 +71,9 @@ import tempfile
 import time
 from contextlib import asynccontextmanager
 
-from .controller_error import ControllerBadRequestError, ControllerError, ControllerNotFoundError
+import aiofiles
+
+from .controller_error import ComputeError, ControllerBadRequestError, ControllerError, ControllerNotFoundError
 
 log = logging.getLogger(__name__)
 
@@ -202,6 +212,173 @@ def read_frame_bytes(path, frame_number):
         return f.read(incl_len).hex()
 
 
+# ---------------------------------------------------------------------------
+# Remote capture sources (fetched from their compute over HTTP)
+# ---------------------------------------------------------------------------
+
+# Bounded number of remote marker pcaps kept on the controller. Only idle
+# entries are ever evicted (see _remote_pcap), so a request's own sources
+# cannot be pulled out from under it.
+_REMOTE_CACHE_MAX = 32
+
+# (project_id, compute_id, rel_path) -> {"path": str, "last_used": float, "_uses": int}
+_remote_cache: dict[tuple[str, str, str], dict] = {}
+_remote_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
+_remote_cache_dir: str | None = None
+
+
+def _cache_path(project, compute, rel_path):
+    """Flat, collision-free cache path: project/compute ids are uuids and
+    basename() strips any path component out of the file name."""
+
+    global _remote_cache_dir
+    if _remote_cache_dir is None:
+        _remote_cache_dir = tempfile.mkdtemp(prefix="gns3-marker-replay-")
+    os.makedirs(_remote_cache_dir, exist_ok=True)
+    return os.path.join(_remote_cache_dir, f"{project.id}_{compute.id}_{os.path.basename(rel_path)}")
+
+
+def _drop_cached_file(key):
+    """Forget a cache entry and delete its file (marker removed on the compute)."""
+
+    entry = _remote_cache.pop(key, None)
+    if entry and entry["path"]:
+        try:
+            os.remove(entry["path"])
+        except OSError:
+            pass
+
+
+def _evict_idle_remote_cache():
+    """Trim the cache down to the cap, least-recently-used first. Only idle
+    entries are evictable; dict access is synchronous, so this cannot
+    interleave with a concurrent acquire's refcount bump."""
+
+    while len(_remote_cache) > _REMOTE_CACHE_MAX:
+        idle = [key for key, entry in _remote_cache.items() if entry["_uses"] == 0]
+        if not idle:
+            break
+        victim = min(idle, key=lambda key: _remote_cache[key]["last_used"])
+        entry = _remote_cache.pop(victim)
+        if entry["path"]:
+            try:
+                os.remove(entry["path"])
+            except OSError:
+                pass
+
+
+async def _fetch_to_cache(project, compute, rel_path, path):
+    """
+    Fetch the pcap from its compute, replacing the cached copy only when the
+    bytes changed — an unchanged copy keeps its mtime, so the resident sharkd
+    session keyed by that path stays valid. The file is refetched on every
+    acquire: the gate already pins replay to paused markers, and a marker
+    resumed and re-paused between two requests rebuilds its pcap.
+
+    :returns: the cache path, or None when the compute has no such file (the
+        marker never matched anything — not an error).
+    :raises ComputeError: the compute could not be reached or refused the file.
+    """
+
+    try:
+        response = await compute.http_query("GET", f"/projects/{project.id}/files/{rel_path}", stream=True)
+    except ControllerNotFoundError:
+        return None
+    except ControllerError as e:
+        raise ComputeError(f"Could not fetch capture file from compute '{compute.id}': {e}") from e
+
+    tmp = f"{path}.tmp"
+    try:
+        async with aiofiles.open(tmp, "wb") as f:
+            async for chunk in response.content.iter_any():
+                if chunk:
+                    await f.write(chunk)
+        await response.release()
+        if os.path.exists(path) and filecmp.cmp(path, tmp, shallow=False):
+            os.remove(tmp)  # unchanged: keep the old copy, mtime intact
+        else:
+            os.replace(tmp, path)
+    finally:
+        if os.path.exists(tmp):
+            try:
+                os.remove(tmp)
+            except OSError:
+                pass
+    return path
+
+
+@asynccontextmanager
+async def _remote_pcap(project, compute, rel_path):
+    """
+    A controller-local copy of a capture file living on a remote compute.
+    Acquires of the same file are serialized per key; the entry is
+    refcounted while yielded so eviction only ever removes idle files.
+
+    Yields the local path, or None when the compute has no such file.
+    """
+
+    key = (project.id, compute.id, rel_path)
+    lock = _remote_locks.setdefault(key, asyncio.Lock())
+    async with lock:
+        entry = _remote_cache.get(key)
+        if entry is None:
+            entry = {"path": _cache_path(project, compute, rel_path), "last_used": 0.0, "_uses": 0}
+            _remote_cache[key] = entry
+        entry["_uses"] += 1
+        try:
+            path = await _fetch_to_cache(project, compute, rel_path, entry["path"])
+            entry["last_used"] = time.monotonic()
+            if path is None:
+                _drop_cached_file(key)
+            yield path
+        finally:
+            entry["_uses"] -= 1
+    _evict_idle_remote_cache()
+
+
+def _capture_node_compute(project, node_id):
+    """
+    The compute hosting a marker's capture node, or None when the capture is
+    local — or the node is gone, in which case only the controller's own
+    markers directory can hold a copy and the caller has already checked it.
+    """
+
+    get_node = getattr(project, "get_node", None)
+    if get_node is None:
+        return None
+    try:
+        compute = get_node(node_id).compute
+    except ControllerError:
+        return None
+    if compute is None or compute.id == "local":
+        return None
+    return compute
+
+
+@asynccontextmanager
+async def _source_pcap(project, node_id, link_id, marker):
+    """
+    Resolve a marker's capture file to a controller-local path.
+
+    Yields None when no capture exists (no packet ever matched on that
+    marker). Local markers resolve straight to the project's markers
+    directory; a marker whose capture node runs on another compute is
+    fetched from that compute (see _remote_pcap).
+    """
+
+    filename = f"{node_id}_{link_id}_{marker}.pcap"
+    path = os.path.join(project.markers_directory, filename)
+    if os.path.exists(path):
+        yield path
+        return
+    compute = _capture_node_compute(project, node_id)
+    if compute is None:
+        yield None
+        return
+    async with _remote_pcap(project, compute, f"project-files/markers/{filename}") as fetched:
+        yield fetched
+
+
 # ---------------------------------------------------------------------------
 # sharkd: process environment and scratch copies
 # ---------------------------------------------------------------------------
@@ -526,13 +703,19 @@ def _get_manager():
 
 async def close_sessions():
     """Server shutdown hook: kill every resident session and drop its scratch
-    copy, so nothing leaks into /tmp across restarts."""
+    copy, and remove the remote-capture cache, so nothing leaks into /tmp
+    across restarts."""
 
     global _manager
+    global _remote_cache_dir
     manager = _manager
     if manager is not None:
         await manager.close_all()
     _manager = None
+    if _remote_cache_dir is not None:
+        shutil.rmtree(_remote_cache_dir, ignore_errors=True)
+        _remote_cache_dir = None
+    _remote_cache.clear()
 
 
 # ---------------------------------------------------------------------------
@@ -655,45 +838,68 @@ async def _merged_frames(project, entries, filter_expr=None, link_id=None):
     ``sources`` is the stable inventory of the tag: EVERY capture source is
     listed with engine-free total counts, unaffected by ``link_id`` /
     ``filter_expr`` — the inventory must not shrink when the view narrows.
+    A capture file that cannot be fetched from its compute keeps its source
+    entry, with ``count: 0`` and an ``error`` message, instead of silently
+    vanishing from the timeline.
     """
 
-    markers_dir = project.markers_directory
     merged = []
     sources = []
     for entry in entries:
-        pcap = os.path.join(markers_dir, f"{entry['node_id']}_{entry['link_id']}_{entry['marker']}.pcap")
-        frames = scan_pcap_frames(pcap) if os.path.exists(pcap) else []
-        # Inventory first: every source, engine-free totals.
-        sources.append(
-            {**{k: entry[k] for k in ("node_id", "link_id", "marker", "data_link_type")}, "count": len(frames)}
-        )
-        if link_id and entry["link_id"] != link_id:
-            continue  # link narrows the stream before any engine work
-        if frames:
-            columns = await _columns_for(pcap, filter_expr)
-        else:
-            columns = {}
-        source_key = f"{entry['node_id']}_{entry['link_id']}_{entry['marker']}"
-        for frame_number, (sec, usec, incl_len) in enumerate(frames, start=1):
-            if filter_expr is not None and frame_number not in columns:
-                continue
-            cols = columns.get(frame_number, {})
-            merged.append(
+        try:
+            async with _source_pcap(project, entry["node_id"], entry["link_id"], entry["marker"]) as pcap:
+                frames = scan_pcap_frames(pcap) if pcap else []
+                # Inventory first: every source, engine-free totals.
+                sources.append(
+                    {
+                        **{k: entry[k] for k in ("node_id", "link_id", "marker", "data_link_type")},
+                        "count": len(frames),
+                    }
+                )
+                if link_id and entry["link_id"] != link_id:
+                    continue  # link narrows the stream before any engine work
+                if frames:
+                    columns = await _columns_for(pcap, filter_expr)
+                else:
+                    columns = {}
+                source_key = f"{entry['node_id']}_{entry['link_id']}_{entry['marker']}"
+                for frame_number, (sec, usec, incl_len) in enumerate(frames, start=1):
+                    if filter_expr is not None and frame_number not in columns:
+                        continue
+                    cols = columns.get(frame_number, {})
+                    merged.append(
+                        {
+                            "ts": _format_ts(sec, usec),
+                            "ts_us": sec * 1_000_000 + usec,
+                            "_source": source_key,
+                            "len": incl_len,
+                            "node_id": entry["node_id"],
+                            "link_id": entry["link_id"],
+                            "marker": entry["marker"],
+                            "frame_number": frame_number,
+                            "src": cols.get("src"),
+                            "dst": cols.get("dst"),
+                            "proto": cols.get("proto"),
+                            "info": cols.get("info"),
+                            "bg": cols.get("bg"),
+                            "fg": cols.get("fg"),
+                        }
+                    )
+        except ComputeError as e:
+            # The capture file lives on a compute that could not be reached:
+            # keep the source in the inventory, flagged, so a partial timeline
+            # is never mistaken for the whole capture.
+            log.warning(
+                "Capture file of marker '%s' on link %s could not be fetched: %s",
+                entry["marker"],
+                entry["link_id"],
+                e,
+            )
+            sources.append(
                 {
-                    "ts": _format_ts(sec, usec),
-                    "ts_us": sec * 1_000_000 + usec,
-                    "_source": source_key,
-                    "len": incl_len,
-                    "node_id": entry["node_id"],
-                    "link_id": entry["link_id"],
-                    "marker": entry["marker"],
-                    "frame_number": frame_number,
-                    "src": cols.get("src"),
-                    "dst": cols.get("dst"),
-                    "proto": cols.get("proto"),
-                    "info": cols.get("info"),
-                    "bg": cols.get("bg"),
-                    "fg": cols.get("fg"),
+                    **{k: entry[k] for k in ("node_id", "link_id", "marker", "data_link_type")},
+                    "count": 0,
+                    "error": str(e),
                 }
             )
     merged.sort(key=lambda f: (f["ts_us"], f["_source"], f["frame_number"]))
@@ -761,7 +967,10 @@ async def decode_frame(project, tag, ts, node_id, link_id, marker, frame_number=
     round-tripped ts still matches the file (guards a rebuild between the
     timeline view and this click), read the raw bytes for the hex view
     straight from the pcap, and rename the sharkd protocol tree into the
-    REST contract (closed key set, values untouched).
+    REST contract (closed key set, values untouched). A marker captured on
+    another compute decodes from its fetched cache copy; a compute that
+    cannot be reached raises ComputeError — distinct from the 404s below,
+    which mean the marker never matched anything or the capture was rebuilt.
 
     ``ts`` is not unique within one pcap (same-microsecond frames are kept
     deliberately); an explicit ``frame_number`` from the frame list entry
@@ -778,40 +987,40 @@ async def decode_frame(project, tag, ts, node_id, link_id, marker, frame_number=
     if entry is None:
         raise ControllerNotFoundError(f"No marker '{marker}' with tag {tag} on link {link_id} captured by {node_id}")
 
-    pcap = os.path.join(project.markers_directory, f"{node_id}_{link_id}_{marker}.pcap")
-    if not os.path.exists(pcap):
-        raise ControllerNotFoundError(f"No capture file for marker '{marker}' (nothing ever matched)")
+    async with _source_pcap(project, node_id, link_id, marker) as pcap:
+        if pcap is None:
+            raise ControllerNotFoundError(f"No capture file for marker '{marker}' (nothing ever matched)")
 
-    frames = scan_pcap_frames(pcap)
-    rebuilt_message = f"No frame at ts {ts} in marker '{marker}' (the capture may have been rebuilt)"
-    if frame_number is None:
-        # The ts must be the exact string the timeline returned; find the frame
-        # it identifies rather than trusting any position hint from the client.
-        frame_number = next(
-            (i for i, (sec, usec, _len) in enumerate(frames, start=1) if _format_ts(sec, usec) == ts),
-            None,
-        )
+        frames = scan_pcap_frames(pcap)
+        rebuilt_message = f"No frame at ts {ts} in marker '{marker}' (the capture may have been rebuilt)"
         if frame_number is None:
-            raise ControllerNotFoundError(rebuilt_message)
-    else:
-        if not 1 <= frame_number <= len(frames):
-            raise ControllerNotFoundError(rebuilt_message)
-        sec, usec, _len = frames[frame_number - 1]
-        if _format_ts(sec, usec) != ts:
+            # The ts must be the exact string the timeline returned; find the frame
+            # it identifies rather than trusting any position hint from the client.
+            frame_number = next(
+                (i for i, (sec, usec, _len) in enumerate(frames, start=1) if _format_ts(sec, usec) == ts),
+                None,
+            )
+            if frame_number is None:
+                raise ControllerNotFoundError(rebuilt_message)
+        else:
+            if not 1 <= frame_number <= len(frames):
+                raise ControllerNotFoundError(rebuilt_message)
+            sec, usec, _len = frames[frame_number - 1]
+            if _format_ts(sec, usec) != ts:
+                raise ControllerNotFoundError(rebuilt_message)
+
+        raw_hex = read_frame_bytes(pcap, frame_number)
+        if raw_hex is None:
+            # The header scan listed it but the bytes are gone — mid-write tail.
             raise ControllerNotFoundError(rebuilt_message)
 
-    raw_hex = read_frame_bytes(pcap, frame_number)
-    if raw_hex is None:
-        # The header scan listed it but the bytes are gone — mid-write tail.
-        raise ControllerNotFoundError(rebuilt_message)
-
-    async with _get_manager().session(pcap) as session:
-        try:
-            result = await session.rpc("frame", {"frame": frame_number, "proto": True})
-        except _SharkdRpcError as e:
-            # The frame range was validated against the file above, so an
-            # engine error here is a real fault (502), not a client 404.
-            raise SharkdError(f"sharkd frame failed: {e.message}")
+        async with _get_manager().session(pcap) as session:
+            try:
+                result = await session.rpc("frame", {"frame": frame_number, "proto": True})
+            except _SharkdRpcError as e:
+                # The frame range was validated against the file above, so an
+                # engine error here is a real fault (502), not a client 404.
+                raise SharkdError(f"sharkd frame failed: {e.message}")
 
     tree = _rename_value(result.get("tree", []))
     return {

gns3server/controller/node.py

@@ -20,18 +20,33 @@ import copy
 import html
 import logging
 import os
+import re
 import uuid
 
 from ..config import Config
 from ..utils import int_to_macaddress, macaddress_to_int
 from ..utils.application_id import is_iol_runner_environment
 from ..utils.images import images_directories
+from ..utils.vxlan import vxlan_device_name
 from .controller_error import ComputeConflictError, ComputeError, ControllerError, ControllerTimeoutError
 from .node_types import BUILTIN_NODE_TYPES
 from .ports.port_factory import DynamipsPortFactory, PortFactory, StandardPortFactory
 
 log = logging.getLogger(__name__)
 
+# The compute's error when it could not create a VXLAN tunnel device — the
+# wording is a same-repo contract with KernelDatapathMixin._kernel_error
+# (only computes running this codebase carry VXLAN links: the capability
+# gate keeps everything else on the relay), and the captured device name
+# identifies the link (a pure function of link id + node id).
+_VXLAN_CONFLICT_RE = re.compile(r"Could not create VXLAN device (?P<device>\S+)")
+
+# Node-start self-heal rounds over a refused VXLAN VNI: the first re-wires
+# the link on a fresh VNI, the second degrades it to the relay (whose NIOs
+# cannot hit the conflict) — a start must not fail permanently because
+# another link already registered the VNI on the shared dstport socket.
+_VXLAN_START_HEALS = 2
+
 
 # Image properties of a node, mapped by node type. The value is the image
 # "type" used by the image manager (see utils.images). Docker images are not
@@ -43,6 +58,7 @@ IMAGE_PROPERTIES_BY_NODE_TYPE = {
         "hdc_disk_image": "qemu",
         "hdd_disk_image": "qemu",
         "cdrom_image": "qemu",
+        "cdrom2_image": "qemu",
         "initrd": "qemu",
         "kernel_image": "qemu",
         "bios_image": "qemu",
@@ -885,6 +901,7 @@ class Node:
                 "hdc_disk_image",
                 "hdd_disk_image",
                 "cdrom_image",
+                "cdrom2_image",
                 "bios_image",
                 "initrd",
                 "kernel_image",
@@ -955,18 +972,32 @@ class Node:
                     f"Cannot start node '{self._name}': {len(failed_links)} deferred link(s) "
                     "could not be restored. Please try again."
                 )
-        try:
-            # For IOU: we need to send the licence everytime we start a node
-            if self.node_type == "iou":
-                license_check = self._project.controller.iou_license.get("license_check", True)
-                iourc_content = self._project.controller.iou_license.get("iourc_content", None)
-                await self.post(
-                    "/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
+        for heal in range(_VXLAN_START_HEALS + 1):
+            try:
+                await self._start_on_compute(data)
+                break
+            except asyncio.TimeoutError:
+                raise ControllerTimeoutError(f"Timeout when starting {self._name}")
+            except ComputeError as e:
+                conflicted = self._vxlan_conflicted_link(e)
+                if conflicted is None or heal == _VXLAN_START_HEALS:
+                    raise
+                degrade = heal > 0  # a second conflict takes the relay
+                log.warning(
+                    "Node '%s': start refused over a VXLAN VNI conflict on link %s — %s and retrying",
+                    self._name,
+                    conflicted.id,
+                    "degrading the link to the uBridge relay" if degrade else "re-wiring the link on a fresh VNI",
                 )
-            else:
-                await self.post("/start", data=data, timeout=240)
-        except TimeoutError:
-            raise ControllerTimeoutError(f"Timeout when starting {self._name}")
+                # The refused start leaves the node half-started (the
+                # emulator process runs, the wiring partial) — stop it
+                # first: a start on a running container early-returns
+                # without wiring anything.
+                await self.stop()
+                if await self._project._redispatch_vxlan_links([conflicted], downgrade=degrade) and not degrade:
+                    # The fresh VNI was already refused before the start
+                    # could even retry — degrade the pair to the relay now.
+                    await self._project._redispatch_vxlan_links([conflicted], downgrade=True)
         self._confirm_status("started")
         # A started node may have brought kernel anchors into existence that
         # a link needs to finish wiring (the Ethernet-switch fast path joins
@@ -974,6 +1005,42 @@ class Node:
         for link in self._links:
             await link.node_started(self)
 
+    async def _start_on_compute(self, data=None):
+        """
+        The compute-side start request (IOU carries its licence along).
+        """
+
+        # For IOU: we need to send the licence everytime we start a node
+        if self.node_type == "iou":
+            license_check = self._project.controller.iou_license.get("license_check", True)
+            iourc_content = self._project.controller.iou_license.get("iourc_content", None)
+            await self.post(
+                "/start", timeout=240, data={"license_check": license_check, "iourc_content": iourc_content}
+            )
+        else:
+            await self.post("/start", data=data, timeout=240)
+
+    def _vxlan_conflicted_link(self, error):
+        """
+        The link whose VXLAN device a failed node start could not create,
+        or None for every other failure.
+
+        The compute's error names the device it could not create and the
+        name is a pure function of (link id, node id), so it identifies the
+        link with no extra API surface. An unmatched message — another
+        failure, a future wording change — simply leaves the error to
+        surface unchanged, which is exactly the behaviour without the heal.
+        """
+
+        match = _VXLAN_CONFLICT_RE.search(str(error))
+        if match is None:
+            return None
+        device = match.group("device")
+        for link in self._links:
+            if vxlan_device_name(link.id, self.id) == device and any(nio.get("vxlan") for nio in link._link_data):
+                return link
+        return None
+
     async def stop(self, strict=False):
         """
         Stop a node

gns3server/controller/project.py

@@ -49,7 +49,7 @@ from .link import _UNSET
 from .node import Node
 from .snapshot import Snapshot
 from .topology import load_topology, project_to_topology
-from .udp_link import UDPLink
+from .udp_link import _VXLAN_ATTEMPTS, UDPLink
 
 log = logging.getLogger(__name__)
 
@@ -1021,6 +1021,84 @@ class Project:
         entries = await link._prepare()
         return (link, entries)
 
+    async def _dispatch_nio_batch(self, compute, nio_entries):
+        """
+        POST one compute's NIO batch for this project, returning the entries
+        it refused (VXLAN entries a compute could not wire — see the batch
+        route; other NIO types fail the whole request).
+        """
+
+        response = await compute.post(
+            f"/projects/{self._id}/nios/batch",
+            data={"nios": nio_entries},
+            timeout=300,
+        )
+        return response.json.get("failed") or []
+
+    async def _redispatch_vxlan_links(self, links, downgrade):
+        """
+        Re-prepare *links* and dispatch their NIOs in a second per-compute
+        batch, returning the links the computes still refused.
+
+        With ``downgrade=False`` each link's VNI salt is bumped first — a
+        fresh VNI, since the refused one was already registered on the
+        host's shared dstport socket. With ``downgrade=True`` the links are
+        marked degraded to the uBridge relay (whose UDP NIOs cannot fail
+        this way) before re-preparing.
+        """
+
+        for link in links:
+            if downgrade:
+                log.warning(
+                    "Project '%s' [%s]: VXLAN wiring failed for link %s — falling back to the uBridge relay datapath",
+                    self._name,
+                    self._id,
+                    link.id,
+                )
+                link._vxlan_downgraded = True
+            else:
+                link._vxlan_salt += 1
+        per_compute = {}
+        by_key = {}
+        for link in links:
+            for node, adapter_number, port_number, nio_data in await link._prepare():
+                by_key[(str(node.id), adapter_number, port_number)] = link
+                per_compute.setdefault(node.compute, []).append(
+                    {
+                        "node_id": node.id,
+                        "adapter_number": adapter_number,
+                        "port_number": port_number,
+                        "nio": nio_data,
+                    }
+                )
+        if not per_compute:
+            return []
+        results = await asyncio.gather(
+            *[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
+        )
+        failed = []
+        for item in (i for sub in results for i in sub):
+            link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
+            if link is not None and link not in failed:
+                failed.append(link)
+        return failed
+
+    async def _heal_refused_vxlan_links(self, pending):
+        """
+        Drive the links a compute refused at open onto a working datapath:
+        fresh-VNI rounds first — the same candidate count an interactive
+        link creation tries — then, for whatever still fails, the
+        degradation to the uBridge relay (whose UDP NIOs cannot hit this
+        refusal).
+        """
+
+        for _ in range(_VXLAN_ATTEMPTS - 1):
+            if not pending:
+                return
+            pending = await self._redispatch_vxlan_links(pending, downgrade=False)
+        if pending:
+            await self._redispatch_vxlan_links(pending, downgrade=True)
+
     async def restore_deferred_links(self, node):
         """
         Create on the computes the NIOs of the links that were deferred while
@@ -1994,15 +2072,30 @@ class Project:
                         }
                     )
 
-            async def _dispatch_batch(compute, nio_entries):
-                await compute.post(
-                    f"/projects/{self._id}/nios/batch",
-                    data={"nios": nio_entries},
-                    timeout=300,
-                )
-
             if per_compute:
-                await asyncio.gather(*[_dispatch_batch(c, n) for c, n in per_compute.items()])
+                results = await asyncio.gather(
+                    *[self._dispatch_nio_batch(compute, entries) for compute, entries in per_compute.items()]
+                )
+                failed = [item for sub in results for item in sub]
+            else:
+                failed = []
+            # A VXLAN entry a compute refused (its VNI already registered on
+            # the host's shared socket, no route to the peer) must not fail
+            # the whole open: the refused links are healed onto a working
+            # datapath (fresh VNIs, then the relay) — see
+            # _heal_refused_vxlan_links.
+            if failed:
+                by_key = {}
+                for link, entries in valid:
+                    for node, adapter_number, port_number, nio_data in entries:
+                        if nio_data.get("vxlan"):
+                            by_key[(str(node.id), adapter_number, port_number)] = link
+                pending = []
+                for item in failed:
+                    link = by_key.get((str(item["node_id"]), item["adapter_number"], item["port_number"]))
+                    if link is not None and link not in pending:
+                        pending.append(link)
+                await self._heal_refused_vxlan_links(pending)
 
             # Finalise every link: wire node/port back-references, mark created,
             # notify clients, and apply project-level marker definitions.

gns3server/controller/udp_link.py

@@ -18,6 +18,7 @@
 
 import asyncio
 import logging
+import uuid
 
 from gns3server.config import Config
 from gns3server.utils.application_id import is_iol_runner_environment
@@ -26,6 +27,7 @@ from gns3server.utils.packet_filter_validation import (
     split_kernel_only_features,
     validate_bpf_syntax,
 )
+from gns3server.utils.vxlan import derive_vni, vxlan_device_name
 
 from .compute import ComputeError
 from .controller_error import ControllerError, ControllerNotFoundError
@@ -52,6 +54,10 @@ _MARKER_CAPABLE_TYPES = frozenset(
 
 log = logging.getLogger(__name__)
 
+# VXLAN wiring attempts per link creation (the first plus the salt-bumped
+# retries) before the link degrades to the uBridge relay.
+_VXLAN_ATTEMPTS = 3
+
 
 def _is_unix_socket_docker(node):
     """
@@ -92,6 +98,15 @@ class UDPLink(Link):
         # of its endpoints has a missing image. The NIO creation is retried
         # once the image is resolved.
         self._deferred = False
+        # Cross-compute VXLAN wiring state: the salt is this link object's
+        # starting point in the VNI space — random per object, so a project
+        # reopen re-rolls every link's VNI and a link whose VNI collided
+        # with an unrelated device on a host is not permanently unlucky
+        # (the device *name*, which cleanup keys on, stays deterministic).
+        # _commit_vxlan bumps it for an in-flight retry; the downgrade flag
+        # marks a link degraded to the UDP relay for this object's lifetime.
+        self._vxlan_salt = uuid.uuid4().int
+        self._vxlan_downgraded = False
 
     @property
     def debug_link_data(self):
@@ -206,9 +221,13 @@ class UDPLink(Link):
         Every adapter owns an anchor, so the datapath is a runtime decision —
         links attach to running nodes too: the compute side enslaves the
         anchor (brctl) or attaches the relay to it (add_nio_ethernet) without
-        touching the interfaces the node itself uses. Both endpoints must be
-        on the same compute (there is no kernel link across hosts) and both
-        must be able to anchor: Docker adapters are born as veth pairs, and
+        touching the interfaces the node itself uses. Both endpoints must sit
+        on computes that can carry their end: on one compute the ends meet in
+        a per-link Linux bridge; across computes each end additionally rides
+        its own VXLAN device (opt-in via enable_vxlan_cross_compute, uBridge's
+        vxlan module) enslaved into the same per-link bridge, so everything
+        below the bridge — filters, markers, capture, carrier — is identical
+        either way. Both must be able to anchor: Docker adapters are born as veth pairs, and
         IOL runner containers — whose guest leg is unix sockets, not a veth —
         anchor on persistent TAPs reached through the bridge module's
         swappable TAP leg (its own capability pair, see
@@ -232,8 +251,21 @@ class UDPLink(Link):
 
         if not Config.instance().settings.Server.enable_kernel_datapath:
             return False
-        if node1.compute.id != node2.compute.id:
-            return False
+        cross_compute = node1.compute.id != node2.compute.id
+        if cross_compute:
+            # Cross-compute kernel links tunnel: each endpoint rides a VXLAN
+            # device into the same per-link bridge the same-host datapath
+            # uses. Opt-in (the tunnel caps the usable MTU at the underlay
+            # minus the outer headers, so it pays off on jumbo fabrics) and
+            # gated on both computes' uBridge being able to create the
+            # device; a link that cannot tunnel stays on the relay, where it
+            # always works. The downgrade flag is the runtime fallback: after
+            # the kernel kept refusing this link's VNI, the link stays on the
+            # relay for this object's lifetime.
+            if not Config.instance().settings.Server.enable_vxlan_cross_compute:
+                return False
+            if self._vxlan_downgraded:
+                return False
         if _is_unix_socket_docker(node1) or _is_unix_socket_docker(node2):
             return False
         for side in self._nodes:
@@ -249,10 +281,17 @@ class UDPLink(Link):
         # pair is the only shape a bridge-to-bridge cable can take).
         switch = self._switch_endpoint()
         if switch is not None:
+            if cross_compute:
+                # The fast path is a one-host shape (the peer's anchor is
+                # absorbed into the switch's bridge); a switch link across
+                # computes stays on the relay.
+                return False
             if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
                 return True
             peer = node2 if switch[0] == 0 else node1
             return self._kernel_endpoint_ready(peer)
+        if cross_compute:
+            return self._vxlan_endpoint_ready(node1) and self._vxlan_endpoint_ready(node2)
         return self._kernel_endpoint_ready(node1) and self._kernel_endpoint_ready(node2)
 
     def _switch_endpoint(self):
@@ -305,6 +344,23 @@ class UDPLink(Link):
             return capabilities.get("ubridge_tap") is True and capabilities.get("vpcs_tap") is True
         return False
 
+    @staticmethod
+    def _vxlan_endpoint_ready(node):
+        """
+        Whether a node can carry its end of a cross-compute VXLAN link: it
+        must be able to anchor (the same per-node-type capability the
+        same-host kernel datapath needs — its end of the tunnel is enslaved
+        into a per-link bridge like any kernel link's) and its compute's
+        uBridge must be able to create the VXLAN device the tunnel rides.
+        Strict True on both, like the other capability gates: an unreported
+        or failed probe keeps the link on the relay, where it always works.
+        """
+
+        if not UDPLink._kernel_endpoint_ready(node):
+            return False
+        capabilities = node.compute.capabilities or {}
+        return capabilities.get("ubridge_vxlan") is True
+
     async def _prepare(self):
         """
         Local-only link setup: resolve peer addresses, reserve UDP ports and
@@ -330,10 +386,77 @@ class UDPLink(Link):
         adapter_number2 = self._nodes[1]["adapter_number"]
         port_number2 = self._nodes[1]["port_number"]
 
-        if self._kernel_datapath_eligible(node1, node2):
+        cross_compute = node1.compute.id != node2.compute.id
+        # A cross-compute link needs the compute pair's addresses on both
+        # datapaths (the VXLAN tunnel's outer endpoints, the relay's UDP
+        # peers): resolve once, and let the no-common-subnet failure stay a
+        # ControllerError either way — it is a deployment error, not a
+        # datapath decision. Equal addresses mean both computes share one
+        # host: the kernel shares one VXLAN socket per dstport, so a tunnel
+        # pair cannot live there, and such links keep the relay, where the
+        # two ends are told apart by their UDP ports.
+        hosts = None
+        if cross_compute:
+            try:
+                hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
+            except ValueError as e:
+                raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
+        same_host = hosts is not None and hosts[0] == hosts[1]
+
+        if self._kernel_datapath_eligible(node1, node2) and not same_host:
             node1_filters, node2_filters = self._get_node_filters(node1, node2, kernel=True)
             node1_markers, node2_markers = self._get_node_markers(node1, node2)
             switch = self._switch_endpoint()
+            if hosts is not None:
+                # Cross-compute VXLAN: each end gets one VXLAN device
+                # carrying the link's VNI, enslaved into the per-link kernel
+                # bridge next to the node's anchor (see _kernel_attach on
+                # the compute side). The controller derives name and VNI —
+                # both ends must agree on the VNI and neither can ask the
+                # other — and the compute resolves the underlay interface
+                # locally from the peer address, so the kernel derives the
+                # device MTU (underlay - 50) and the bridge reports the
+                # honest "what this fabric can carry" value. A VNI the
+                # kernel refuses as already registered on the shared
+                # dstport socket surfaces as a commit failure; create()
+                # retries with a bumped salt and degrades to the relay when
+                # the retries run out.
+                node1_host, node2_host = hosts
+                dstport = Config.instance().settings.Server.vxlan_dstport
+                vni = derive_vni(self._id, self._vxlan_salt)
+                bridge_name = "gns3" + self._id.replace("-", "")[:11]
+                self._link_data = [
+                    {
+                        "type": "nio_bridge",
+                        "bridge": bridge_name,
+                        "vxlan": {
+                            "name": vxlan_device_name(self._id, node1.id),
+                            "vni": vni,
+                            "remote": node2_host,
+                            "dstport": dstport,
+                        },
+                        "filters": node1_filters,
+                        "markers": node1_markers,
+                        "suspend": self._suspended,
+                    },
+                    {
+                        "type": "nio_bridge",
+                        "bridge": bridge_name,
+                        "vxlan": {
+                            "name": vxlan_device_name(self._id, node2.id),
+                            "vni": vni,
+                            "remote": node1_host,
+                            "dstport": dstport,
+                        },
+                        "filters": node2_filters,
+                        "markers": node2_markers,
+                        "suspend": self._suspended,
+                    },
+                ]
+                return [
+                    (node1, adapter_number1, port_number1, self._link_data[0]),
+                    (node2, adapter_number2, port_number2, self._link_data[1]),
+                ]
             if switch is not None:
                 if node1.node_type == "ethernet_switch" and node2.node_type == "ethernet_switch":
                     # Switch-to-switch cascade: one veth pair joins the two
@@ -449,11 +572,16 @@ class UDPLink(Link):
                 (node2, adapter_number2, port_number2, self._link_data[1]),
             ]
 
-        # Get an IP allowing communication between both host
-        try:
-            (node1_host, node2_host) = await node1.compute.get_ip_on_same_subnet(node2.compute)
-        except ValueError as e:
-            raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
+        # Relay datapath: UDP tunnels between the computes' addresses —
+        # already resolved for a cross-compute pair, resolved here otherwise
+        # (a same-compute relay pair resolves to the host's own address on
+        # both sides, distinguished by their UDP ports).
+        if hosts is None:
+            try:
+                hosts = await node1.compute.get_ip_on_same_subnet(node2.compute)
+            except ValueError as e:
+                raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
+        node1_host, node2_host = hosts
 
         # Reserve a UDP port on both sides in parallel. Pre-allocated ports
         # (used during batch project loading) are popped from memory; otherwise
@@ -537,17 +665,70 @@ class UDPLink(Link):
             raise errors[0]
         self._created = True
 
+    @staticmethod
+    def _has_vxlan(entries):
+        """
+        Whether any prepared NIO spec of this link carries a VXLAN tunnel.
+        """
+
+        return any(nio_data.get("vxlan") for _, _, _, nio_data in entries)
+
     async def create(self):
         """
         Create the link on the nodes (interactive path: prepare + commit).
         """
 
         entries = await self._prepare()
-        await self._commit_nios(entries)
+        if self._has_vxlan(entries):
+            await self._commit_vxlan(entries)
+        else:
+            await self._commit_nios(entries)
         # New links automatically inherit every active project-level marker
         # definition so the user doesn't have to reconfigure.
         await self._project.apply_defs_to_new_link(self)
 
+    async def _commit_vxlan(self, entries):
+        """
+        Commit a cross-compute VXLAN link, retrying with a bumped VNI salt
+        on failure and degrading the link to the UDP relay when the retries
+        run out.
+
+        The kernel refuses a VNI already registered on the shared dstport
+        socket at device creation (EEXIST) — the one conflict a freshly
+        created, deterministically named device can hit: the other end's
+        device lives on another host, and a leftover under this end's name
+        is replaced by the attach path, not refused. Any other commit
+        failure (a stopped node, a missing underlay route) is retried too,
+        harmlessly — it fails the same way every attempt, and the final
+        relay attempt surfaces the real error if the link genuinely cannot
+        be created. VXLAN is opt-in and the relay always works, so a tunnel
+        that cannot be wired degrades instead of failing the link; the
+        degradation lasts for this link object's lifetime (a project reopen
+        retries VXLAN from a clean state).
+        """
+
+        last_error = None
+        for attempt in range(_VXLAN_ATTEMPTS):
+            if attempt:
+                self._vxlan_salt += 1
+                entries = await self._prepare()
+                if not self._has_vxlan(entries):
+                    # Eligibility changed under us (a capability disappeared):
+                    # commit the relay entries as prepared.
+                    await self._commit_nios(entries)
+                    return
+            try:
+                await self._commit_nios(entries)
+                return
+            except (ComputeError, ControllerError) as e:
+                last_error = e
+        log.warning(
+            "Link %s: VXLAN wiring failed (%s) — falling back to the uBridge relay datapath", self._id, last_error
+        )
+        self._vxlan_downgraded = True
+        entries = await self._prepare()
+        await self._commit_nios(entries)
+
     async def update(self):
         """
         Update the link on the nodes