Merge remote-tracking branch 'origin/3.1' into 3.1

gns3server/controller/__init__.py

@@ -741,6 +741,9 @@ class Controller:
         topo_data.pop("version")
         topo_data.pop("revision")
         topo_data.pop("type")
+        # marker_definitions is restored by Project.open() from the topology
+        # file; it must not be passed to Project.__init__.
+        topo_data.pop("marker_definitions", None)
 
         if topo_data["project_id"] in self._projects:
             project = self._projects[topo_data["project_id"]]

gns3server/controller/link.py

@@ -88,6 +88,7 @@ class Link:
         self._link_type = "ethernet"
         self._suspended = False
         self._filters = {}
+        self._markers = {}
         self._link_style = {}
         self._wireshark = False
         self._show_filters_icon = True
@@ -99,6 +100,40 @@ class Link:
         """
         return self._filters
 
+    @property
+    def markers(self):
+        """
+        Get the traffic insight markers dict: name → {bpf, tag, enabled}
+        """
+        return self._markers
+
+    async def inherit_marker(self, def_name, marker_def):
+        """
+        Apply a project-level marker definition to this link.
+
+        The marker is stored under ``global-{def_name}`` so it cannot collide
+        with a per-link private marker of the same name.  It carries an
+        ``inherited_from`` back-reference that (a) guards against per-link
+        edits and (b) lets the project sync changes to every copy at once.
+        """
+
+        await self.start_marker(
+            name=f"global-{def_name}",
+            bpf=marker_def["bpf"],
+            tag=marker_def.get("tag"),
+            color=marker_def.get("color"),
+            highlight_duration=marker_def.get("highlight_duration"),
+            inherited_from=def_name,
+        )
+
+    def _persist_markers(self):
+        """
+        Return only the per-link (non-inherited) markers suitable for
+        persistence in a topology dump.  Inherited markers are re-created from
+        ``project._marker_definitions`` on load so they do not need to be saved.
+        """
+        return {k: v for k, v in self._markers.items() if not v.get("inherited_from")}
+
     @property
     def show_filters_icon(self):
         """
@@ -298,6 +333,27 @@ class Link:
 
         raise NotImplementedError
 
+    async def start_marker(self, name, bpf, tag=None):
+        """
+        Attach a traffic-insight marker to this link (base — UDPLink overrides).
+        """
+        raise NotImplementedError
+
+    async def stop_marker(self, name):
+        """
+        Remove a traffic-insight marker from this link (base — UDPLink overrides).
+        """
+        raise NotImplementedError
+
+    async def update_marker(self, name, bpf=None, tag=None, enabled=None):
+        """
+        Update an existing marker's BPF, tag, or enabled flag.
+
+        A BPF change is a delete+re-add on the ubridge side so the pcap is
+        flushed and the new filter takes effect.
+        """
+        raise NotImplementedError
+
     async def start_capture(self, data_link_type="DLT_EN10MB", capture_file_name=None, wireshark=False, jwt_token=None):
         """
         Start capture on the link
@@ -571,6 +627,7 @@ class Link:
                 "nodes": res,
                 "link_id": self._id,
                 "filters": self._filters,
+                "markers": self._persist_markers(),
                 "link_style": self._link_style,
                 "suspend": self._suspended,
                 "show_filters_icon": getattr(self, '_show_filters_icon', True),
@@ -585,6 +642,7 @@ class Link:
             "capture_compute_id": self.capture_compute_id,
             "link_type": self._link_type,
             "filters": self._filters,
+            "markers": self._markers,
             "suspend": self._suspended,
             "link_style": self._link_style,
             "wireshark": self._wireshark,

gns3server/controller/node.py

@@ -639,6 +639,14 @@ class Node:
             except asyncio.TimeoutError:
                 raise ControllerTimeoutError(f"Timeout when reset console {self._name}")
 
+    async def get(self, path="", **kwargs):
+        """
+        HTTP get on the node
+        """
+        return await self._compute.get(
+            f"/projects/{self._project.id}/{self._node_type}/nodes/{self._id}{path}", **kwargs
+        )
+
     async def post(self, path, data=None, **kwargs):
         """
         HTTP post on the node

gns3server/controller/project.py

@@ -41,6 +41,7 @@ from ..config import Config
 from ..utils.path import check_path_allowed, get_default_project_directory
 from ..utils.application_id import get_next_application_id
 from ..utils.asyncio.pool import Pool
+from ..utils.packet_filter_validation import validate_bpf_syntax
 from ..utils.asyncio import locking
 from ..utils.asyncio import aiozipstream
 from ..utils.asyncio import wait_run_in_executor
@@ -211,6 +212,7 @@ class Project:
         self._allocated_node_names = set()
         self._nodes = {}
         self._links = {}
+        self._marker_definitions = {}  # name → {bpf, tag, color, highlight_duration}
         self._drawings = {}
         self._snapshots = {}
         self._computes = []
@@ -765,6 +767,31 @@ class Project:
                     "Dropping invalid filters on link %s: %s",
                     link_data.get("link_id"), e
                 )
+        # Restore traffic-insight markers directly into link state (mirrors how
+        # filters are restored via update_filters). The capture_node_id persisted
+        # last time is reused for NIO routing; no side resolution is possible here
+        # because the link's nodes are added later. The marker is applied to
+        # uBridge by _ubridge_apply_markers when create() runs. Invalid BPF is
+        # dropped (like invalid filters).
+        for name, marker in (link_data.get("markers") or {}).items():
+            bpf = marker.get("bpf")
+            if not bpf:
+                log.warning("Dropping marker %s on link %s: missing bpf", name, link_data.get("link_id"))
+                continue
+            result = validate_bpf_syntax(bpf)
+            if not result.get("valid"):
+                log.warning(
+                    "Dropping marker %s on link %s: invalid BPF (%s)",
+                    name, link_data.get("link_id"), result.get("error")
+                )
+                continue
+            link._markers[name] = {
+                "bpf": bpf,
+                "tag": marker.get("tag"),
+                "enabled": marker.get("enabled", True),
+                "color": marker.get("color"),
+                "capture_node_id": marker.get("capture_node_id"),
+            }
         if "link_style" in link_data:
             await link.update_link_style(link_data["link_style"])
         if "show_filters_icon" in link_data:
@@ -872,6 +899,146 @@ class Project:
             return self._get_closed_data("links", "link_id")
         return self._links
 
+    @property
+    def markers(self):
+        """
+        Project-level read-only aggregation of all markers across every link.
+
+        Each entry is keyed ``"{link_id}/{marker_name}"`` so the flat dict is
+        globally unique within the project.  The value is a clone of the link's
+        per-marker dict plus ``link_id`` and ``node_id`` (the capture-side node)
+        for convenience — the frontend can filter/group by link or node without
+        extra round-trips.
+
+        :returns: dict[str, dict] — keyed by "{link_id}/{marker_name}"
+        """
+        result = {}
+        for link_id, link in self._links.items():
+            for name, info in link.markers.items():
+                key = f"{link_id}/{name}"
+                result[key] = {
+                    **info,
+                    "link_id": link_id,
+                    "node_id": info.get("capture_node_id"),
+                }
+        return result
+
+    @property
+    def marker_definitions(self):
+        """
+        :returns: dict of project-level marker definitions (name → {bpf, tag, color, highlight_duration})
+        """
+        return self._marker_definitions
+
+    async def create_marker_definition(self, name, bpf, tag=None, color=None, highlight_duration=None):
+        """
+        Create a project-level marker definition and fan out to every existing
+        link that has a capable node.  Links without a capable node are silently
+        skipped.
+        """
+
+        if name in self._marker_definitions:
+            raise ControllerError(
+                f"Marker definition '{name}' already exists in this project"
+            )
+
+        self._marker_definitions[name] = {"bpf": bpf, "tag": tag, "color": color, "highlight_duration": highlight_duration}
+        await self._apply_def_to_all_links(name)
+        self.dump()
+        self.emit_notification("project.updated", self.asdict())
+
+    async def update_marker_definition(self, name, bpf=None, tag=None, color=None, highlight_duration=None):
+        """
+        Update a marker definition and sync every inherited copy on every link.
+        """
+
+        if name not in self._marker_definitions:
+            raise ControllerNotFoundError(
+                f"Marker definition '{name}' not found in this project"
+            )
+
+        d = self._marker_definitions[name]
+        if bpf is not None:
+            d["bpf"] = bpf
+        if tag is not None:
+            d["tag"] = tag
+        if color is not None:
+            d["color"] = color
+        if highlight_duration is not None:
+            d["highlight_duration"] = highlight_duration
+
+        # Sync: update every inherited copy across all links.
+        for link in list(self._links.values()):
+            marker_name = f"global-{name}"
+            if marker_name in link.markers and link.markers[marker_name].get("inherited_from") == name:
+                await link.update_marker(
+                    marker_name, bpf=d["bpf"], tag=d.get("tag"), color=d.get("color"),
+                    highlight_duration=d.get("highlight_duration"), inherited=True
+                )
+        self.dump()
+        self.emit_notification("project.updated", self.asdict())
+
+    async def delete_marker_definition(self, name):
+        """
+        Delete a marker definition and remove every inherited copy from every link.
+        """
+
+        if name not in self._marker_definitions:
+            raise ControllerNotFoundError(
+                f"Marker definition '{name}' not found in this project"
+            )
+
+        del self._marker_definitions[name]
+
+        for link in list(self._links.values()):
+            marker_name = f"global-{name}"
+            if marker_name in link.markers and link.markers[marker_name].get("inherited_from") == name:
+                try:
+                    await link.stop_marker(marker_name, inherited=True)
+                except ControllerError:
+                    # A missing compute or broken link shouldn't block the delete.
+                    log.warning(
+                        "Failed to remove inherited marker %s from link %s",
+                        marker_name, link.id
+                    )
+
+        self.dump()
+        self.emit_notification("project.updated", self.asdict())
+
+    async def _apply_def_to_all_links(self, def_name):
+        """
+        Fan out a single marker definition to every existing link in the project.
+        Links that have no capable node (``_MARKER_CAPABLE_TYPES``) are silently
+        skipped — the marker can only live on a uBridge bridge.
+        """
+
+        d = self._marker_definitions[def_name]
+        for link in list(self._links.values()):
+            try:
+                await link.inherit_marker(def_name, d)
+            except ControllerError as e:
+                # Per-link failures (e.g. no capable node) shouldn't block the
+                # definition from serving the rest.
+                log.warning(
+                    "Marker definition '%s' could not be applied to link %s: %s",
+                    def_name, link.id, e
+                )
+
+    async def apply_defs_to_new_link(self, link):
+        """
+        Apply every active marker definition to a newly created link so it
+        inherits project-level rules automatically.
+        """
+
+        for def_name, d in self._marker_definitions.items():
+            try:
+                await link.inherit_marker(def_name, d)
+            except ControllerError as e:
+                log.warning(
+                    "Marker definition '%s' could not be applied to new link %s: %s",
+                    def_name, link.id, e
+                )
+
     @property
     def snapshots(self):
         """
@@ -1262,6 +1429,12 @@ class Project:
                 if val is not None:
                     setattr(self, key, val)
 
+            # marker_definitions is loaded separately (it is not a __init__ kwarg
+            # nor a simple attribute — it backs a read-only property).
+            defs = project_data.get("marker_definitions")
+            if isinstance(defs, dict):
+                self._marker_definitions = defs
+
             topology = project_data["topology"]
             for compute in topology.get("computes", []):
                 compute_id = compute.get("compute_id")
@@ -1328,6 +1501,10 @@ class Project:
             for drawing_data in topology.get("drawings", []):
                 await self.add_drawing(dump=False, **drawing_data)
 
+            # Note: project-level marker definitions are applied to each link
+            # inside UDPLink.create() (the inheritance hook), so they are
+            # already present once links are loaded — no separate fan-out here.
+
             self.dump()
         # We catch all error to be able to roll back the .gns3 to the previous state
         except Exception as e:
@@ -1684,6 +1861,7 @@ class Project:
             "links": len(self._links),
             "drawings": len(self._drawings),
             "snapshots": len(self._snapshots),
+            "markers": sum(len(link.markers) for link in self._links.values()),
         }
 
     def asdict(self):
@@ -1708,6 +1886,7 @@ class Project:
             "supplier": self._supplier,
             "variables": self._variables,
             "created_by": self._created_by,
+            "marker_definitions": self._marker_definitions,
         }
 
     def __repr__(self):

gns3server/controller/topology.py

@@ -88,6 +88,7 @@ def project_to_topology(project):
         "variables": project.variables,
         "supplier": project.supplier,
         "created_by": project.created_by,
+        "marker_definitions": project.marker_definitions,
         "topology": {"nodes": [], "links": [], "computes": [], "drawings": []},
         "type": "topology",
         "revision": GNS3_FILE_FORMAT_REVISION,

gns3server/controller/udp_link.py

@@ -19,6 +19,15 @@
 from .controller_error import ControllerError, ControllerNotFoundError
 from .link import Link
 from .node_types import BUILTIN_NODE_TYPES
+from gns3server.utils.packet_filter_validation import validate_bpf_syntax, FilterValidationError
+
+# Node types without a uBridge bridge — a marker filter has nothing to attach to.
+# Node types that can host a marker (have a uBridge bridge to attach the
+# `mark` filter to).  Mirrors _get_filter_node in link.py, minus "nat"
+# (which has no uBridge).
+_MARKER_CAPABLE_TYPES = frozenset({
+    "vpcs", "qemu", "docker", "iou", "dynamips", "cloud",
+})
 
 
 class UDPLink(Link):
@@ -37,7 +46,7 @@ class UDPLink(Link):
     def _get_node_filters(self, node1, node2):
         """
         Determine which node gets the active filters applied.
-        
+
         :returns: Tuple of (node1_filters, node2_filters)
         """
         filter_node = self._get_filter_node()
@@ -46,6 +55,26 @@ class UDPLink(Link):
             self.get_active_filters() if filter_node == node2 else {},
         )
 
+    def _markers_for_node(self, node):
+        """
+        Marker specs (name -> {bpf, tag, link_id}) for the markers whose capture
+        side is ``node`` and that are enabled. Routed by capture_node_id so a
+        marker only rides the NIO of the node whose uBridge will host it.
+        """
+        return {
+            name: {"bpf": m["bpf"], "tag": m.get("tag"), "link_id": self._id}
+            for name, m in self._markers.items()
+            if m.get("enabled", True) and m.get("capture_node_id") == node.id
+        }
+
+    def _get_node_markers(self, node1, node2):
+        """
+        Determine which node gets which markers applied.
+
+        :returns: Tuple of (node1_markers, node2_markers)
+        """
+        return self._markers_for_node(node1), self._markers_for_node(node2)
+
     async def create(self):
         """
         Create the link on the nodes
@@ -80,6 +109,7 @@ class UDPLink(Link):
             self._node2_port = response.json["udp_port"]
 
         node1_filters, node2_filters = self._get_node_filters(node1, node2)
+        node1_markers, node2_markers = self._get_node_markers(node1, node2)
 
         # Create the tunnel on both side
         self._link_data.append(
@@ -89,6 +119,7 @@ class UDPLink(Link):
                 "rport": self._node2_port,
                 "type": "nio_udp",
                 "filters": node1_filters,
+                "markers": node1_markers,
                 "suspend": self._suspended,
             }
         )
@@ -101,6 +132,7 @@ class UDPLink(Link):
                 "rport": self._node1_port,
                 "type": "nio_udp",
                 "filters": node2_filters,
+                "markers": node2_markers,
                 "suspend": self._suspended,
             }
         )
@@ -113,6 +145,9 @@ class UDPLink(Link):
             await node1.delete(f"/adapters/{adapter_number1}/ports/{port_number1}/nio", timeout=120)
             raise e
         self._created = True
+        # New links automatically inherit every active project-level marker
+        # definition so the user doesn't have to reconfigure.
+        await self._project.apply_defs_to_new_link(self)
 
     async def update(self):
         """
@@ -125,10 +160,12 @@ class UDPLink(Link):
         node2 = self._nodes[1]["node"]
 
         node1_filters, node2_filters = self._get_node_filters(node1, node2)
+        node1_markers, node2_markers = self._get_node_markers(node1, node2)
 
         adapter_number1 = self._nodes[0]["adapter_number"]
         port_number1 = self._nodes[0]["port_number"]
         self._link_data[0]["filters"] = node1_filters
+        self._link_data[0]["markers"] = node1_markers
         self._link_data[0]["suspend"] = self._suspended
         if node1.node_type not in ("ethernet_switch", "ethernet_hub"):
             await node1.put(
@@ -138,6 +175,7 @@ class UDPLink(Link):
         adapter_number2 = self._nodes[1]["adapter_number"]
         port_number2 = self._nodes[1]["port_number"]
         self._link_data[1]["filters"] = node2_filters
+        self._link_data[1]["markers"] = node2_markers
         self._link_data[1]["suspend"] = self._suspended
         if node2.node_type not in ("ethernet_switch", "ethernet_hub"):
             await node2.put(
@@ -245,9 +283,161 @@ class UDPLink(Link):
 
         raise ControllerError("Cannot capture because there is no running device on this link")
 
+    def _choose_marker_side(self):
+        """
+        Pick the node that will host the marker, mirroring ``_get_filter_node``
+        in link.py.  Only types with a uBridge bridge (``_MARKER_CAPABLE_TYPES``)
+        are eligible.  A running node is preferred, but a stopped one is
+        accepted — like packet filters, the marker is stored on the NIO and
+        applied when the node starts.
+        """
+
+        # Prefer started.
+        for node in self._nodes:
+            if (
+                node["node"].node_type in _MARKER_CAPABLE_TYPES
+                and node["node"].status == "started"
+            ):
+                return node
+
+        # Accept stopped but capable (marker rides NIO, applied at start).
+        for node in self._nodes:
+            if node["node"].node_type in _MARKER_CAPABLE_TYPES:
+                return node
+
+        raise ControllerError(
+            "Cannot add marker because no device on this link supports "
+            "traffic insight"
+        )
+
     async def node_updated(self, node):
         """
         Called when a node member of the link is updated
         """
         if self._capture_node and node == self._capture_node["node"] and node.status != "started":
             await self.stop_capture()
+        # Marker clean-up is *not* done on node stop — markers are a persistent
+        # link-scoped feature that recovers via NIO on restart (see
+        # _ubridge_apply_markers in add_ubridge_udp_connection).  The user
+        # explicitly deletes a marker via the REST API, and a marker is torn
+        # down automatically only when its link is deleted.
+
+    async def start_marker(self, name, bpf, tag=None, color=None, highlight_duration=None, inherited_from=None):
+        """
+        Attach a traffic-insight marker to this link.
+
+        State-only model (mirrors ``update_filters``): record the marker in
+        ``_markers`` (with its capture-side node id for NIO routing), then push
+        via ``self.update()`` so it rides the NIO and is applied by
+        ``_ubridge_apply_markers``. No dedicated uBridge round-trip — exactly
+        how packet filters are applied.
+
+        :param name: stable filter name — echoed in MARK signals + pcap identity
+        :param bpf: libpcap BPF expression
+        :param tag: optional correlation id
+        :param color: optional hex color for the Web UI (e.g. '#ff5722'); stored
+            with the link and persisted in the topology, never sent to uBridge
+        :param highlight_duration: optional UI-only hint (milliseconds) for how
+            long a match keeps the marker highlighted; stored, never sent to uBridge
+        :param inherited_from: def name when this marker is a project-level
+            inheritance copy; set automatically, never exposed to REST callers
+        """
+
+        if name in self._markers:
+            raise ControllerError(f"Marker '{name}' already exists on link {self._id}")
+
+        result = validate_bpf_syntax(bpf)
+        if not result.get("valid"):
+            raise ControllerError(f"Invalid BPF expression: {result.get('error', 'unknown error')}")
+
+        marker_side = self._choose_marker_side()
+        marker_entry = {
+            "bpf": bpf,
+            "tag": tag,
+            "enabled": True,
+            "color": color,
+            "highlight_duration": highlight_duration,
+            "capture_node_id": marker_side["node"].id,
+        }
+        if inherited_from:
+            marker_entry["inherited_from"] = inherited_from
+        self._markers[name] = marker_entry
+        if self._created:
+            await self.update()
+        self._project.emit_notification("link.updated", self.asdict())
+        self._project.dump()
+
+    async def stop_marker(self, name, inherited=False):
+        """
+        Remove a traffic-insight marker from this link.
+
+        Drop it from ``_markers`` and push via ``self.update()``: the NIO
+        reset+reapply in ``_ubridge_apply_filters``/``_ubridge_apply_markers``
+        drops it from uBridge. Mirrors how deleting a packet filter works.
+
+        :param name: filter name to remove
+        :param inherited: set by project-level def-delete to bypass the
+            inheritance guard (the project layer is the legitimate remover)
+        """
+
+        if name not in self._markers:
+            raise ControllerNotFoundError(f"Marker '{name}' not found on link {self._id}")
+
+        if self._markers[name].get("inherited_from") and not inherited:
+            raise ControllerError(
+                f"Marker '{name}' is inherited from the project-level "
+                f"definition '{self._markers[name]['inherited_from']}'. "
+                "Delete or update it via the marker-definitions API instead."
+            )
+
+        del self._markers[name]
+        if self._created:
+            await self.update()
+        self._project.emit_notification("link.updated", self.asdict())
+        self._project.dump()
+
+    async def update_marker(self, name, bpf=None, tag=None, enabled=None, color=None, highlight_duration=None, inherited=False):
+        """
+        Update an existing marker's BPF/tag/enabled/color. Any change pushes via
+        ``self.update()``; uBridge picks up the new params on the next NIO
+        reset+reapply (same as packet filters).
+
+        :param name: filter name to update
+        :param bpf: new BPF expression (None = keep existing)
+        :param tag: new tag id (None = keep existing)
+        :param enabled: toggle (None = keep existing)
+        :param color: new hex color (None = keep existing)
+        :param highlight_duration: new UI highlight duration in ms (None = keep existing)
+        :param inherited: set by project-level sync to bypass the inheritance
+            guard (the project layer is the legitimate editor)
+        """
+
+        marker_info = self._markers.get(name)
+        if not marker_info:
+            raise ControllerNotFoundError(f"Marker '{name}' not found on link {self._id}")
+
+        if marker_info.get("inherited_from") and not inherited:
+            raise ControllerError(
+                f"Marker '{name}' is inherited from the project-level "
+                f"definition '{marker_info['inherited_from']}'. "
+                "Update it via the marker-definitions API instead."
+            )
+
+        if bpf is not None and bpf != marker_info["bpf"]:
+            result = validate_bpf_syntax(bpf)
+            if not result.get("valid"):
+                raise ControllerError(f"Invalid BPF expression: {result.get('error', 'unknown error')}")
+            marker_info["bpf"] = bpf
+        if tag is not None:
+            marker_info["tag"] = tag
+        if enabled is not None:
+            marker_info["enabled"] = enabled
+        if color is not None:
+            marker_info["color"] = color
+        if highlight_duration is not None:
+            marker_info["highlight_duration"] = highlight_duration
+
+        if self._created:
+            await self.update()
+        self._project.emit_notification("link.updated", self.asdict())
+        self._project.dump()