NgTech-LTD/gns3-server
2
0 Fork 0
mirror of https://github.com/GNS3/gns3-server.git synced 2026-11-03 14:01:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: drive marker replay with resident sharkd sessions

Browse Source 
sharkd (the Wireshark daemon) is now the single decode engine — the
tshark/PDML path is gone, and without sharkd every replay endpoint
returns 501 (no degraded mode: one engine, one rendering shape for the
Web UI).

- Frame entries gain packet-list columns from sharkd's frames RPC:
  src/dst/proto/info plus the Wireshark coloring hints bg/fg
- range and frames accept ?filter=<display filter>, applied before
  counting and slicing; invalid expressions are 400 carrying sharkd's
  original text; filters travel as single argv-style elements, capped
  at 2000 chars; filtered frames keep their original pcap frame numbers
- frame detail returns sharkd's protocol tree with keys renamed into
  the REST contract (element/label/name/filter_expr/pos+size/expert/
  generated/children): a census-verified closed key set, values
  untouched, unknown keys passed through verbatim, Wireshark-internal
  hf ids dropped. filter_expr gives the UI click-to-filter; pos/size
  drives hex highlighting (hex still read straight from the pcap)
- one resident 'sharkd -' session per source pcap: lazy spawn, /tmp
  scratch copy + scratch HOME (hardened profiles), per-request
  (mtime,size) validation with respawn, LRU bound, per-session lock,
  per-RPC timeout, bounded close

Timeline backbone (gate, record-header scan, merge ordering, canonical
ts strings, hex reads) stays plain Python — identity and ordering never
depend on the engine.
This commit is contained in:
YueGuobin committed 2026-09-08 00:04:04 +08:00
1 parent 0c540abbf2
commit 790c423c26
5 files changed
+1018 -401
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/api/routes/controller/test_marker_replay.py
+159 -33
View File
@@ -16,27 +16,39 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
HTTP-route tests for the tag replay endpoints: the tag gate (409 while any
marker captures, 404 unknown tag), the merged timeline, window queries
(empty window = success), and the lazy frame detail (ts guard, isomorphic
JSON, 501 when tshark is unavailable).
HTTP-route tests for the tag replay endpoints (sharkd edition): the tag gate
(409 while any marker captures, 404 unknown tag), the merged timeline with
columns, window queries (empty window = success), the display-filter
parameter (400 with sharkd's error text on a bad expression), the lazy frame
detail with the renamed sharkd tree, and 501 when sharkd is unavailable
(hard engine requirement, no degraded mode).
"""
import shutil
from unittest.mock import patch
import pytest
import pytest_asyncio
from fastapi import FastAPI, status
from httpx import AsyncClient
from gns3server.controller.project import Project
from gns3server.controller.udp_link import UDPLink
from gns3server.controller import marker_replay
from tests.controller.test_marker_replay import _write_pcap, _icmp_frame
from tests.controller.test_marker_replay import _write_pcap, _icmp_frame, _tcp_syn_frame, _cols
pytestmark = pytest.mark.asyncio
tshark_present = pytest.mark.skipif(shutil.which("tshark") is None, reason="tshark not installed")
sharkd_present = pytest.mark.skipif(shutil.which("sharkd") is None, reason="sharkd not installed")
@pytest_asyncio.fixture
async def no_residual_sessions():
yield
manager = marker_replay._manager
if manager is not None:
await manager.close_all()
def _add_marker(project, tag, enabled, node_id, frames=None):
@@ -77,18 +89,48 @@ class TestReplayRoutes:
)
assert response.status_code == status.HTTP_404_NOT_FOUND
async def test_range_merges_sources_in_ts_order(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
async def test_range_501_without_sharkd(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
# A non-empty pcap: the engine must be consulted, and without sharkd
# the whole feature is unavailable (hard requirement, no degraded mode).
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 123456, _icmp_frame()),
])
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
response = await client.get(
app.url_path_for("replay_tag_range", project_id=project.id, tag=7)
)
assert response.status_code == status.HTTP_501_NOT_IMPLEMENTED
# The app's HTTPException handler unifies the body as {"message": …}.
assert "sharkd" in response.json()["message"]
async def test_range_merges_sources_in_ts_order(
self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch
) -> None:
# Columns injected hermetically — the merge/order/tiebreak contract
# must not depend on the engine being installed.
r1, r2 = UDPLink(project), UDPLink(project)
project._links.update({r1.id: r1, r2.id: r2})
def _wire(link, node_id, frames):
link._markers["icmp"] = {"bpf": "icmp", "tag": 7, "enabled": False, "color": None,
"highlight_duration": None, "capture_node_id": node_id,
"direction": None, "data_link_type": "DLT_EN10MB"}
_write_pcap(f"{project.markers_directory}/{node_id}_{link.id}_icmp.pcap", frames)
# r1→r2 captures at t1 and t3; r2→r3 captures at t2 and t3 (same µs
# as source A's t3 — the tiebreak must keep both frames).
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 500000, b"a" * 60),
(1693472002, 0, b"a" * 60),
])
_add_marker(project, tag=7, enabled=False, node_id="n2", frames=[
(1693472001, 0, b"b" * 60),
(1693472002, 0, b"b" * 60),
])
_wire(r1, "n1", [(1693472000, 500000, b"a" * 60), (1693472002, 0, b"a" * 60)])
_wire(r2, "n2", [(1693472001, 0, b"b" * 60), (1693472002, 0, b"b" * 60)])
async def fake_columns(pcap, filter_expr):
name = pcap.rsplit("/", 1)[-1]
src = "10.0.0.1" if name.startswith("n1") else "10.0.0.2"
return {1: _cols(src=src), 2: _cols(src=src)}
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns)
response = await client.get(
app.url_path_for("replay_tag_range", project_id=project.id, tag=7)
@@ -105,16 +147,25 @@ class TestReplayRoutes:
"1693472000.500000", "1693472001.000000",
"1693472002.000000", "1693472002.000000",
]
# Wireshark-style columns ride along on every frame entry.
assert body["frames"][0]["src"] == "10.0.0.1"
assert body["frames"][1]["src"] == "10.0.0.2"
assert body["frames"][0]["proto"] == "ICMP"
assert len(body["sources"]) == 2
async def test_frames_window_miss_is_empty_success(
self, app: FastAPI, client: AsyncClient, project: Project
self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch
) -> None:
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 0, b"a" * 60),
])
async def fake_columns(pcap, filter_expr):
return {1: _cols()}
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns)
response = await client.get(
app.url_path_for("replay_tag_frames", project_id=project.id, tag=7),
params={"ts": "1693472001.000000", "window_ms": 100},
@@ -122,13 +173,20 @@ class TestReplayRoutes:
assert response.status_code == status.HTTP_200_OK
assert response.json() == {"frames": []}
async def test_frames_window_hit(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
async def test_frames_window_hit(
self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch
) -> None:
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 0, b"a" * 60),
(1693472000, 150000, b"a" * 60),
])
async def fake_columns(pcap, filter_expr):
return {1: _cols(), 2: _cols()}
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns)
response = await client.get(
app.url_path_for("replay_tag_frames", project_id=project.id, tag=7),
params={"ts": "1693472000.000000", "window_ms": 150},
@@ -151,22 +209,64 @@ class TestReplayRoutes:
assert response.status_code == status.HTTP_404_NOT_FOUND
assert "rebuilt" in response.json()["message"]
async def test_detail_501_without_tshark(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
@sharkd_present
async def test_range_columns_and_filter_end_to_end(
self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions
) -> None:
link = _add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 123456, _icmp_frame()),
(1693472001, 0, _tcp_syn_frame()),
])
# Unfiltered: both frames with real engine columns.
response = await client.get(
app.url_path_for("replay_tag_range", project_id=project.id, tag=7)
)
assert response.status_code == status.HTTP_200_OK
frames = response.json()["frames"]
assert [f["proto"] for f in frames] == ["ICMP", "TCP"]
assert frames[0]["src"] == "10.0.0.1" and frames[0]["dst"] == "10.0.0.3"
assert "Echo" in frames[0]["info"]
assert frames[0]["frame_number"] == 1
# Display filter applied before counting: only the TCP frame survives.
response = await client.get(
app.url_path_for("replay_tag_range", project_id=project.id, tag=7),
params={"filter": "tcp"},
)
body = response.json()
assert body["frame_count"] == 1
assert [f["frame_number"] for f in body["frames"]] == [2] # original pcap identity
assert body["start"] == "1693472001.000000"
@sharkd_present
async def test_range_invalid_filter_is_400_with_sharkd_text(
self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions
) -> None:
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 123456, _icmp_frame()),
])
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
response = await client.get(
app.url_path_for("replay_tag_frame_detail", project_id=project.id, tag=7),
params={"ts": "1693472000.123456", "node_id": "n1",
"link_id": link.id, "marker": "icmp"},
)
assert response.status_code == status.HTTP_501_NOT_IMPLEMENTED
response = await client.get(
app.url_path_for("replay_tag_range", project_id=project.id, tag=7),
params={"filter": "this is (not valid"},
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "filter" in response.json()["message"].lower()
@tshark_present
async def test_detail_decodes_single_frame(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
# Oversized filters are rejected before reaching the engine.
response = await client.get(
app.url_path_for("replay_tag_range", project_id=project.id, tag=7),
params={"filter": "x" * (marker_replay.FILTER_MAX_LENGTH + 1)},
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
@sharkd_present
async def test_detail_decodes_single_frame(
self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions
) -> None:
link = _add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 123456, _icmp_frame()),
@@ -182,9 +282,35 @@ class TestReplayRoutes:
assert body["source"]["frame_number"] == 1
assert body["hex"] == _icmp_frame().hex()
assert body["field_count"] > 10
assert "tshark" in body["tshark_version"].lower()
ip = next(p for p in body["tree"] if p.get("name") == "ip")
ttl = next(f for f in ip["children"] if f.get("name") == "ip.ttl")
# Values arrive as strings, exactly as tshark emitted them.
assert ttl["show"] == "64" and ttl["showname"] == "Time to Live: 64"
def find(node, name):
stack = node if isinstance(node, list) else [node]
for child in stack:
if child.get("name") == name:
return child
deep = find(child.get("children", []), name)
if deep is not None:
return deep
return None
ttl = find(body["tree"], "ip.ttl")
assert ttl["label"] == "Time to Live: 64"
assert ttl["filter_expr"] == "ip.ttl == 64"
assert ttl["pos"] == 22 and ttl["size"] == 1
@sharkd_present
async def test_detail_501_when_sharkd_disappears(
self, app: FastAPI, client: AsyncClient, project: Project
) -> None:
link = _add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
(1693472000, 123456, _icmp_frame()),
])
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
response = await client.get(
app.url_path_for("replay_tag_frame_detail", project_id=project.id, tag=7),
params={"ts": "1693472000.123456", "node_id": "n1",
"link_id": link.id, "marker": "icmp"},
)
assert response.status_code == status.HTTP_501_NOT_IMPLEMENTED