mirror of
https://github.com/GNS3/gns3-server.git
synced 2026-11-04 22:41:12 +02:00
feat: drive marker replay with resident sharkd sessions
sharkd (the Wireshark daemon) is now the single decode engine — the tshark/PDML path is gone, and without sharkd every replay endpoint returns 501 (no degraded mode: one engine, one rendering shape for the Web UI). - Frame entries gain packet-list columns from sharkd's frames RPC: src/dst/proto/info plus the Wireshark coloring hints bg/fg - range and frames accept ?filter=<display filter>, applied before counting and slicing; invalid expressions are 400 carrying sharkd's original text; filters travel as single argv-style elements, capped at 2000 chars; filtered frames keep their original pcap frame numbers - frame detail returns sharkd's protocol tree with keys renamed into the REST contract (element/label/name/filter_expr/pos+size/expert/ generated/children): a census-verified closed key set, values untouched, unknown keys passed through verbatim, Wireshark-internal hf ids dropped. filter_expr gives the UI click-to-filter; pos/size drives hex highlighting (hex still read straight from the pcap) - one resident 'sharkd -' session per source pcap: lazy spawn, /tmp scratch copy + scratch HOME (hardened profiles), per-request (mtime,size) validation with respawn, LRU bound, per-session lock, per-RPC timeout, bounded close Timeline backbone (gate, record-header scan, merge ordering, canonical ts strings, hex reads) stays plain Python — identity and ordering never depend on the engine.
This commit is contained in:
5 files changed
+1018
-401
No files matched your search
@@ -16,27 +16,39 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
HTTP-route tests for the tag replay endpoints: the tag gate (409 while any
|
||||
marker captures, 404 unknown tag), the merged timeline, window queries
|
||||
(empty window = success), and the lazy frame detail (ts guard, isomorphic
|
||||
JSON, 501 when tshark is unavailable).
|
||||
HTTP-route tests for the tag replay endpoints (sharkd edition): the tag gate
|
||||
(409 while any marker captures, 404 unknown tag), the merged timeline with
|
||||
columns, window queries (empty window = success), the display-filter
|
||||
parameter (400 with sharkd's error text on a bad expression), the lazy frame
|
||||
detail with the renamed sharkd tree, and 501 when sharkd is unavailable
|
||||
(hard engine requirement, no degraded mode).
|
||||
"""
|
||||
|
||||
import shutil
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from fastapi import FastAPI, status
|
||||
from httpx import AsyncClient
|
||||
|
||||
from gns3server.controller.project import Project
|
||||
from gns3server.controller.udp_link import UDPLink
|
||||
from gns3server.controller import marker_replay
|
||||
|
||||
from tests.controller.test_marker_replay import _write_pcap, _icmp_frame
|
||||
from tests.controller.test_marker_replay import _write_pcap, _icmp_frame, _tcp_syn_frame, _cols
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
tshark_present = pytest.mark.skipif(shutil.which("tshark") is None, reason="tshark not installed")
|
||||
sharkd_present = pytest.mark.skipif(shutil.which("sharkd") is None, reason="sharkd not installed")
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def no_residual_sessions():
|
||||
yield
|
||||
manager = marker_replay._manager
|
||||
if manager is not None:
|
||||
await manager.close_all()
|
||||
|
||||
|
||||
def _add_marker(project, tag, enabled, node_id, frames=None):
|
||||
@@ -77,18 +89,48 @@ class TestReplayRoutes:
|
||||
)
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
|
||||
async def test_range_merges_sources_in_ts_order(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
|
||||
async def test_range_501_without_sharkd(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
|
||||
|
||||
# A non-empty pcap: the engine must be consulted, and without sharkd
|
||||
# the whole feature is unavailable (hard requirement, no degraded mode).
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 123456, _icmp_frame()),
|
||||
])
|
||||
|
||||
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_range", project_id=project.id, tag=7)
|
||||
)
|
||||
assert response.status_code == status.HTTP_501_NOT_IMPLEMENTED
|
||||
# The app's HTTPException handler unifies the body as {"message": …}.
|
||||
assert "sharkd" in response.json()["message"]
|
||||
|
||||
async def test_range_merges_sources_in_ts_order(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch
|
||||
) -> None:
|
||||
|
||||
# Columns injected hermetically — the merge/order/tiebreak contract
|
||||
# must not depend on the engine being installed.
|
||||
r1, r2 = UDPLink(project), UDPLink(project)
|
||||
project._links.update({r1.id: r1, r2.id: r2})
|
||||
|
||||
def _wire(link, node_id, frames):
|
||||
link._markers["icmp"] = {"bpf": "icmp", "tag": 7, "enabled": False, "color": None,
|
||||
"highlight_duration": None, "capture_node_id": node_id,
|
||||
"direction": None, "data_link_type": "DLT_EN10MB"}
|
||||
_write_pcap(f"{project.markers_directory}/{node_id}_{link.id}_icmp.pcap", frames)
|
||||
|
||||
# r1→r2 captures at t1 and t3; r2→r3 captures at t2 and t3 (same µs
|
||||
# as source A's t3 — the tiebreak must keep both frames).
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 500000, b"a" * 60),
|
||||
(1693472002, 0, b"a" * 60),
|
||||
])
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n2", frames=[
|
||||
(1693472001, 0, b"b" * 60),
|
||||
(1693472002, 0, b"b" * 60),
|
||||
])
|
||||
_wire(r1, "n1", [(1693472000, 500000, b"a" * 60), (1693472002, 0, b"a" * 60)])
|
||||
_wire(r2, "n2", [(1693472001, 0, b"b" * 60), (1693472002, 0, b"b" * 60)])
|
||||
|
||||
async def fake_columns(pcap, filter_expr):
|
||||
name = pcap.rsplit("/", 1)[-1]
|
||||
src = "10.0.0.1" if name.startswith("n1") else "10.0.0.2"
|
||||
return {1: _cols(src=src), 2: _cols(src=src)}
|
||||
|
||||
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns)
|
||||
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_range", project_id=project.id, tag=7)
|
||||
@@ -105,16 +147,25 @@ class TestReplayRoutes:
|
||||
"1693472000.500000", "1693472001.000000",
|
||||
"1693472002.000000", "1693472002.000000",
|
||||
]
|
||||
# Wireshark-style columns ride along on every frame entry.
|
||||
assert body["frames"][0]["src"] == "10.0.0.1"
|
||||
assert body["frames"][1]["src"] == "10.0.0.2"
|
||||
assert body["frames"][0]["proto"] == "ICMP"
|
||||
assert len(body["sources"]) == 2
|
||||
|
||||
async def test_frames_window_miss_is_empty_success(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project
|
||||
self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch
|
||||
) -> None:
|
||||
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 0, b"a" * 60),
|
||||
])
|
||||
|
||||
async def fake_columns(pcap, filter_expr):
|
||||
return {1: _cols()}
|
||||
|
||||
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns)
|
||||
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_frames", project_id=project.id, tag=7),
|
||||
params={"ts": "1693472001.000000", "window_ms": 100},
|
||||
@@ -122,13 +173,20 @@ class TestReplayRoutes:
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json() == {"frames": []}
|
||||
|
||||
async def test_frames_window_hit(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
|
||||
async def test_frames_window_hit(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch
|
||||
) -> None:
|
||||
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 0, b"a" * 60),
|
||||
(1693472000, 150000, b"a" * 60),
|
||||
])
|
||||
|
||||
async def fake_columns(pcap, filter_expr):
|
||||
return {1: _cols(), 2: _cols()}
|
||||
|
||||
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns)
|
||||
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_frames", project_id=project.id, tag=7),
|
||||
params={"ts": "1693472000.000000", "window_ms": 150},
|
||||
@@ -151,22 +209,64 @@ class TestReplayRoutes:
|
||||
assert response.status_code == status.HTTP_404_NOT_FOUND
|
||||
assert "rebuilt" in response.json()["message"]
|
||||
|
||||
async def test_detail_501_without_tshark(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
|
||||
@sharkd_present
|
||||
async def test_range_columns_and_filter_end_to_end(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions
|
||||
) -> None:
|
||||
|
||||
link = _add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 123456, _icmp_frame()),
|
||||
(1693472001, 0, _tcp_syn_frame()),
|
||||
])
|
||||
|
||||
# Unfiltered: both frames with real engine columns.
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_range", project_id=project.id, tag=7)
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
frames = response.json()["frames"]
|
||||
assert [f["proto"] for f in frames] == ["ICMP", "TCP"]
|
||||
assert frames[0]["src"] == "10.0.0.1" and frames[0]["dst"] == "10.0.0.3"
|
||||
assert "Echo" in frames[0]["info"]
|
||||
assert frames[0]["frame_number"] == 1
|
||||
|
||||
# Display filter applied before counting: only the TCP frame survives.
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_range", project_id=project.id, tag=7),
|
||||
params={"filter": "tcp"},
|
||||
)
|
||||
body = response.json()
|
||||
assert body["frame_count"] == 1
|
||||
assert [f["frame_number"] for f in body["frames"]] == [2] # original pcap identity
|
||||
assert body["start"] == "1693472001.000000"
|
||||
|
||||
@sharkd_present
|
||||
async def test_range_invalid_filter_is_400_with_sharkd_text(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions
|
||||
) -> None:
|
||||
|
||||
_add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 123456, _icmp_frame()),
|
||||
])
|
||||
|
||||
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_frame_detail", project_id=project.id, tag=7),
|
||||
params={"ts": "1693472000.123456", "node_id": "n1",
|
||||
"link_id": link.id, "marker": "icmp"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_501_NOT_IMPLEMENTED
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_range", project_id=project.id, tag=7),
|
||||
params={"filter": "this is (not valid"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "filter" in response.json()["message"].lower()
|
||||
|
||||
@tshark_present
|
||||
async def test_detail_decodes_single_frame(self, app: FastAPI, client: AsyncClient, project: Project) -> None:
|
||||
# Oversized filters are rejected before reaching the engine.
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_range", project_id=project.id, tag=7),
|
||||
params={"filter": "x" * (marker_replay.FILTER_MAX_LENGTH + 1)},
|
||||
)
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
|
||||
@sharkd_present
|
||||
async def test_detail_decodes_single_frame(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions
|
||||
) -> None:
|
||||
|
||||
link = _add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 123456, _icmp_frame()),
|
||||
@@ -182,9 +282,35 @@ class TestReplayRoutes:
|
||||
assert body["source"]["frame_number"] == 1
|
||||
assert body["hex"] == _icmp_frame().hex()
|
||||
assert body["field_count"] > 10
|
||||
assert "tshark" in body["tshark_version"].lower()
|
||||
|
||||
ip = next(p for p in body["tree"] if p.get("name") == "ip")
|
||||
ttl = next(f for f in ip["children"] if f.get("name") == "ip.ttl")
|
||||
# Values arrive as strings, exactly as tshark emitted them.
|
||||
assert ttl["show"] == "64" and ttl["showname"] == "Time to Live: 64"
|
||||
def find(node, name):
|
||||
stack = node if isinstance(node, list) else [node]
|
||||
for child in stack:
|
||||
if child.get("name") == name:
|
||||
return child
|
||||
deep = find(child.get("children", []), name)
|
||||
if deep is not None:
|
||||
return deep
|
||||
return None
|
||||
|
||||
ttl = find(body["tree"], "ip.ttl")
|
||||
assert ttl["label"] == "Time to Live: 64"
|
||||
assert ttl["filter_expr"] == "ip.ttl == 64"
|
||||
assert ttl["pos"] == 22 and ttl["size"] == 1
|
||||
|
||||
@sharkd_present
|
||||
async def test_detail_501_when_sharkd_disappears(
|
||||
self, app: FastAPI, client: AsyncClient, project: Project
|
||||
) -> None:
|
||||
|
||||
link = _add_marker(project, tag=7, enabled=False, node_id="n1", frames=[
|
||||
(1693472000, 123456, _icmp_frame()),
|
||||
])
|
||||
|
||||
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
|
||||
response = await client.get(
|
||||
app.url_path_for("replay_tag_frame_detail", project_id=project.id, tag=7),
|
||||
params={"ts": "1693472000.123456", "node_id": "n1",
|
||||
"link_id": link.id, "marker": "icmp"},
|
||||
)
|
||||
assert response.status_code == status.HTTP_501_NOT_IMPLEMENTED
|
||||
@@ -16,36 +16,47 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Unit tests for the tag-keyed aggregate replay module (controller layer):
|
||||
Unit tests for the tag-keyed aggregate replay module (sharkd edition):
|
||||
|
||||
* pcap record-header scanning (ts extraction, truncated-tail tolerance,
|
||||
nanosecond-magic normalization) and raw-bytes reads for the hex view
|
||||
* the tag gate (404 unknown tag, 409 while any marker captures)
|
||||
* timeline assembly: cross-source merge ordered by (ts, source, frame number),
|
||||
same-microsecond tiebreak, missing-pcap sources, frame-cap degradation to
|
||||
per-second buckets
|
||||
* window queries (inclusive bounds, empty-window success)
|
||||
* the lazy frame detail: round-tripped-ts guard, and — where tshark is
|
||||
installed — the isomorphic PDML → JSON mapping with a round-trip check
|
||||
(element count and attribute coverage).
|
||||
* the tag gate (404 unknown tag, 409 while any marker captures) — engine-free
|
||||
* timeline assembly with injected columns: cross-source merge ordered by
|
||||
(ts, source, frame number), same-microsecond tiebreak, frame-cap
|
||||
degradation to buckets, display-filter application before count/slice
|
||||
* the tree key renaming (closed census key set, values untouched, unknown
|
||||
keys pass through verbatim, internal hf ids dropped)
|
||||
* the resident sharkd sessions — spawn/load/reuse, (mtime, size)
|
||||
invalidation, LRU bound — against the real sharkd where installed, plus
|
||||
the frame detail end-to-end (hex + renamed tree + filter expressions).
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from gns3server.controller.controller_error import ControllerError, ControllerNotFoundError
|
||||
from gns3server.controller.controller_error import (
|
||||
ControllerBadRequestError,
|
||||
ControllerError,
|
||||
ControllerNotFoundError,
|
||||
)
|
||||
from gns3server.controller import marker_replay
|
||||
from gns3server.controller.marker_replay import (
|
||||
FilterError,
|
||||
SharkdMissingError,
|
||||
build_timeline,
|
||||
decode_frame,
|
||||
query_frames,
|
||||
read_frame_bytes,
|
||||
scan_pcap_frames,
|
||||
_count_tree_nodes,
|
||||
_format_ts,
|
||||
_parse_ts,
|
||||
_rename_value,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
@@ -53,6 +64,8 @@ pytestmark = pytest.mark.asyncio
|
||||
PCAP_MAGIC_US = 0xA1B2C3D4
|
||||
PCAP_MAGIC_NS = 0xA1B23C4D
|
||||
|
||||
sharkd_present = pytest.mark.skipif(shutil.which("sharkd") is None, reason="sharkd not installed")
|
||||
|
||||
|
||||
def _write_pcap(path, frames, magic=PCAP_MAGIC_US, snaplen=65535):
|
||||
"""frames: list of (sec, frac, payload bytes); frac is µs (or ns for the ns magic)."""
|
||||
@@ -64,27 +77,39 @@ def _write_pcap(path, frames, magic=PCAP_MAGIC_US, snaplen=65535):
|
||||
f.write(payload)
|
||||
|
||||
|
||||
def _cksum(data):
|
||||
if len(data) % 2:
|
||||
data = data + b"\x00" # RFC 1071 odd-length padding
|
||||
s = 0
|
||||
for i in range(0, len(data), 2):
|
||||
s += (data[i] << 8) + data[i + 1]
|
||||
while s >> 16:
|
||||
s = (s & 0xFFFF) + (s >> 16)
|
||||
return (~s) & 0xFFFF
|
||||
|
||||
|
||||
def _icmp_frame():
|
||||
"""A minimal well-formed ICMP echo request (10.0.0.1 → 10.0.0.3)."""
|
||||
|
||||
def cksum(data):
|
||||
if len(data) % 2:
|
||||
data = data + b"\x00" # RFC 1071 odd-length padding
|
||||
s = 0
|
||||
for i in range(0, len(data), 2):
|
||||
s += (data[i] << 8) + data[i + 1]
|
||||
while s >> 16:
|
||||
s = (s & 0xFFFF) + (s >> 16)
|
||||
return (~s) & 0xFFFF
|
||||
|
||||
icmp = bytes([8, 0, 0, 0]) + struct.pack(">HHH", 1, 1, 0) + b"payload12"
|
||||
icmp = icmp[:2] + struct.pack(">H", cksum(icmp)) + icmp[4:]
|
||||
icmp = icmp[:2] + struct.pack(">H", _cksum(icmp)) + icmp[4:]
|
||||
ip0 = struct.pack(">BBHHHBBH4s4s", 0x45, 0, 20 + len(icmp), 1, 0, 64, 1, 0,
|
||||
bytes([10, 0, 0, 1]), bytes([10, 0, 0, 3]))
|
||||
ip = ip0[:10] + struct.pack(">H", cksum(ip0)) + ip0[12:]
|
||||
ip = ip0[:10] + struct.pack(">H", _cksum(ip0)) + ip0[12:]
|
||||
return bytes.fromhex("0200000000020200000000010800") + ip + icmp
|
||||
|
||||
|
||||
def _tcp_syn_frame():
|
||||
"""A minimal TCP SYN (10.0.0.1:472 → 10.0.0.3:22)."""
|
||||
|
||||
tcp = struct.pack(">HHIIBBHHH", 472, 22, 0, 0, 0x50, 0x02, 64240, 0, 0)
|
||||
ip0 = struct.pack(">BBHHHBBH4s4s", 0x45, 0, 20 + len(tcp), 2, 0, 64, 6, 0,
|
||||
bytes([10, 0, 0, 1]), bytes([10, 0, 0, 3]))
|
||||
ip = ip0[:10] + struct.pack(">H", _cksum(ip0)) + ip0[12:]
|
||||
tcp = tcp[:16] + struct.pack(">H", _cksum(ip0[12:] + tcp)) + tcp[18:]
|
||||
return bytes.fromhex("0200000000020200000000010800") + ip + tcp
|
||||
|
||||
|
||||
def _fake_project(tmp_path, markers, markers_dir=None):
|
||||
"""markers: the flat project.markers shape ({'link/name': {..., node_id}})."""
|
||||
|
||||
@@ -98,8 +123,26 @@ def _marker_entry(tag, enabled=True, node_id="node-1"):
|
||||
"node_id": node_id}
|
||||
|
||||
|
||||
def _fake_columns(monkeypatch, mapping):
|
||||
"""Inject canned sharkd columns: mapping pcap-basename → {frame#: cols or None}.
|
||||
|
||||
Frames absent from the dict get no columns; when the caller passes a
|
||||
filter, the injected set IS the matching set (mirroring the real engine).
|
||||
"""
|
||||
|
||||
async def fake_columns_for(pcap, filter_expr):
|
||||
return mapping.get(os.path.basename(pcap), {})
|
||||
|
||||
monkeypatch.setattr(marker_replay, "_columns_for", fake_columns_for)
|
||||
|
||||
|
||||
def _cols(src="10.0.0.1", dst="10.0.0.3", proto="ICMP", info="Echo (ping) request"):
|
||||
return {"src": src, "dst": dst, "proto": proto, "info": info,
|
||||
"bg": "ffffff", "fg": "000000"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# pcap scanning
|
||||
# pcap scanning (engine-free backbone)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestScanPcap:
|
||||
@@ -140,15 +183,15 @@ class TestScanPcap:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tag gate + timeline
|
||||
# Tag gate (engine-free — raised before any sharkd work)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGateAndTimeline:
|
||||
class TestGate:
|
||||
|
||||
async def test_unknown_tag_404(self, tmp_path):
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=1)})
|
||||
with pytest.raises(ControllerNotFoundError):
|
||||
build_timeline(project, tag=7)
|
||||
await build_timeline(project, tag=7)
|
||||
|
||||
async def test_gate_409_while_capturing(self, tmp_path):
|
||||
project = _fake_project(tmp_path, {
|
||||
@@ -156,9 +199,22 @@ class TestGateAndTimeline:
|
||||
"linkB/icmp": _marker_entry(tag=7, enabled=True, node_id="n2"),
|
||||
})
|
||||
with pytest.raises(ControllerError, match="linkB"):
|
||||
build_timeline(project, tag=7)
|
||||
await build_timeline(project, tag=7)
|
||||
|
||||
async def test_merge_orders_by_ts_with_stable_tiebreak(self, tmp_path):
|
||||
async def test_filter_length_capped(self, tmp_path, monkeypatch):
|
||||
_fake_columns(monkeypatch, {})
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False)})
|
||||
with pytest.raises(ControllerBadRequestError, match="too long"):
|
||||
await build_timeline(project, tag=7, filter_expr="x" * (marker_replay.FILTER_MAX_LENGTH + 1))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Timeline assembly (columns injected — no engine needed)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTimeline:
|
||||
|
||||
async def test_merge_orders_by_ts_with_stable_tiebreak(self, tmp_path, monkeypatch):
|
||||
# Two sources, deliberately interleaved in time, colliding on one µs.
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [
|
||||
(1693472000, 500000, b"a" * 60), # t1 sourceA
|
||||
@@ -168,12 +224,16 @@ class TestGateAndTimeline:
|
||||
(1693472001, 000000, b"b" * 60), # t2 sourceB
|
||||
(1693472002, 000000, b"b" * 60), # t3 sourceB — same µs as t3 sourceA
|
||||
])
|
||||
_fake_columns(monkeypatch, {
|
||||
"n1_linkA_icmp.pcap": {1: _cols(), 2: _cols()},
|
||||
"n2_linkB_icmp.pcap": {1: _cols(src="10.0.0.2"), 2: _cols(src="10.0.0.2")},
|
||||
})
|
||||
project = _fake_project(tmp_path, {
|
||||
"linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1"),
|
||||
"linkB/icmp": _marker_entry(tag=7, enabled=False, node_id="n2"),
|
||||
})
|
||||
|
||||
timeline = build_timeline(project, tag=7)
|
||||
timeline = await build_timeline(project, tag=7)
|
||||
assert timeline["frame_count"] == 4
|
||||
assert timeline["start"] == "1693472000.500000"
|
||||
assert timeline["end"] == "1693472002.000000"
|
||||
@@ -181,24 +241,42 @@ class TestGateAndTimeline:
|
||||
# Same-microsecond pair keeps both frames (a ts dict key would drop one).
|
||||
assert [f["ts"] for f in timeline["frames"]][2:] == ["1693472002.000000"] * 2
|
||||
assert [f["frame_number"] for f in timeline["frames"]] == [1, 1, 2, 2]
|
||||
# Columns ride along verbatim.
|
||||
assert timeline["frames"][0]["src"] == "10.0.0.1"
|
||||
assert timeline["frames"][1]["src"] == "10.0.0.2"
|
||||
assert timeline["frames"][0]["proto"] == "ICMP"
|
||||
assert {s["count"] for s in timeline["sources"]} == {2}
|
||||
|
||||
async def test_missing_pcap_is_zero_count_source(self, tmp_path):
|
||||
async def test_columns_missing_for_a_frame_still_lists_it(self, tmp_path, monkeypatch):
|
||||
# A frame the (injected) engine did not describe keeps its place with
|
||||
# null columns — the timeline backbone never depends on the engine.
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [(1693472000, 0, b"a" * 60)])
|
||||
_fake_columns(monkeypatch, {}) # engine describes nothing
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1")})
|
||||
|
||||
timeline = await build_timeline(project, tag=7)
|
||||
frame = timeline["frames"][0]
|
||||
assert frame["ts"] == "1693472000.000000"
|
||||
assert frame["src"] is None and frame["info"] is None
|
||||
|
||||
async def test_missing_pcap_is_zero_count_source(self, tmp_path, monkeypatch):
|
||||
_fake_columns(monkeypatch, {})
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False)})
|
||||
timeline = build_timeline(project, tag=7)
|
||||
timeline = await build_timeline(project, tag=7)
|
||||
assert timeline["frame_count"] == 0
|
||||
assert timeline["start"] is None and timeline["end"] is None
|
||||
assert timeline["frames"] == []
|
||||
assert timeline["sources"][0]["count"] == 0
|
||||
|
||||
async def test_over_cap_degrades_to_buckets(self, tmp_path):
|
||||
async def test_over_cap_degrades_to_buckets(self, tmp_path, monkeypatch):
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [
|
||||
(1693472000, 0, b"a" * 60), (1693472000, 500000, b"a" * 60),
|
||||
(1693472001, 0, b"a" * 60),
|
||||
])
|
||||
_fake_columns(monkeypatch, {"n1_linkA_icmp.pcap": {1: _cols(), 2: _cols(), 3: _cols()}})
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1")})
|
||||
|
||||
timeline = build_timeline(project, tag=7, frame_cap=2)
|
||||
timeline = await build_timeline(project, tag=7, frame_cap=2)
|
||||
assert timeline["truncated"] is True
|
||||
assert "frames" not in timeline
|
||||
assert timeline["buckets"] == [
|
||||
@@ -206,40 +284,180 @@ class TestGateAndTimeline:
|
||||
{"ts": "1693472001.000000", "count": 1},
|
||||
]
|
||||
|
||||
async def test_filter_applies_before_count_and_slice(self, tmp_path, monkeypatch):
|
||||
# Three frames; the injected "matching set" (what a real engine would
|
||||
# return for the filter) contains only frames 1 and 3.
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [
|
||||
(1693472000, 0, b"a" * 60),
|
||||
(1693472001, 0, b"a" * 60),
|
||||
(1693472002, 0, b"a" * 60),
|
||||
])
|
||||
_fake_columns(monkeypatch, {"n1_linkA_icmp.pcap": {1: _cols(), 3: _cols(proto="TCP")}})
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1")})
|
||||
|
||||
timeline = await build_timeline(project, tag=7, filter_expr="tcp")
|
||||
assert timeline["frame_count"] == 2
|
||||
assert timeline["start"] == "1693472000.000000"
|
||||
assert timeline["end"] == "1693472002.000000"
|
||||
# frame numbers keep their ORIGINAL pcap identity through the filter.
|
||||
assert [f["frame_number"] for f in timeline["frames"]] == [1, 3]
|
||||
assert timeline["sources"][0]["count"] == 2
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Window query
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestQueryFrames:
|
||||
|
||||
def _project(self, tmp_path):
|
||||
def _project(self, tmp_path, monkeypatch):
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [
|
||||
(1693472000, 0, b"a" * 60),
|
||||
(1693472000, 150000, b"a" * 60),
|
||||
(1693472005, 0, b"a" * 60),
|
||||
])
|
||||
_fake_columns(monkeypatch, {"n1_linkA_icmp.pcap": {i: _cols() for i in (1, 2, 3)}})
|
||||
return _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1")})
|
||||
|
||||
async def test_window_inclusive_bounds(self, tmp_path):
|
||||
result = query_frames(self._project(tmp_path), tag=7, ts="1693472000.000000", window_ms=150)
|
||||
async def test_window_inclusive_bounds(self, tmp_path, monkeypatch):
|
||||
result = await query_frames(self._project(tmp_path, monkeypatch), tag=7,
|
||||
ts="1693472000.000000", window_ms=150)
|
||||
assert [f["ts"] for f in result["frames"]] == ["1693472000.000000", "1693472000.150000"]
|
||||
|
||||
async def test_window_miss_is_empty_success(self, tmp_path):
|
||||
result = query_frames(self._project(tmp_path), tag=7, ts="1693472001.000000", window_ms=100)
|
||||
async def test_window_miss_is_empty_success(self, tmp_path, monkeypatch):
|
||||
result = await query_frames(self._project(tmp_path, monkeypatch), tag=7,
|
||||
ts="1693472001.000000", window_ms=100)
|
||||
assert result == {"frames": []}
|
||||
|
||||
async def test_limit_applies(self, tmp_path):
|
||||
result = query_frames(self._project(tmp_path), tag=7, ts="1693472000.000000",
|
||||
window_ms=150, limit=1)
|
||||
async def test_limit_applies(self, tmp_path, monkeypatch):
|
||||
result = await query_frames(self._project(tmp_path, monkeypatch), tag=7,
|
||||
ts="1693472000.000000", window_ms=150, limit=1)
|
||||
assert len(result["frames"]) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frame detail (tshark path)
|
||||
# Tree key renaming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
tshark_present = pytest.mark.skipif(shutil.which("tshark") is None, reason="tshark not installed")
|
||||
class TestRename:
|
||||
|
||||
async def test_renames_closed_key_set_and_drops_hf_id(self):
|
||||
node = {
|
||||
"t": "proto", "l": "Time to Live: 64", "fn": "ip.ttl",
|
||||
"f": "ip.ttl == 64", "h": [22, 1], "s": None, "g": False,
|
||||
"e": 8472,
|
||||
"n": [{"l": "nested", "h": [23, 2], "n": []}],
|
||||
}
|
||||
renamed = _rename_value(node)
|
||||
assert renamed == {
|
||||
"element": "proto", "label": "Time to Live: 64", "name": "ip.ttl",
|
||||
"filter_expr": "ip.ttl == 64", "pos": 22, "size": 1,
|
||||
"expert": None, "generated": False,
|
||||
"children": [{"label": "nested", "pos": 23, "size": 2, "children": []}],
|
||||
}
|
||||
|
||||
async def test_unknown_keys_pass_through_verbatim(self):
|
||||
# A future Wireshark adding a key must never silently lose data.
|
||||
node = {"l": "x", "future_key": {"deep": [1, 2]}, "n": []}
|
||||
renamed = _rename_value(node)
|
||||
assert renamed["future_key"] == {"deep": [1, 2]}
|
||||
|
||||
async def test_count_tree_nodes_counts_dicts_only(self):
|
||||
assert _count_tree_nodes({"a": [{"b": 1}, "str", 3]}) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sharkd sessions + engine-backed behaviour
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSessions:
|
||||
|
||||
async def test_missing_sharkd_raises_501_error(self, tmp_path):
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [(1693472000, 0, b"a" * 60)])
|
||||
project = _fake_project(tmp_path, {"linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1")})
|
||||
with patch("gns3server.controller.marker_replay.shutil.which", return_value=None):
|
||||
with pytest.raises(SharkdMissingError):
|
||||
await build_timeline(project, tag=7)
|
||||
|
||||
async def test_lru_bound_evicts_least_recently_used(self, tmp_path):
|
||||
manager = marker_replay._SharkdManager()
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, pcap, used):
|
||||
self.pcap, self.last_used = pcap, used
|
||||
self.closed = False
|
||||
self.mtime_ns, self.size = 0, 0
|
||||
|
||||
def matches(self, stat):
|
||||
return False # always respawn → exercises the eviction path
|
||||
|
||||
def alive(self):
|
||||
return False
|
||||
|
||||
def touch(self):
|
||||
pass
|
||||
|
||||
async def close(self):
|
||||
self.closed = True
|
||||
|
||||
fake_by_pcap = {}
|
||||
|
||||
async def fake_spawn(pcap, stat):
|
||||
session = FakeSession(pcap, 0)
|
||||
fake_by_pcap[pcap] = session
|
||||
return session
|
||||
|
||||
with patch.object(manager, "_spawn", side_effect=fake_spawn):
|
||||
for i in range(marker_replay.SESSION_MAX + 2):
|
||||
pcap = tmp_path / f"pcap{i}"
|
||||
_write_pcap(pcap, [(1693472000, 0, b"a" * 60)])
|
||||
await manager.session_for(str(pcap))
|
||||
# Bounded to SESSION_MAX; the earliest (least recently used) got evicted.
|
||||
assert len(manager._sessions) == marker_replay.SESSION_MAX
|
||||
assert fake_by_pcap[str(tmp_path / "pcap0")].closed is True
|
||||
assert fake_by_pcap[str(tmp_path / "pcap1")].closed is True
|
||||
|
||||
@sharkd_present
|
||||
async def test_real_session_columns_and_filter(self, tmp_path):
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [
|
||||
(1693472000, 123456, _icmp_frame()),
|
||||
(1693472001, 0, _tcp_syn_frame()),
|
||||
])
|
||||
manager = marker_replay._get_manager()
|
||||
try:
|
||||
columns = await marker_replay._columns_for(str(tmp_path / "n1_linkA_icmp.pcap"), None)
|
||||
assert columns[1]["src"] == "10.0.0.1"
|
||||
assert columns[1]["proto"] == "ICMP"
|
||||
assert "Echo" in columns[1]["info"]
|
||||
assert columns[2]["proto"] == "TCP"
|
||||
|
||||
only_tcp = await marker_replay._columns_for(str(tmp_path / "n1_linkA_icmp.pcap"), "tcp")
|
||||
assert set(only_tcp) == {2}
|
||||
finally:
|
||||
await manager.close_all()
|
||||
|
||||
@sharkd_present
|
||||
async def test_real_session_invalid_filter_raises_filter_error(self, tmp_path):
|
||||
_write_pcap(tmp_path / "n1_linkA_icmp.pcap", [(1693472000, 123456, _icmp_frame())])
|
||||
manager = marker_replay._get_manager()
|
||||
try:
|
||||
with pytest.raises(FilterError):
|
||||
await marker_replay._columns_for(str(tmp_path / "n1_linkA_icmp.pcap"), "this is (not valid")
|
||||
finally:
|
||||
await manager.close_all()
|
||||
|
||||
@sharkd_present
|
||||
async def test_real_session_respawns_on_stat_change(self, tmp_path):
|
||||
pcap = tmp_path / "n1_linkA_icmp.pcap"
|
||||
_write_pcap(pcap, [(1693472000, 123456, _icmp_frame())])
|
||||
manager = marker_replay._get_manager()
|
||||
try:
|
||||
first = await manager.session_for(str(pcap))
|
||||
assert await manager.session_for(str(pcap)) is first # warm reuse
|
||||
# Rewrite the file (simulating a truncated/rebuilt capture).
|
||||
_write_pcap(pcap, [(1693473000, 0, _icmp_frame())])
|
||||
second = await manager.session_for(str(pcap))
|
||||
assert second is not first
|
||||
columns = await marker_replay._columns_for(str(pcap), None)
|
||||
assert set(columns) == {1}
|
||||
finally:
|
||||
await manager.close_all()
|
||||
|
||||
|
||||
class TestDecodeFrame:
|
||||
@@ -264,83 +482,65 @@ class TestDecodeFrame:
|
||||
await decode_frame(project, tag=7, ts="1693472000.123456",
|
||||
node_id="nobody", link_id="linkA", marker="icmp")
|
||||
|
||||
async def test_decode_feeds_tshark_a_scratch_copy(self, tmp_path):
|
||||
"""Hardened tshark profiles deny the project dir — tshark must read a
|
||||
/tmp copy (a real copy, not a symlink) that is unlinked afterwards."""
|
||||
|
||||
import tempfile
|
||||
from unittest.mock import patch, AsyncMock
|
||||
|
||||
observed = []
|
||||
PDML = (b'<pdml><packet><proto name="frame" showname="Frame 1: 60 bytes">'
|
||||
b'<field name="frame.len" show="60" showname="Frame Length: 60"/>'
|
||||
b'</proto></packet></pdml>')
|
||||
|
||||
class FakeProc:
|
||||
returncode = 0
|
||||
|
||||
async def communicate(self):
|
||||
return PDML, b""
|
||||
|
||||
async def fake_exec(*args, **kwargs):
|
||||
r_index = args.index("-r")
|
||||
observed.append((args[r_index + 1], kwargs.get("env")))
|
||||
return FakeProc()
|
||||
|
||||
project = self._project(tmp_path)
|
||||
with patch("gns3server.controller.marker_replay.shutil.which", return_value="tshark"), \
|
||||
patch("gns3server.controller.marker_replay._tshark_version", AsyncMock(return_value="tshark 4.6.7")), \
|
||||
patch("gns3server.controller.marker_replay.asyncio.create_subprocess_exec", side_effect=fake_exec):
|
||||
@sharkd_present
|
||||
async def test_decode_end_to_end(self, tmp_path):
|
||||
manager = marker_replay._get_manager()
|
||||
try:
|
||||
project = self._project(tmp_path)
|
||||
detail = await decode_frame(project, tag=7, ts="1693472000.123456",
|
||||
node_id="n1", link_id="linkA", marker="icmp")
|
||||
|
||||
assert detail["field_count"] == 2 # proto + field from the canned PDML
|
||||
(scratch, env), = observed
|
||||
original = str(tmp_path / "n1_linkA_icmp.pcap")
|
||||
assert scratch != original
|
||||
assert scratch.startswith(tempfile.gettempdir()) and scratch.endswith(".pcap")
|
||||
assert env["HOME"] == tempfile.gettempdir()
|
||||
assert not os.path.exists(scratch) # cleaned up after the decode
|
||||
|
||||
@tshark_present
|
||||
async def test_decode_isomorphic_mapping(self, tmp_path):
|
||||
import asyncio
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
project = self._project(tmp_path)
|
||||
detail = await decode_frame(project, tag=7, ts="1693472000.123456",
|
||||
node_id="n1", link_id="linkA", marker="icmp")
|
||||
finally:
|
||||
await manager.close_all()
|
||||
|
||||
assert detail["source"]["frame_number"] == 1
|
||||
assert detail["hex"] == _icmp_frame().hex()
|
||||
assert detail["field_count"] > 0
|
||||
assert "tshark" in detail["tshark_version"].lower()
|
||||
assert detail["field_count"] > 10
|
||||
|
||||
# Round-trip fidelity: node count equals the PDML element count
|
||||
# (protos + fields, excluding the <packet> container itself)…
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"tshark", "-r", str(tmp_path / "n1_linkA_icmp.pcap"), "-T", "pdml",
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
stdout, _ = await proc.communicate()
|
||||
packet = ET.fromstring(stdout).find("./packet")
|
||||
xml_elements = [e for e in packet.iter() if e is not packet]
|
||||
assert detail["field_count"] == len(xml_elements)
|
||||
# Renamed tree: every label/name present, filter expressions with
|
||||
# values baked in, byte ranges for hex highlighting.
|
||||
def find(node, name):
|
||||
for child in node if isinstance(node, list) else node.get("children", []):
|
||||
if child.get("name") == name:
|
||||
return child
|
||||
deep = find(child, name)
|
||||
if deep is not None:
|
||||
return deep
|
||||
return None
|
||||
|
||||
# …and every XML attribute survives verbatim as a JSON string key.
|
||||
def walk(element, node):
|
||||
for key, value in element.attrib.items():
|
||||
assert node.get(key) == value
|
||||
assert all(isinstance(v, str) for k, v in node.items() if k != "children")
|
||||
for child, child_node in zip(element, node["children"]):
|
||||
walk(child, child_node)
|
||||
ttl = find(detail["tree"], "ip.ttl")
|
||||
assert ttl is not None
|
||||
assert ttl["label"] == "Time to Live: 64"
|
||||
assert ttl["filter_expr"] == "ip.ttl == 64"
|
||||
assert ttl["pos"] == 22 and ttl["size"] == 1
|
||||
|
||||
for element, node in zip(packet, detail["tree"]):
|
||||
walk(element, node)
|
||||
@sharkd_present
|
||||
async def test_key_census_closed_set(self, tmp_path):
|
||||
"""The rename correctness guarantee: across protocol-diverse real
|
||||
trees, sharkd's raw key set stays within the census-known keys."""
|
||||
|
||||
# Values stay strings (no numeric re-typing).
|
||||
ttl = next(
|
||||
f for p in detail["tree"] if p.get("name") == "ip"
|
||||
for f in p["children"] if f.get("name") == "ip.ttl"
|
||||
)
|
||||
assert ttl["show"] == "64" and ttl["showname"] == "Time to Live: 64"
|
||||
pcap = tmp_path / "proto_mix.pcap"
|
||||
_write_pcap(pcap, [
|
||||
(1693472000, 100000, _icmp_frame()),
|
||||
(1693472001, 0, _tcp_syn_frame()),
|
||||
])
|
||||
manager = marker_replay._get_manager()
|
||||
try:
|
||||
session = await manager.session_for(str(pcap))
|
||||
known = set(marker_replay._KEY_RENAME) | {"h", "e"}
|
||||
seen = set()
|
||||
|
||||
def walk(value):
|
||||
if isinstance(value, dict):
|
||||
seen.update(value.keys())
|
||||
for item in value.values():
|
||||
walk(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
walk(item)
|
||||
|
||||
for frame_number in (1, 2):
|
||||
result = await session.rpc("frame", {"frame": frame_number, "proto": True})
|
||||
walk(result.get("tree", []))
|
||||
assert seen <= known, f"unknown sharkd keys appeared: {seen - known}"
|
||||
finally:
|
||||
await manager.close_all()
|
||||
Reference in new issue
Block a user