diff --git a/docs/features/marker-tag-replay.md b/docs/features/marker-tag-replay.md index 033d25b79..b29273635 100644 --- a/docs/features/marker-tag-replay.md +++ b/docs/features/marker-tag-replay.md @@ -124,8 +124,8 @@ without it. | Method | Path | Description | |--------|------|-------------| -| GET | `/v3/projects/{pid}/markers/tags/{tag}/replay/range[?filter=]` | Timeline metadata + full merged frame list with packet-list columns | -| GET | `/v3/projects/{pid}/markers/tags/{tag}/replay/frames?ts=&window_ms=&limit=[&filter=]` | Frames with ts in `[T, T+window]`, merged across sources | +| GET | `/v3/projects/{pid}/markers/tags/{tag}/replay/range[?filter=&link=]` | Timeline metadata + full merged frame list with packet-list columns | +| GET | `/v3/projects/{pid}/markers/tags/{tag}/replay/frames?ts=&window_ms=&limit=[&filter=&link=]` | Frames with ts in `[T, T+window]`, merged across sources | | GET | `/v3/projects/{pid}/markers/tags/{tag}/replay/frame/detail?ts=&node_id=&link_id=&marker=` | Single frame: protocol tree + raw hex (lazy — one call per frame the user opens) | ### `range` — the timeline @@ -167,12 +167,26 @@ without it. `?filter=` on both `range` and `frames` is a Wireshark display filter, applied **before** counting and slicing — `start` / `end` / `frame_count` / -`frames` | `buckets` and the per-source `sources[].count` are all computed on the -matching frames only. Filtered frames keep their original pcap frame numbers. The filter -travels as one argv-style element (never through a shell) and is capped at 2000 -characters. An invalid expression is a **400** whose message carries sharkd's original -error text — suitable for inline display in the filter bar, and distinct from the 409 -gate / 404 unknown-tag semantics. +`frames` | `buckets` are all computed on the matching frames only. Filtered frames keep +their original pcap frame numbers. The filter travels as one argv-style element (never +through a shell) and is capped at 2000 characters. An invalid expression is a **400** +whose message carries sharkd's original error text — suitable for inline display in the +filter bar, and distinct from the 409 gate / 404 unknown-tag semantics. + +### Capture-source selection + +`?link=` on both `range` and `frames` narrows the frame stream to one +capture source — a pure identity filter applied **before** any engine work (only the +selected link's pcap gets a sharkd pass), AND-composing with `filter`. Windows and the +histogram therefore always agree with the link-filtered view. Two boundaries by +design: + +- **`sources` is the stable inventory of the tag**: every capture source is always + listed, with engine-free **total** counts, unaffected by `link` / `filter` — a + source dropdown must not shrink when the view narrows. +- **An unknown `link_id` matches nothing**: `frame_count: 0`, `start: null`, empty + `frames` / `buckets` — the same shape as a zero-match display filter, deliberately + not a 404. ### `frames` — point / window query (paging) diff --git a/gns3server/api/routes/controller/projects.py b/gns3server/api/routes/controller/projects.py index e1e4872cf..66fe27c2c 100644 --- a/gns3server/api/routes/controller/projects.py +++ b/gns3server/api/routes/controller/projects.py @@ -242,6 +242,7 @@ async def _replay_response(awaitable): async def replay_tag_range( tag: int, filter: Optional[str] = None, + link: Optional[str] = None, project: Project = Depends(dep_project), ) -> dict: """ @@ -259,12 +260,20 @@ async def replay_tag_range( counting and slicing — start / end / frame_count / frames | buckets are all computed on the matching frames only. An invalid expression is a 400 carrying sharkd's original error text (for inline display in the UI - filter bar). Requires sharkd — 501 without it. + filter bar). + + ``link`` narrows the frame stream to one capture source (link_id), + AND-composing with ``filter``; ``sources`` always lists the tag's full + inventory regardless. An unknown link_id yields an empty timeline (same + shape as a zero-match filter), not a 404. Requires sharkd — 501 without + it. Required privilege: Project.Audit """ - return await _replay_response(marker_replay.build_timeline(project, tag, filter_expr=filter)) + return await _replay_response( + marker_replay.build_timeline(project, tag, filter_expr=filter, link_id=link) + ) @router.get( @@ -277,6 +286,7 @@ async def replay_tag_frames( window_ms: int = 100, limit: int = 1000, filter: Optional[str] = None, + link: Optional[str] = None, project: Project = Depends(dep_project), ) -> dict: """ @@ -285,15 +295,19 @@ async def replay_tag_frames( ``{"frames": []}``. The tag gate applies (409 while any marker captures). ``ts`` must be the exact string returned by the range response — never - re-serialize it through a float. ``filter`` (optional display filter) has - the same semantics as on the range endpoint. Requires sharkd — 501 - without it. + re-serialize it through a float. ``filter`` and ``link`` (optional + display filter / capture-source link_id) have the same semantics as on + the range endpoint — windows and the histogram always agree. Requires + sharkd — 501 without it. Required privilege: Project.Audit """ return await _replay_response( - marker_replay.query_frames(project, tag, ts, window_ms=window_ms, limit=limit, filter_expr=filter) + marker_replay.query_frames( + project, tag, ts, window_ms=window_ms, limit=limit, + filter_expr=filter, link_id=link, + ) ) diff --git a/gns3server/controller/marker_replay.py b/gns3server/controller/marker_replay.py index 13d3aba2f..5a0bdc945 100644 --- a/gns3server/controller/marker_replay.py +++ b/gns3server/controller/marker_replay.py @@ -500,7 +500,7 @@ def gate_tag(project, tag): return entries -async def _merged_frames(project, entries, filter_expr=None): +async def _merged_frames(project, entries, filter_expr=None, link_id=None): """ Scan every source pcap's record headers, ask sharkd for columns (and, with a filter, the matching set), and merge into one list sorted by @@ -508,6 +508,16 @@ async def _merged_frames(project, entries, filter_expr=None): can hit the same microsecond); the tiebreaker yields a stable, determined order instead of a fictional one. With a filter, only frames sharkd matched survive, keeping their original pcap frame numbers. + + ``link_id`` narrows the frame stream to one capture source **before** any + engine work (a pure identity filter — only the selected link's pcap gets + a sharkd pass) and AND-composes with ``filter_expr``. An unknown link + matches nothing: an empty stream, same shape as a zero-match display + filter — deliberately not a 404. + + ``sources`` is the stable inventory of the tag: EVERY capture source is + listed with engine-free total counts, unaffected by ``link_id`` / + ``filter_expr`` — the inventory must not shrink when the view narrows. """ markers_dir = project.markers_directory @@ -518,12 +528,16 @@ async def _merged_frames(project, entries, filter_expr=None): markers_dir, f"{entry['node_id']}_{entry['link_id']}_{entry['marker']}.pcap" ) frames = scan_pcap_frames(pcap) if os.path.exists(pcap) else [] + # Inventory first: every source, engine-free totals. + sources.append({**{k: entry[k] for k in ("node_id", "link_id", "marker", "data_link_type")}, + "count": len(frames)}) + if link_id and entry["link_id"] != link_id: + continue # link narrows the stream before any engine work if frames: columns = await _columns_for(pcap, filter_expr) else: columns = {} source_key = f"{entry['node_id']}_{entry['link_id']}_{entry['marker']}" - count = 0 for frame_number, (sec, usec, incl_len) in enumerate(frames, start=1): if filter_expr is not None and frame_number not in columns: continue @@ -544,9 +558,6 @@ async def _merged_frames(project, entries, filter_expr=None): "bg": cols.get("bg"), "fg": cols.get("fg"), }) - count += 1 - sources.append({**{k: entry[k] for k in ("node_id", "link_id", "marker", "data_link_type")}, - "count": count}) merged.sort(key=lambda f: (f["ts_us"], f["_source"], f["frame_number"])) for frame in merged: del frame["ts_us"] @@ -561,17 +572,18 @@ def _validate_filter(filter_expr): ) -async def build_timeline(project, tag, frame_cap=FRAME_LIST_CAP, filter_expr=None): +async def build_timeline(project, tag, frame_cap=FRAME_LIST_CAP, filter_expr=None, link_id=None): """ The ``range`` response: timeline bounds, per-source stats, and (under ``frame_cap``) the full merged frame list for one-request timeline layout. Over the cap the list is replaced by per-second buckets. With a - ``filter_expr`` every figure is computed on the matching frames only. + ``filter_expr`` and/or a ``link_id`` every figure is computed on the + matching frames only (``sources`` stays the full tag inventory). """ _validate_filter(filter_expr) entries = gate_tag(project, tag) - frames, sources = await _merged_frames(project, entries, filter_expr) + frames, sources = await _merged_frames(project, entries, filter_expr, link_id=link_id) response = { "tag": tag, @@ -595,15 +607,17 @@ async def build_timeline(project, tag, frame_cap=FRAME_LIST_CAP, filter_expr=Non return response -async def query_frames(project, tag, ts, window_ms=100, limit=1000, filter_expr=None): +async def query_frames(project, tag, ts, window_ms=100, limit=1000, filter_expr=None, link_id=None): """ - Frames with ts in ``[T, T+window_ms]`` merged across sources. A time with - no frames is a normal, successful answer — ``{"frames": []}``. + Frames with ts in ``[T, T+window_ms]`` merged across sources — narrowed + by ``filter_expr`` / ``link_id`` with the same semantics as the range + endpoint, so windowed seconds and the histogram always agree. A time + with no frames is a normal, successful answer — ``{"frames": []}``. """ _validate_filter(filter_expr) entries = gate_tag(project, tag) - frames, _sources = await _merged_frames(project, entries, filter_expr) + frames, _sources = await _merged_frames(project, entries, filter_expr, link_id=link_id) start_us = _parse_ts(ts) end_us = start_us + max(window_ms, 0) * 1000 diff --git a/tests/api/routes/controller/test_marker_replay.py b/tests/api/routes/controller/test_marker_replay.py index 2629ec110..24fdf5c8e 100644 --- a/tests/api/routes/controller/test_marker_replay.py +++ b/tests/api/routes/controller/test_marker_replay.py @@ -209,6 +209,47 @@ class TestReplayRoutes: assert response.status_code == status.HTTP_404_NOT_FOUND assert "rebuilt" in response.json()["message"] + async def test_range_link_param_narrows_and_keeps_sources( + self, app: FastAPI, client: AsyncClient, project: Project, monkeypatch + ) -> None: + + r1, r2 = UDPLink(project), UDPLink(project) + project._links.update({r1.id: r1, r2.id: r2}) + + def _wire(link, node_id, frames): + link._markers["icmp"] = {"bpf": "icmp", "tag": 7, "enabled": False, "color": None, + "highlight_duration": None, "capture_node_id": node_id, + "direction": None, "data_link_type": "DLT_EN10MB"} + _write_pcap(f"{project.markers_directory}/{node_id}_{link.id}_icmp.pcap", frames) + + _wire(r1, "n1", [(1693472000, 0, b"a" * 60), (1693472002, 0, b"a" * 60)]) + _wire(r2, "n2", [(1693472001, 0, b"b" * 60)]) + + async def fake_columns(pcap, filter_expr): + return {1: _cols(), 2: _cols()} + + monkeypatch.setattr(marker_replay, "_columns_for", fake_columns) + + response = await client.get( + app.url_path_for("replay_tag_range", project_id=project.id, tag=7), + params={"link": r1.id}, + ) + body = response.json() + assert body["frame_count"] == 2 + assert {f["link_id"] for f in body["frames"]} == {r1.id} + # The source dropdown keeps the whole tag inventory. + assert sorted(s["count"] for s in body["sources"]) == [1, 2] + + # Unknown link: 200 with an empty timeline, same shape as a zero-match + # filter — not a 404. + response = await client.get( + app.url_path_for("replay_tag_range", project_id=project.id, tag=7), + params={"link": "00000000-0000-0000-0000-000000000000"}, + ) + body = response.json() + assert body["frame_count"] == 0 and body["frames"] == [] and body["start"] is None + assert len(body["sources"]) == 2 + @sharkd_present async def test_range_columns_and_filter_end_to_end( self, app: FastAPI, client: AsyncClient, project: Project, no_residual_sessions diff --git a/tests/controller/test_marker_replay.py b/tests/controller/test_marker_replay.py index 748c72d15..a2f992440 100644 --- a/tests/controller/test_marker_replay.py +++ b/tests/controller/test_marker_replay.py @@ -301,7 +301,77 @@ class TestTimeline: assert timeline["end"] == "1693472002.000000" # frame numbers keep their ORIGINAL pcap identity through the filter. assert [f["frame_number"] for f in timeline["frames"]] == [1, 3] - assert timeline["sources"][0]["count"] == 2 + # sources[] is the stable inventory: engine-free TOTAL counts, + # unaffected by link/filter (the WebUI source dropdown must not + # shrink when the view narrows). + assert timeline["sources"][0]["count"] == 3 + + +class TestLinkFilter: + """``link_id`` narrows the frame stream to one capture source before + counting/slicing/bucketing; sources stay the full inventory.""" + + def _project(self, tmp_path, monkeypatch, columns_override=None): + _write_pcap(tmp_path / "n1_linkA_icmp.pcap", [ + (1693472000, 500000, b"a" * 60), + (1693472002, 000000, b"a" * 60), + ]) + _write_pcap(tmp_path / "n2_linkB_icmp.pcap", [ + (1693472001, 000000, b"b" * 60), + (1693472002, 000000, b"b" * 60), + ]) + + async def fake_columns(pcap, filter_expr): + if columns_override is not None: + return await columns_override(os.path.basename(pcap), filter_expr) + src = "10.0.0.1" if "n1" in pcap else "10.0.0.2" + return {1: _cols(src=src), 2: _cols(src=src)} + + monkeypatch.setattr(marker_replay, "_columns_for", fake_columns) + return _fake_project(tmp_path, { + "linkA/icmp": _marker_entry(tag=7, enabled=False, node_id="n1"), + "linkB/icmp": _marker_entry(tag=7, enabled=False, node_id="n2"), + }) + + async def test_link_narrows_before_count_and_slice(self, tmp_path, monkeypatch): + timeline = await build_timeline(self._project(tmp_path, monkeypatch), tag=7, link_id="linkA") + assert timeline["frame_count"] == 2 + assert timeline["start"] == "1693472000.500000" + assert [f["link_id"] for f in timeline["frames"]] == ["linkA", "linkA"] + # sources stay the FULL inventory with engine-free totals. + assert sorted((s["link_id"], s["count"]) for s in timeline["sources"]) == [ + ("linkA", 2), ("linkB", 2) + ] + + async def test_unknown_link_is_empty_success(self, tmp_path, monkeypatch): + timeline = await build_timeline(self._project(tmp_path, monkeypatch), tag=7, link_id="nope") + assert timeline["frame_count"] == 0 + assert timeline["start"] is None and timeline["end"] is None + assert timeline["frames"] == [] + assert len(timeline["sources"]) == 2 + + async def test_link_and_filter_compose_as_and(self, tmp_path, monkeypatch): + # The injected "matching set" contains only frame 2 per source — + # combined with link=linkA the view is exactly that one frame. + async def override(basename, filter_expr): + assert filter_expr == "tcp" + return {2: _cols(proto="TCP")} + + project = self._project(tmp_path, monkeypatch, columns_override=override) + timeline = await build_timeline(project, tag=7, filter_expr="tcp", link_id="linkA") + assert timeline["frame_count"] == 1 + assert timeline["frames"][0]["frame_number"] == 2 + assert timeline["frames"][0]["link_id"] == "linkA" + + async def test_empty_link_string_means_absent(self, tmp_path, monkeypatch): + timeline = await build_timeline(self._project(tmp_path, monkeypatch), tag=7, link_id="") + assert timeline["frame_count"] == 4 + + async def test_query_frames_window_over_link_stream(self, tmp_path, monkeypatch): + project = self._project(tmp_path, monkeypatch) + result = await query_frames(project, tag=7, ts="1693472002.000000", + window_ms=0, link_id="linkB") + assert [f["link_id"] for f in result["frames"]] == ["linkB"] class TestQueryFrames: