feat: concurrent kernel captures, one per interface

uBridge's capture module now serves one AF_PACKET capture per interface,
any number per process, and `capture stop_kernel` takes the interface it
stops — this rides that (breaking, paired-binary) protocol change.

The server-side single-slot tracker (_kernel_capture_ifc with its
reserve/release/owned_by helpers) is gone: same-port duplicates were
already refused by the NIO capturing flag, uBridge answers EALREADY on a
same-interface duplicate, and per-interface stops removed the process-
wide stop the ownership gating guarded against.

The docker e2e drives ten concurrent captures on one node's uBridge and
asserts a per-interface stop freezes exactly the stopped half while the
live half keeps recording.
This commit is contained in:
YueGuobin committed 2026-10-14 21:52:34 +08:00
1 parent 149549e029
commit 9e0d5050db
15 files changed
+155 -171

No files matched your search

@@ -792,7 +792,7 @@ class TestEthernetSwitchNodesRoutes:
)
)
assert response.status_code == status.HTTP_204_NO_CONTENT
node._ubridge_send.assert_any_call("capture stop_kernel")
node._ubridge_send.assert_any_call(f'capture stop_kernel {anchor}')
async def test_ethernet_switch_batch_create_anchor_nio(
self, app: FastAPI, compute_client: AsyncClient, compute_project: Project, ethernet_switch: dict
@@ -997,14 +997,16 @@ async def test_stop_capture_kernel(vm):
await vm.stop_capture(0)
assert nio.capturing is False
vm._ubridge_send.assert_any_call("capture stop_kernel")
vm._ubridge_send.assert_any_call(f'capture stop_kernel {host_ifc}')
@pytest.mark.asyncio
async def test_start_capture_second_concurrent_fails(vm):
async def test_start_capture_ubridge_ealready_rolls_the_flag_back(vm):
"""
uBridge's kernel capture is a singleton per process: a second concurrent
capture returns EALREADY and must surface to the caller.
uBridge answers EALREADY when its interface is already being captured
(the server's per-port flag should have refused earlier; this is the
desync net): the error must surface and the port must not stay marked
as capturing.
"""
async def send(command):
@@ -1022,19 +1024,16 @@ async def test_start_capture_second_concurrent_fails(vm):
await vm.start_capture(0, "/tmp/capture.pcap")
# the failed start must not leave the port marked as capturing: a stuck
# flag refuses the retry and lets this port's stop kill the winner
# flag refuses the retry
assert nio.capturing is False
assert vm._kernel_capture_ifc is None
@pytest.mark.asyncio
async def test_start_capture_second_port_on_the_same_node_is_refused(vm):
async def test_capture_two_ports_at_once_and_stop_them_independently(vm):
"""
uBridge serves one kernel capture per process while the server models
captures per port: a second port's capture must be refused with a clear
error and its flag rolled back — otherwise its stop would issue the
process-wide, argument-less ``capture stop_kernel`` and kill the first
port's capture while its own flag still says capturing.
uBridge serves one kernel capture per interface: two ports of the same
node capture concurrently, and each port's stop names only its own
anchor — a sibling port's capture is never touched.
"""
vm._ubridge_hypervisor = MagicMock()
@@ -1051,48 +1050,22 @@ async def test_start_capture_second_port_on_the_same_node_is_refused(vm):
vm._ethernet_adapters[1].add_nio(0, nio2)
await vm.start_capture(0, "/tmp/capture.pcap")
with pytest.raises(DockerError, match="one kernel capture per node"):
await vm.start_capture(1, "/tmp/capture2.pcap")
await vm.start_capture(1, "/tmp/capture2.pcap")
assert nio1.capturing is True
assert nio2.capturing is False
assert nio2.capturing is True
vm._ubridge_send.assert_any_call(f'capture start_kernel {host_ifc1} "/tmp/capture.pcap"')
assert not any("stop_kernel" in str(c) for c in vm._ubridge_send.call_args_list)
@pytest.mark.asyncio
async def test_stop_capture_on_a_port_that_does_not_own_the_slot_skips_stop_kernel(vm):
"""
``capture stop_kernel`` is process-wide and takes no interface: a stop on
a port whose flag says capturing while another port owns the slot must
not reach uBridge — it would stop the owner's capture.
"""
vm._ubridge_hypervisor = MagicMock()
vm._ubridge_send = AsyncioMagicMock()
vm.status = "started"
vm._ethernet_adapters.append(type(vm._ethernet_adapters[0])())
host_ifc1, _ = vm._veth_names(0, 0)
host_ifc2, _ = vm._veth_names(1, 0)
vm._kernel_veths[(0, 0)] = host_ifc1
vm._kernel_veths[(1, 0)] = host_ifc2
nio1 = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE})
nio2 = vm.manager.create_nio({"type": "nio_bridge", "bridge": BRIDGE})
vm._ethernet_adapters[0].add_nio(0, nio1)
vm._ethernet_adapters[1].add_nio(0, nio2)
await vm.start_capture(0, "/tmp/capture.pcap") # port 0 owns the slot
nio2.start_packet_capture("/tmp/capture2.pcap") # a stale second flag
vm._ubridge_send.assert_any_call(f'capture start_kernel {host_ifc2} "/tmp/capture2.pcap"')
await vm.stop_capture(1)
assert nio1.capturing is True
assert nio2.capturing is False
assert not any("stop_kernel" in str(c) for c in vm._ubridge_send.call_args_list)
vm._ubridge_send.assert_any_call(f'capture stop_kernel {host_ifc2}')
assert not any(f'stop_kernel {host_ifc1}' in str(c) for c in vm._ubridge_send.call_args_list)
# the owner's capture is untouched and still stoppable
await vm.stop_capture(0)
vm._ubridge_send.assert_any_call("capture stop_kernel")
vm._ubridge_send.assert_any_call(f'capture stop_kernel {host_ifc1}')
@pytest.mark.asyncio
@@ -581,7 +581,7 @@ async def test_capture_on_a_kernel_link_uses_the_anchor(router):
assert not any("bind_filter" in c for c in _hypervisor_commands(router))
await router.stop_capture(0, 0)
router._ubridge_send.assert_any_call("capture stop_kernel")
router._ubridge_send.assert_any_call(f'capture stop_kernel {TAP00}')
@pytest.mark.asyncio
@@ -371,7 +371,7 @@ async def test_capture_on_a_kernel_link_uses_the_anchor(vm):
vm._ubridge_send.assert_any_call(f'capture start_kernel {TAP00} "/tmp/test.pcap"')
await vm.stop_capture(0, 0)
vm._ubridge_send.assert_any_call("capture stop_kernel")
vm._ubridge_send.assert_any_call(f'capture stop_kernel {TAP00}')
@pytest.mark.asyncio
@@ -328,7 +328,7 @@ async def test_capture_on_a_kernel_link_uses_the_anchor(vm):
vm._ubridge_send.assert_any_call(f'capture start_kernel {TAP0} "/tmp/test.pcap"')
await vm.stop_capture(0)
vm._ubridge_send.assert_any_call("capture stop_kernel")
vm._ubridge_send.assert_any_call(f'capture stop_kernel {TAP0}')
@pytest.mark.asyncio
+102
View File
@@ -316,6 +316,108 @@ def test_docker_kernel_fast_path():
assert harness.wait_until(lambda: harness.bridge_members(bridge) is None, timeout=10), bridge
def test_docker_kernel_ten_concurrent_captures():
"""Ten kernel captures at once on one node's uBridge. The capture side
picks the first local started node, so all ten links capture on node 1's
single uBridge process — the scale the old one-capture-per-process limit
could never reach. Each link gets its own traffic (its own subnet on its
own eth), and a partial stop must leave the untouched captures running."""
server = harness.live_server(kernel=True)
compute = server.compute
image = harness.ensure_docker_image(server)
project = compute.create_project("docker-multicap-e2e")
pid = project["project_id"]
try:
daemon = harness.DockerDaemon()
nodes = []
for suffix, address in (("1", R1_IP), ("2", R2_IP)):
node = compute.create_docker_node(pid, f"MC-{suffix}", image, adapters=10)
harness.configure_docker_interfaces(node, address)
nodes.append(_resolve_container(daemon, node))
n1, n2 = nodes
c1, c2 = n1["container_id"], n2["container_id"]
n1_id, n2_id = n1["node_id"], n2["node_id"]
links = [compute.create_link(pid, (n1_id, i, 0), (n2_id, i, 0)) for i in range(10)]
assert all(link["kernel_datapath"] is True for link in links), links
print(".. starting containers")
for node in nodes:
compute.call("POST", f"/projects/{pid}/nodes/{node['node_id']}/start")
harness.docker_wait_address(daemon, c1, R1_IP)
harness.docker_wait_address(daemon, c2, R2_IP)
# ten captures before any of the per-link interfaces is addressed, so
# every later frame (up to and including the ARP each first ping
# triggers) lands in a pcap
captures = [
compute.call("POST", f"/projects/{pid}/links/{link['link_id']}/capture/start", {"data_link_type": "DLT_EN10MB"})
for link in links
]
# eth1..eth9: one subnet each, wired by hand inside the guests; the
# settle window lets the guests' address-add chatter (each v4 addr
# fires two IGMP reports ~1 s apart) land inside an open capture
# instead of a later snapshot
for i in range(1, 10):
for container_id, host in ((c1, 1), (c2, 2)):
code, output = daemon.exec(
container_id,
["sh", "-c", f"ip link set eth{i} up && ip addr add 10.20.{i}.{host}/24 dev eth{i}"],
)
assert code == 0, output
time.sleep(2)
def ping_link(i):
# link 0 rides the init.sh-addressed eth0; the rest their own subnet
if i == 0:
result = harness.docker_ping(daemon, c1, R2_IP, count=2)
assert result["loss"] == 0, result["raw"]
else:
code, output = daemon.exec(c1, ["ping", "-c", "2", "-W", "1", f"10.20.{i}.2"])
assert code == 0, output
for i in range(10):
ping_link(i)
time.sleep(0.5)
paths = [capture["capture_file_path"] for capture in captures]
first_counts = [harness.pcap_records(path)[0] for path in paths]
# 2 ARP + 2 echo requests + 2 replies is the floor per link
assert all(count >= 4 for count in first_counts), list(zip(paths, first_counts))
print(f".. all 10 pcaps captured concurrently: {first_counts}")
# stop half. A stop is synchronous (the reply follows the reader
# thread's join), so from its return that pcap can no longer grow —
# the post-stop counts are an exact-freeze baseline
for link in links[5:]:
compute.call("POST", f"/projects/{pid}/links/{link['link_id']}/capture/stop")
stopped_base = [harness.pcap_records(paths[i])[0] for i in range(5, 10)]
# fresh traffic on ALL segments: the stopped half must stay frozen
# even while its links carry frames, the live half must record them
for i in range(10):
ping_link(i)
time.sleep(0.5)
for i in range(5, 10):
assert harness.pcap_records(paths[i])[0] == stopped_base[i - 5], f"stopped pcap {i} grew"
for i in range(5):
assert harness.pcap_records(paths[i])[0] >= first_counts[i] + 4, f"live pcap {i} did not grow"
print(".. per-interface stop froze exactly the stopped half")
for link in links[:5]:
compute.call("POST", f"/projects/{pid}/links/{link['link_id']}/capture/stop")
assert all(
compute.call("GET", f"/projects/{pid}/links/{link['link_id']}")["capturing"] is False for link in links
)
except BaseException:
harness.release(server, pid, failed=True)
raise
harness.release(server, pid, failed=False)
def test_docker_kernel_filter_matrix():
"""One filter type at a time on a live kernel link, each asserted by its
effect on real traffic and its tc state. Types the uBridge on this