mirror of
https://github.com/GNS3/gns3-server.git
synced 2026-11-03 22:11:14 +02:00
perf: batch NIO dispatch on project open (one HTTP per compute)
Project open used to create each link by issuing two NIO POSTs from the
controller to the compute — ~5000 HTTP round-trips for a 2500-link
topology, all funnelling through the single shared controller/compute
event loop and capping throughput near 12 links/s.
Replace it with a bulk path:
- UDPLink split into _prepare() (local: ports, peer addrs, link_data)
and _commit_nios() (dispatch). create() = prepare + commit (interactive).
- Link.add_node gains batch=True: attach both nodes without triggering
per-link NIO HTTP.
- compute: new POST /projects/{id}/nios/batch endpoint with a unified
_add_nio_binding dispatch across node types (docker/qemu/iou/vpcs/
builtin differ in signature).
- project.open: prepare all links locally, group NIO entries by compute,
send each compute a single /nios/batch, then finalise (wire node/port
refs, mark created, notify, apply marker defs) in parallel.
Cuts controller->compute HTTP from O(links) to O(computes). Test added
for the batch endpoint.
This commit is contained in:
7 files changed
+280
-29
No files matched your search
@@ -17,7 +17,6 @@
|
||||
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import logging
|
||||
|
||||
from .controller_error import ControllerError, ControllerNotFoundError
|
||||
@@ -88,9 +87,15 @@ class UDPLink(Link):
|
||||
"""
|
||||
return self._markers_for_node(node1), self._markers_for_node(node2)
|
||||
|
||||
async def create(self):
|
||||
async def _prepare(self):
|
||||
"""
|
||||
Create the link on the nodes
|
||||
Local-only link setup: resolve peer addresses, reserve UDP ports and
|
||||
build the two NIO tunnel specs (``self._link_data``). No NIO is sent to
|
||||
the computes — the caller decides how to dispatch them (one-by-one via
|
||||
:meth:`create`, or batched via the project-open bulk path).
|
||||
|
||||
:returns: list of two ``(node, adapter_number, port_number, nio_data)``
|
||||
tuples, ready to be POSTed to each node's compute.
|
||||
"""
|
||||
|
||||
node1 = self._nodes[0]["node"]
|
||||
@@ -101,12 +106,10 @@ class UDPLink(Link):
|
||||
port_number2 = self._nodes[1]["port_number"]
|
||||
|
||||
# Get an IP allowing communication between both host
|
||||
_t0 = time.perf_counter()
|
||||
try:
|
||||
(node1_host, node2_host) = await node1.compute.get_ip_on_same_subnet(node2.compute)
|
||||
except ValueError as e:
|
||||
raise ControllerError(f"Cannot get an IP address on same subnet: {e}")
|
||||
_t1 = time.perf_counter()
|
||||
|
||||
# Reserve a UDP port on both sides in parallel. Pre-allocated ports
|
||||
# (used during batch project loading) are popped from memory; otherwise
|
||||
@@ -121,7 +124,6 @@ class UDPLink(Link):
|
||||
self._node1_port, self._node2_port = await asyncio.gather(
|
||||
_allocate_port(node1.compute), _allocate_port(node2.compute)
|
||||
)
|
||||
_t2 = time.perf_counter()
|
||||
|
||||
node1_filters, node2_filters = self._get_node_filters(node1, node2)
|
||||
node1_markers, node2_markers = self._get_node_markers(node1, node2)
|
||||
@@ -151,17 +153,32 @@ class UDPLink(Link):
|
||||
}
|
||||
)
|
||||
|
||||
# Create the NIO tunnel on both sides in parallel. The two ends are
|
||||
# independent once the ports and peer addresses are known -- each node
|
||||
# talks to its own compute/uBridge with no shared lock between them --
|
||||
# so the two POSTs overlap. If either fails, roll back whichever side
|
||||
# succeeded before re-raising the first error.
|
||||
return [
|
||||
(node1, adapter_number1, port_number1, self._link_data[0]),
|
||||
(node2, adapter_number2, port_number2, self._link_data[1]),
|
||||
]
|
||||
|
||||
async def _commit_nios(self, entries):
|
||||
"""
|
||||
Send the two NIO tunnel POSTs in parallel and roll back on failure.
|
||||
|
||||
:param entries: the two ``(node, adapter_number, port_number, nio_data)``
|
||||
tuples returned by :meth:`_prepare`.
|
||||
"""
|
||||
|
||||
(node1, adapter_number1, port_number1, nio_data1), \
|
||||
(node2, adapter_number2, port_number2, nio_data2) = entries
|
||||
|
||||
# The two ends are independent once the ports and peer addresses are
|
||||
# known — each node talks to its own compute/uBridge with no shared
|
||||
# lock between them — so the two POSTs overlap. If either fails, roll
|
||||
# back whichever side succeeded before re-raising the first error.
|
||||
results = await asyncio.gather(
|
||||
node1.post(
|
||||
f"/adapters/{adapter_number1}/ports/{port_number1}/nio", data=self._link_data[0], timeout=120
|
||||
f"/adapters/{adapter_number1}/ports/{port_number1}/nio", data=nio_data1, timeout=120
|
||||
),
|
||||
node2.post(
|
||||
f"/adapters/{adapter_number2}/ports/{port_number2}/nio", data=self._link_data[1], timeout=120
|
||||
f"/adapters/{adapter_number2}/ports/{port_number2}/nio", data=nio_data2, timeout=120
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
@@ -179,12 +196,15 @@ class UDPLink(Link):
|
||||
if cleanup:
|
||||
await asyncio.gather(*cleanup, return_exceptions=True)
|
||||
raise errors[0]
|
||||
_t3 = time.perf_counter()
|
||||
log.info(
|
||||
"UDPLink.create timing get_ip=%.3fms ports=%.3fms nio=%.3fms total=%.3fms",
|
||||
1000 * (_t1 - _t0), 1000 * (_t2 - _t1), 1000 * (_t3 - _t2), 1000 * (_t3 - _t0)
|
||||
)
|
||||
self._created = True
|
||||
|
||||
async def create(self):
|
||||
"""
|
||||
Create the link on the nodes (interactive path: prepare + commit).
|
||||
"""
|
||||
|
||||
entries = await self._prepare()
|
||||
await self._commit_nios(entries)
|
||||
# New links automatically inherit every active project-level marker
|
||||
# definition so the user doesn't have to reconfigure.
|
||||
await self._project.apply_defs_to_new_link(self)
|
||||
|
||||
Reference in new issue
Block a user