From 5d0284e7be6dab5fa1a3218249419d19071d0ac4 Mon Sep 17 00:00:00 2001 From: YueGuobin Date: Mon, 15 Jun 2026 12:42:18 +0800 Subject: [PATCH] Performance: accelerate project opening with parallel link creation and batch UDP port allocation - Narrow IOU lock scope to only cover application_id allocation, allowing concurrent IOU node creation via Pool(concurrency=5) - Parallelize link creation during project.open() using Pool(concurrency=5) instead of sequential processing - Add batch UDP port allocation endpoint on compute to allocate N ports in a single HTTP call - Pre-allocate UDP ports per compute before link creation during project loading, reducing HTTP round-trips - UDPLink.create() falls back to individual port allocation if no pre-allocated port is available --- gns3server/api/routes/compute/compute.py | 17 +++ gns3server/controller/project.py | 145 ++++++++++++++++------- gns3server/controller/udp_link.py | 17 ++- 3 files changed, 131 insertions(+), 48 deletions(-) diff --git a/gns3server/api/routes/compute/compute.py b/gns3server/api/routes/compute/compute.py index ba6156fc0..6e8d4bb4a 100644 --- a/gns3server/api/routes/compute/compute.py +++ b/gns3server/api/routes/compute/compute.py @@ -55,6 +55,23 @@ def allocate_udp_port(project_id: UUID) -> dict: return {"udp_port": udp_port} +@router.post("/projects/{project_id}/ports/udp/batch", status_code=status.HTTP_201_CREATED) +def batch_allocate_udp_ports(project_id: UUID, body: dict) -> dict: + """ + Allocate multiple UDP ports on the compute in a single call. + + Used during project loading to pre-allocate all required UDP ports + before creating links, reducing HTTP round-trips. + """ + + count = body.get("count", 1) + pm = ProjectManager.instance() + project = pm.get_project(str(project_id)) + m = PortManager.instance() + udp_ports = [m.get_free_udp_port(project) for _ in range(count)] + return {"udp_ports": udp_ports} + + @router.get("/network/interfaces") def network_interfaces() -> List[dict]: """ diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 463ff5345..535e414a8 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -158,6 +158,7 @@ class Project: self.dump() self._iou_id_lock = asyncio.Lock() + self._preallocated_udp_ports = {} # compute_id -> list of pre-allocated UDP ports log.debug(f'Project "{self.name}" [{self._id}] loaded') self.emit_controller_notification("project.created", self.asdict()) @@ -216,6 +217,7 @@ class Project: self._load_snapshot_config() # Create the project on demand on the compute node self._project_created_on_compute = set() + self._preallocated_udp_ports = {} @property def scene_height(self): @@ -618,9 +620,9 @@ class Project: if node_type == "iou": async with self._iou_id_lock: - # wait for an IOU node to be completely created before adding a new one - # this is important otherwise we allocate the same application ID (used - # to generate MAC addresses) when creating multiple IOU node at the same time + # Only hold the lock while allocating a unique application ID to avoid + # serializing the HTTP call to the compute across all IOU nodes. + # Multiple IOU nodes can be created concurrently once IDs are assigned. if "properties" in kwargs.keys(): # allocate a new application id for nodes loaded from the project kwargs.get("properties")["application_id"] = get_next_application_id( @@ -629,7 +631,7 @@ class Project: elif "application_id" not in kwargs.keys() and not kwargs.get("properties"): # allocate a new application id for nodes added to the project kwargs["application_id"] = get_next_application_id(self._controller.projects, self._computes) - node = await self._create_node(compute, name, node_id, node_type, **kwargs) + node = await self._create_node(compute, name, node_id, node_type, **kwargs) else: node = await self._create_node(compute, name, node_id, node_type, **kwargs) self.emit_notification("node.created", node.asdict()) @@ -751,6 +753,56 @@ class Project: self.dump() self.emit_notification("drawing.deleted", drawing.asdict()) + async def _create_link_from_topology_data(self, link_data): + """ + Create a link from topology data (used during project loading). + + Extracted into a separate method so links can be created in parallel + via Pool() during project.open(). + + :param link_data: Link data from the topology file + """ + link = await self.add_link(link_id=link_data["link_id"]) + if "filters" in link_data: + try: + await link.update_filters(link_data["filters"]) + except ControllerError as e: + log.warning( + "Dropping invalid filters on link %s: %s", + link_data.get("link_id"), e + ) + if "link_style" in link_data: + await link.update_link_style(link_data["link_style"]) + if "show_filters_icon" in link_data: + await link.update_show_filters_icon(link_data["show_filters_icon"]) + for node_link in link_data.get("nodes", []): + node = self.get_node(node_link["node_id"]) + port = node.get_port(node_link["adapter_number"], node_link["port_number"]) + if port is None: + log.warning( + "Port {}/{} for {} not found".format( + node_link["adapter_number"], node_link["port_number"], node.name + ) + ) + continue + if port.link is not None: + log.warning( + "Port {}/{} is already connected to link ID {}".format( + node_link["adapter_number"], node_link["port_number"], port.link.id + ) + ) + continue + await link.add_node( + node, + node_link["adapter_number"], + node_link["port_number"], + label=node_link.get("label"), + dump=False, + ) + if len(link.nodes) != 2: + # a link should have 2 attached nodes, this can happen with corrupted projects + await self.delete_link(link.id, force_delete=True) + @open_required async def add_link(self, link_id=None, dump=True): """ @@ -766,6 +818,35 @@ class Project: self.dump() return link + async def preallocate_udp_ports_for_compute(self, compute, count): + """ + Pre-allocate UDP ports from a compute in a single batch call. + + Used during project loading to reduce HTTP round-trips when + creating many links. + + :param compute: Compute instance + :param count: Number of UDP ports to pre-allocate + """ + if count <= 0: + return + response = await compute.post(f"/projects/{self._id}/ports/udp/batch", data={"count": count}) + ports = response.json["udp_ports"] + self._preallocated_udp_ports.setdefault(compute.id, []) + self._preallocated_udp_ports[compute.id].extend(ports) + + def pop_preallocated_udp_port(self, compute_id): + """ + Pop a pre-allocated UDP port for a compute. + + :param compute_id: Compute ID + :returns: UDP port number or None if no pre-allocated port is available + """ + ports = self._preallocated_udp_ports.get(compute_id, []) + if ports: + return ports.pop(0) + return None + @open_required async def delete_link(self, link_id, force_delete=False): link = self.get_link(link_id) @@ -1225,50 +1306,26 @@ class Project: for compute, name, node_id, node_data in nodes_to_create: pool.append(self.add_node, compute, name, node_id, dump=False, **node_data) await pool.join() + # Pre-allocate UDP ports for all links in batch to reduce HTTP round-trips + ports_per_compute = {} for link_data in topology.get("links", []): if "link_id" not in link_data.keys(): - # skip the link continue - link = await self.add_link(link_id=link_data["link_id"]) - if "filters" in link_data: - try: - await link.update_filters(link_data["filters"]) - except ControllerError as e: - log.warning( - "Dropping invalid filters on link %s: %s", - link_data.get("link_id"), e - ) - if "link_style" in link_data: - await link.update_link_style(link_data["link_style"]) - if "show_filters_icon" in link_data: - await link.update_show_filters_icon(link_data["show_filters_icon"]) for node_link in link_data.get("nodes", []): - node = self.get_node(node_link["node_id"]) - port = node.get_port(node_link["adapter_number"], node_link["port_number"]) - if port is None: - log.warning( - "Port {}/{} for {} not found".format( - node_link["adapter_number"], node_link["port_number"], node.name - ) - ) - continue - if port.link is not None: - log.warning( - "Port {}/{} is already connected to link ID {}".format( - node_link["adapter_number"], node_link["port_number"], port.link.id - ) - ) - continue - await link.add_node( - node, - node_link["adapter_number"], - node_link["port_number"], - label=node_link.get("label"), - dump=False, - ) - if len(link.nodes) != 2: - # a link should have 2 attached nodes, this can happen with corrupted projects - await self.delete_link(link.id, force_delete=True) + node = self._nodes.get(node_link["node_id"]) + if node: + ports_per_compute[node.compute.id] = ports_per_compute.get(node.compute.id, 0) + 1 + for compute in self.computes: + count = ports_per_compute.get(compute.id, 0) + if count > 0: + await self.preallocate_udp_ports_for_compute(compute, count) + # Create links in parallel for improved performance + pool = Pool(concurrency=5) + for link_data in topology.get("links", []): + if "link_id" not in link_data.keys(): + continue + pool.append(self._create_link_from_topology_data, link_data) + await pool.join() for drawing_data in topology.get("drawings", []): await self.add_drawing(dump=False, **drawing_data) diff --git a/gns3server/controller/udp_link.py b/gns3server/controller/udp_link.py index 330e6a4f7..7c2e6f187 100644 --- a/gns3server/controller/udp_link.py +++ b/gns3server/controller/udp_link.py @@ -65,10 +65,19 @@ class UDPLink(Link): raise ControllerError(f"Cannot get an IP address on same subnet: {e}") # Reserve a UDP port on both side - response = await node1.compute.post(f"/projects/{self._project.id}/ports/udp") - self._node1_port = response.json["udp_port"] - response = await node2.compute.post(f"/projects/{self._project.id}/ports/udp") - self._node2_port = response.json["udp_port"] + # Try pre-allocated ports first (used during batch project loading) + port = self._project.pop_preallocated_udp_port(node1.compute.id) + if port is not None: + self._node1_port = port + else: + response = await node1.compute.post(f"/projects/{self._project.id}/ports/udp") + self._node1_port = response.json["udp_port"] + port = self._project.pop_preallocated_udp_port(node2.compute.id) + if port is not None: + self._node2_port = port + else: + response = await node2.compute.post(f"/projects/{self._project.id}/ports/udp") + self._node2_port = response.json["udp_port"] node1_filters, node2_filters = self._get_node_filters(node1, node2)