Fix: Check compute connectivity before open() during project deletion

Previously the disconnected compute check ran after open(), which would
block for 120s trying to connect to unreachable remote computes before
rejecting the deletion. Now reads the topology file directly to extract
compute IDs and checks connectivity before calling open(), enabling
immediate rejection of deletions involving offline computes.

Also removes the redundant post-open() check since the early check
covers both opened and closed project states.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
YueGuobin 2026-05-07 09:58:20 +08:00
parent 8ecd449dd8
commit 30ebde0cb0
No known key found for this signature in database

View File

@ -1022,6 +1022,16 @@ class Project:
async def delete(self):
# Check compute connectivity before open() to avoid 120s timeout
# when remote computes are unreachable
disconnected = self._get_disconnected_computes()
if disconnected:
compute_names = ", ".join([f"'{c.name}'" for c in disconnected])
raise ControllerForbiddenError(
f"Cannot delete project '{self.name}': {len(disconnected)} compute(s) are disconnected: {compute_names}. "
f"Please fix the connection or delete the project manually on those computes."
)
if self._status != "opened":
try:
await self.open()
@ -1029,28 +1039,6 @@ class Project:
# ignore missing images or other conflicts when deleting a project
log.warning(f"Conflict while deleting project: {e}")
# Check if all computes used by this project are connected before deletion
# We need to check from the topology file because _project_created_on_compute
# gets reset during open()
disconnected_computes = []
for compute_id in self._computes:
try:
compute = self._controller.get_compute(compute_id)
if not compute.connected:
disconnected_computes.append(compute)
except ControllerError:
# Compute doesn't exist anymore, consider it disconnected
log.warning(f"Compute '{compute_id}' not found in controller")
# We can't add it to disconnected_computes without the compute object
pass
if disconnected_computes:
compute_names = ", ".join([f"'{c.name}'" for c in disconnected_computes])
raise ControllerForbiddenError(
f"Cannot delete project '{self.name}': {len(disconnected_computes)} compute(s) are disconnected: {compute_names}. "
f"Please fix the connection or delete the project manually on those computes."
)
await self.delete_on_computes()
await self.close()
@ -1068,6 +1056,42 @@ class Project:
raise ControllerError(f"Cannot delete project directory {self.path}: {str(e)}")
self.emit_controller_notification("project.deleted", self.asdict())
def _get_disconnected_computes(self):
"""
Check compute connectivity by reading the topology file directly,
without opening the project (which would try to connect to computes).
Returns a list of disconnected Compute objects.
"""
if self._status == "opened":
# Project is already open, use the already-loaded _computes list
compute_ids = self._computes
else:
# Read compute IDs from topology file without connecting
path = self._topology_file()
if not os.path.exists(path):
return []
try:
project_data = load_topology(path)
except (ValueError, OSError) as e:
log.warning(f"Could not read topology file for project '{self._name}': {e}")
return []
topology = project_data.get("topology", {})
compute_ids = set()
for node in topology.get("nodes", []):
compute_id = node.get("compute_id")
if compute_id:
compute_ids.add(compute_id)
disconnected = []
for compute_id in compute_ids:
try:
compute = self._controller.get_compute(compute_id)
if not compute.connected:
disconnected.append(compute)
except ControllerError:
log.warning(f"Compute '{compute_id}' not found in controller")
return disconnected
async def delete_on_computes(self):
"""
Delete the project on computes but not on controller