feat: Node file streaming, recursive listing, file type detection, and file delete

- Stream file GET/POST through controller without buffering in memory
- Add recursive and subdirectory filtering to node file listing
- Replace file extension with magic-based file type detection
- Add DELETE endpoint for node and project files
- Include directories in listing response
- Add params and stream support to http_query
- Fix lambda closures, streamer exception scope, and delete error codes

gns3server/api/routes/compute/projects.py

@@ -19,13 +19,14 @@ API routes for projects.
 """
 
 import os
+import shutil
 import urllib.parse
 
 import logging
 
 log = logging.getLogger()
 
-from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+from fastapi import APIRouter, Depends, HTTPException, Request, Response, status, Query
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import FileResponse
 from typing import List
@@ -129,59 +130,6 @@ async def delete_compute_project(project: Project = Depends(dep_project)) -> Non
     await project.delete()
     ProjectManager.instance().remove_project(project.id)
 
-# @Route.get(
-#     r"/projects/{project_id}/notifications",
-#     description="Receive notifications about the project",
-#     parameters={
-#         "project_id": "Project UUID",
-#     },
-#     status_codes={
-#         200: "End of stream",
-#         404: "The project doesn't exist"
-#     })
-# async def notification(request, response):
-#
-#     pm = ProjectManager.instance()
-#     project = pm.get_project(request.match_info["project_id"])
-#
-#     response.content_type = "application/json"
-#     response.set_status(200)
-#     response.enable_chunked_encoding()
-#
-#     response.start(request)
-#     queue = project.get_listen_queue()
-#     ProjectHandler._notifications_listening.setdefault(project.id, 0)
-#     ProjectHandler._notifications_listening[project.id] += 1
-#     await response.write("{}\n".format(json.dumps(ProjectHandler._getPingMessage())).encode("utf-8"))
-#     while True:
-#         try:
-#             (action, msg) = await asyncio.wait_for(queue.get(), 5)
-#             if hasattr(msg, "asdict"):
-#                 msg = json.dumps({"action": action, "event": msg.asdict()}, sort_keys=True)
-#             else:
-#                 msg = json.dumps({"action": action, "event": msg}, sort_keys=True)
-#             log.debug("Send notification: %s", msg)
-#             await response.write(("{}\n".format(msg)).encode("utf-8"))
-#         except asyncio.TimeoutError:
-#             await response.write("{}\n".format(json.dumps(ProjectHandler._getPingMessage())).encode("utf-8"))
-#     project.stop_listen_queue(queue)
-#     if project.id in ProjectHandler._notifications_listening:
-#         ProjectHandler._notifications_listening[project.id] -= 1
-
-# def _getPingMessage(cls):
-#     """
-#     Ping messages are regularly sent to the client to
-#     keep the connection open. We send with it some information about server load.
-#
-#     :returns: hash
-#     """
-#     stats = {}
-#     # Non blocking call in order to get cpu usage. First call will return 0
-#     stats["cpu_usage_percent"] = CpuPercent.get(interval=None)
-#     stats["memory_usage_percent"] = psutil.virtual_memory().percent
-#     stats["disk_usage_percent"] = psutil.disk_usage(get_default_project_directory()).percent
-#     return {"action": "ping", "event": stats}
-
 
 @router.get("/projects/{project_id}/files", response_model=List[schemas.ProjectFile])
 async def get_compute_project_files(project: Project = Depends(dep_project)) -> List[schemas.ProjectFile]:
@@ -196,14 +144,16 @@ async def get_compute_project_files(project: Project = Depends(dep_project)) ->
 async def get_compute_node_files(
     node_type: str,
     node_id: str,
-    project: Project = Depends(dep_project)
+    project: Project = Depends(dep_project),
+    path: str = Query("", description="Subdirectory path within node directory"),
+    recursive: bool = Query(False, description="Recursively list all files")
 ) -> List[schemas.NodeFile]:
     """
     Return files belonging to a specific node with detailed metadata.
     """
 
     node_path = f"project-files/{node_type}/{node_id}"
-    return await project.list_node_files(node_path)
+    return await project.list_node_files(node_path, subpath=path, recursive=recursive)
 
 
 @router.get("/projects/{project_id}/files/{file_path:path}")
@@ -255,3 +205,32 @@ async def write_compute_project_file(
         raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
     except PermissionError:
         raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
+
+
+@router.delete("/projects/{project_id}/files/{file_path:path}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_compute_project_file(
+        file_path: str,
+        project: Project = Depends(dep_project)
+) -> None:
+
+    file_path = urllib.parse.unquote(file_path)
+    path = os.path.normpath(file_path)
+
+    if not is_safe_path(path, project.path):
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
+
+    path = os.path.join(project.path, path)
+    if not os.path.exists(path):
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+
+    try:
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+    except FileNotFoundError:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND)
+    except PermissionError:
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN)
+    except OSError as e:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e))

gns3server/api/routes/controller/nodes.py

@@ -24,6 +24,7 @@ import ipaddress
 
 from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect, Request, Response, status, Query, HTTPException
 from fastapi.encoders import jsonable_encoder
+from fastapi.responses import StreamingResponse
 from fastapi.routing import APIRoute
 from typing import List, Callable, Optional
 from uuid import UUID
@@ -523,17 +524,30 @@ async def delete_disk_image(
 
 
 @router.get("/{node_id}/files", response_model=List[schemas.NodeFile], dependencies=[Depends(has_privilege("Node.Audit"))])
-async def list_node_files(node: Node = Depends(dep_node)) -> List[schemas.NodeFile]:
+async def list_node_files(
+    node: Node = Depends(dep_node),
+    path: str = Query("", description="Subdirectory path within node directory"),
+    recursive: bool = Query(False, description="Recursively list all files")
+) -> List[schemas.NodeFile]:
     """
     List files in a node directory with detailed metadata.
 
+    By default lists only the current directory level (non-recursive).
+    Use recursive=true for a full recursive listing.
+
     Required privilege: Node.Audit
     """
 
     node_type = node.node_type
+    url = f"/projects/{node.project.id}/nodes/{node_type}/{node.id}/files"
+    params = {}
+    if path:
+        params["path"] = path
+    if recursive:
+        params["recursive"] = "true"
     res = await node.compute.http_query(
-        "GET",
-        f"/projects/{node.project.id}/nodes/{node_type}/{node.id}/files",
+        "GET", url,
+        params=params if params else None,
         timeout=None
     )
     return res.json
@@ -550,14 +564,32 @@ async def get_file(file_path: str, node: Node = Depends(dep_node)) -> Response:
     path = force_unix_path(file_path)
 
     # Raise error if user try to escape
-    if path[0] == ".":
+    if path.startswith(".."):
         raise ControllerForbiddenError("It is forbidden to get a file outside the project directory")
 
     node_type = node.node_type
     path = f"/project-files/{node_type}/{node.id}/{path}"
 
-    res = await node.compute.http_query("GET", f"/projects/{node.project.id}/files{path}", timeout=None, raw=True)
-    return Response(res.body, media_type="application/octet-stream", status_code=res.status)
+    compute_resp = await node.compute.http_query(
+        "GET", f"/projects/{node.project.id}/files{path}",
+        timeout=None, stream=True
+    )
+
+    async def streamer():
+        try:
+            async for chunk in compute_resp.content.iter_chunked(65536):
+                yield chunk
+        except (IOError, OSError, asyncio.TimeoutError) as e:
+            log.error(f"Error streaming file '{path}' from compute: {e}")
+            raise
+        finally:
+            compute_resp.close()
+
+    return StreamingResponse(
+        streamer(),
+        media_type="application/octet-stream",
+        status_code=compute_resp.status,
+    )
 
 
 @router.post(
@@ -575,15 +607,43 @@ async def post_file(file_path: str, request: Request, node: Node = Depends(dep_n
     path = force_unix_path(file_path)
 
     # Raise error if user try to escape
-    if path[0] == ".":
+    if path.startswith(".."):
         raise ControllerForbiddenError("Cannot write outside the node directory")
 
     node_type = node.node_type
     path = f"/project-files/{node_type}/{node.id}/{path}"
 
-    data = await request.body()  # FIXME: are we handling timeout or large files correctly?
-    await node.compute.http_query("POST", f"/projects/{node.project.id}/files{path}", data=data, timeout=None, raw=True)
-    # FIXME: response with correct status code (from compute)
+    # Stream request body directly to compute node
+    await node.compute.http_query(
+        "POST", f"/projects/{node.project.id}/files{path}",
+        data=request.stream(), timeout=None
+    )
+
+
+@router.delete(
+    "/{node_id}/files/{file_path:path}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    dependencies=[Depends(has_privilege("Node.Modify"))]
+)
+async def delete_node_file(file_path: str, node: Node = Depends(dep_node)) -> None:
+    """
+    Delete a file from the node directory.
+
+    Required privilege: Node.Modify
+    """
+
+    path = force_unix_path(file_path)
+
+    if path.startswith(".."):
+        raise ControllerForbiddenError("It is forbidden to delete a file outside the project directory")
+
+    node_type = node.node_type
+    path = f"/project-files/{node_type}/{node.id}/{path}"
+
+    await node.compute.http_query(
+        "DELETE", f"/projects/{node.project.id}/files{path}",
+        timeout=None
+    )
 
 
 @router.websocket("/{node_id}/console/ws")

gns3server/compute/project.py

@@ -16,6 +16,7 @@
 
 import os
 import shutil
+import magic
 import asyncio
 import hashlib
 import datetime
@@ -424,65 +425,141 @@ class Project:
 
         return files
 
-    async def list_node_files(self, node_path: str):
+    async def list_node_files(self, node_path: str, subpath: str = "", recursive: bool = False):
         """
         List files in a specific node directory with detailed metadata.
 
         :param node_path: Relative path to node directory (e.g., "project-files/qemu/node-id")
+        :param subpath: Optional subdirectory path. Defaults to root of node directory.
+        :param recursive: If True, recursively list all files (use with caution on large directories).
         :returns: Array of files in the node directory with metadata
         """
 
         node_full_path = os.path.normpath(os.path.join(self.path, node_path))
+        subpath = subpath.lstrip("/")
 
-        # Security check: ensure the path is within the project directory
-        if not os.path.commonpath([node_full_path, self.path]) == self.path:
+        if subpath:
+            target_path = os.path.normpath(os.path.join(node_full_path, subpath))
+        else:
+            target_path = node_full_path
+
+        # Security check: ensure the path is within the node directory
+        if not os.path.commonpath([target_path, node_full_path]) == node_full_path:
             raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,
-                              detail="Path is outside the project directory")
-
-        if not os.path.exists(node_full_path):
+                              detail="Path is outside the node directory")
+        if not os.path.exists(target_path):
             raise HTTPException(status_code=status.HTTP_404_NOT_FOUND,
-                              detail="Node directory not found")
+                              detail="Path not found")
+
+        if recursive:
+            return await self._list_node_files_recursive(node_full_path, target_path,
+                                                         node_full_path if subpath else None)
+
+        # Non-recursive: list only the current directory level
+        files = []
+        for entry in os.scandir(target_path):
+            name = entry.name
+            rel_path = name if not subpath else os.path.join(subpath, name)
+            try:
+                stat_info = await wait_run_in_executor(lambda e=entry: e.stat())
+                is_dir = await wait_run_in_executor(lambda e=entry: e.is_dir())
+                if is_dir:
+                    try:
+                        created_at = datetime.datetime.fromtimestamp(stat_info.st_ctime).isoformat()
+                        modified_at = datetime.datetime.fromtimestamp(stat_info.st_mtime).isoformat()
+                    except (OSError, OverflowError, ValueError):
+                        created_at = modified_at = ""
+                    files.append({
+                        "path": rel_path,
+                        "size": stat_info.st_size,
+                        "created_at": created_at,
+                        "modified_at": modified_at,
+                        "file_type": "directory"
+                    })
+                else:
+                    if name.endswith(".ghost"):
+                        continue
+                    try:
+                        file_type = await wait_run_in_executor(
+                            lambda e=entry: magic.from_file(e.path, mime=False)
+                        )
+                    except Exception as e:
+                        log.warning(f"Error getting file type for '{rel_path}': {e}")
+                        file_type = ""
+                    try:
+                        created_at = datetime.datetime.fromtimestamp(stat_info.st_ctime).isoformat()
+                        modified_at = datetime.datetime.fromtimestamp(stat_info.st_mtime).isoformat()
+                    except (OSError, OverflowError, ValueError):
+                        created_at = modified_at = ""
+                    files.append({
+                        "path": rel_path,
+                        "size": stat_info.st_size,
+                        "created_at": created_at,
+                        "modified_at": modified_at,
+                        "file_type": file_type
+                    })
+            except OSError:
+                continue
+        return files
+
+    async def _list_node_files_recursive(self, node_full_path, start_path, base_path=None):
+        """
+        Recursively list all files and directories under start_path.
+        """
+        if base_path is None:
+            base_path = node_full_path
 
         files = []
-        try:
-            filenames = os.listdir(node_full_path)
-        except OSError as e:
-            log.error(f"Error listing node directory: {e}")
-            return files
-
-        for filename in filenames:
-            file_path = os.path.join(node_full_path, filename)
-            if not os.path.isfile(file_path) or filename.endswith(".ghost"):
-                continue
-
-            try:
-                # Get file stat information
-                stat_info = await wait_run_in_executor(os.stat, file_path)
-
-                # Get file extension
-                _, extension = os.path.splitext(filename)
-                extension = extension.lstrip('.')
-
-                # Format timestamps as ISO 8601
+        for dirpath, dirnames, filenames in os.walk(start_path, followlinks=False):
+            for dirname in dirnames:
+                dir_full_path = os.path.join(dirpath, dirname)
                 try:
-                    created_at = datetime.datetime.fromtimestamp(stat_info.st_ctime).isoformat()
-                    modified_at = datetime.datetime.fromtimestamp(stat_info.st_mtime).isoformat()
-                except (OSError, OverflowError, ValueError) as e:
-                    log.warning(f"Invalid timestamp for '{filename}': {e}")
-                    created_at = modified_at = ""
-
-                file_info = {
-                    "path": filename,
-                    "size": stat_info.st_size,
-                    "created_at": created_at,
-                    "modified_at": modified_at,
-                    "extension": extension
-                }
-                files.append(file_info)
-            except OSError as e:
-                log.warning(f"Error getting metadata for file '{filename}': {e}")
-                continue
+                    stat_info = await wait_run_in_executor(os.stat, dir_full_path)
+                    try:
+                        created_at = datetime.datetime.fromtimestamp(stat_info.st_ctime).isoformat()
+                        modified_at = datetime.datetime.fromtimestamp(stat_info.st_mtime).isoformat()
+                    except (OSError, OverflowError, ValueError):
+                        created_at = modified_at = ""
+                    files.append({
+                        "path": os.path.relpath(dir_full_path, base_path),
+                        "size": stat_info.st_size,
+                        "created_at": created_at,
+                        "modified_at": modified_at,
+                        "file_type": "directory"
+                    })
+                except OSError:
+                    continue
 
+            for filename in filenames:
+                if filename.endswith(".ghost"):
+                    continue
+                file_path = os.path.join(dirpath, filename)
+                rel_path = os.path.relpath(file_path, base_path)
+                try:
+                    stat_info = await wait_run_in_executor(os.stat, file_path)
+                    try:
+                        file_type = await wait_run_in_executor(
+                            lambda fp=file_path: magic.from_file(fp, mime=False)
+                        )
+                    except Exception as e:
+                        log.warning(f"Error getting file type for '{rel_path}': {e}")
+                        file_type = ""
+                    try:
+                        created_at = datetime.datetime.fromtimestamp(stat_info.st_ctime).isoformat()
+                        modified_at = datetime.datetime.fromtimestamp(stat_info.st_mtime).isoformat()
+                    except (OSError, OverflowError, ValueError) as e:
+                        log.warning(f"Invalid timestamp for '{rel_path}': {e}")
+                        created_at = modified_at = ""
+                    files.append({
+                        "path": rel_path,
+                        "size": stat_info.st_size,
+                        "created_at": created_at,
+                        "modified_at": modified_at,
+                        "file_type": file_type
+                    })
+                except OSError as e:
+                    log.warning(f"Error getting metadata for file '{rel_path}': {e}")
+                    continue
         return files
 
     def _hash_file(self, path):

gns3server/controller/compute.py

@@ -341,9 +341,11 @@ class Compute:
             raise ControllerNotFoundError(f"{image} not found on compute")
         return response
 
-    async def http_query(self, method, path, data=None, dont_connect=False, **kwargs):
+    async def http_query(self, method, path, data=None, dont_connect=False, stream=False, params=None, **kwargs):
         """
         :param dont_connect: If true do not reconnect if not connected
+        :param stream: If True, return raw aiohttp response for streaming
+        :param params: Optional dict of query parameters to append to the URL
         """
 
         if not self._connected and not dont_connect:
@@ -352,7 +354,7 @@ class Compute:
             await self.connect()
         if not self._connected and not dont_connect:
             raise ComputeError(f"Cannot connect to compute '{self._name}' with request {method} {path}")
-        response = await self._run_http_query(method, path, data=data, **kwargs)
+        response = await self._run_http_query(method, path, data=data, stream=stream, params=params, **kwargs)
         return response
 
     async def _try_reconnect(self):
@@ -515,7 +517,7 @@ class Compute:
         """ Returns URL for specific path at Compute"""
         return self._getUrl(path)
 
-    async def _run_http_query(self, method, path, data=None, timeout=120, raw=False):
+    async def _run_http_query(self, method, path, data=None, timeout=120, raw=False, stream=False, params=None):
         async with asynctimeout(delay=timeout):
             url = self._getUrl(path)
             headers = {"content-type": "application/json"}
@@ -531,6 +533,10 @@ class Compute:
                 elif isinstance(data, aiohttp.streams.StreamReader) or isinstance(data, bytes):
                     chunked = True
                     headers["content-type"] = "application/octet-stream"
+                # Stream from an async iterable (e.g. Starlette request.stream())
+                elif hasattr(data, "__aiter__"):
+                    chunked = True
+                    headers["content-type"] = "application/octet-stream"
                 # If the data is an open file we will iterate on it
                 elif isinstance(data, io.BufferedIOBase):
                     chunked = True
@@ -540,7 +546,7 @@ class Compute:
         try:
             log.debug(f"Attempting request to compute: {method} {url} {headers}")
             response = await self._session().request(
-                method, url, headers=headers, data=data, auth=self._auth, chunked=chunked, timeout=timeout
+                method, url, headers=headers, data=data, auth=self._auth, params=params, chunked=chunked, timeout=timeout
             )
         except asyncio.TimeoutError:
             raise ComputeError(f"Timeout error for {method} call to {url} after {timeout}s")
@@ -554,6 +560,18 @@ class Compute:
         ) as e:
             #  aiohttp 2.3.1 raises socket.gaierror when cannot find host
             raise ComputeError(str(e))
+
+        if stream:
+            if response.status >= 300:
+                body = await response.read()
+                msg = body.decode() if body else ""
+                if response.status == 404:
+                    raise ControllerNotFoundError(f"{method} {path} not found")
+                elif response.status == 403:
+                    raise ControllerForbiddenError(msg)
+                raise ControllerError(f"HTTP {response.status}: {msg}")
+            return response
+
         body = await response.read()
         if body and not raw:
             body = body.decode()

gns3server/schemas/controller/projects.py

@@ -114,7 +114,7 @@ class NodeFile(BaseModel):
     size: int = Field(..., description="File size in bytes")
     created_at: str = Field(..., description="File creation time (ISO 8601)")
     modified_at: str = Field(..., description="File modification time (ISO 8601)")
-    extension: str = Field(..., description="File extension")
+    file_type: str = Field(..., description="File type determined by the file command")
 
 
 class ProjectCompression(str, Enum):