fix(images): reconcile the image inventory with filesystem changes

gns3server/api/routes/controller/images.py

@@ -21,12 +21,14 @@ API routes for images.
 import os
 import logging
 import urllib.parse
+import tempfile
 
-from fastapi import APIRouter, Request, Depends, status
+from fastapi import APIRouter, Request, Response, Depends, Query, status
 from fastapi.encoders import jsonable_encoder
 from starlette.requests import ClientDisconnect
 from sqlalchemy.orm.exc import MultipleResultsFound
-from typing import List, Optional
+from sqlalchemy.exc import SQLAlchemyError
+from typing import List, Optional, Literal
 
 from gns3server import schemas
 from gns3server.config import Config
@@ -36,6 +38,8 @@ from gns3server.db.repositories.images import ImagesRepository
 from gns3server.db.repositories.templates import TemplatesRepository
 from gns3server.db.repositories.rbac import RbacRepository
 from gns3server.controller import Controller
+from gns3server.services.image_reconciliation import get_image_reconciliation_service
+from gns3server.utils.image_inventory import contained_path, image_lock, publish_image, fingerprint, ImageLockBusy
 from gns3server.controller.controller_error import (
     ControllerError,
     ControllerNotFoundError,
@@ -51,6 +55,37 @@ log = logging.getLogger(__name__)
 
 router = APIRouter()
 
+
+def image_destination(image_path):
+    root = os.path.realpath(os.path.expanduser(Config.instance().settings.Server.images_path))
+    full_path = os.path.abspath(os.path.join(root, image_path))
+    if not contained_path(os.path.realpath(full_path), root):
+        raise ControllerForbiddenError(f"Cannot write image, '{image_path}' is forbidden")
+    return full_path
+
+
+@router.post("/sync", response_model=schemas.ImageSyncJob, status_code=status.HTTP_202_ACCEPTED,
+             dependencies=[Depends(has_privilege("Image.Allocate"))])
+async def sync_images(options: schemas.ImageSyncRequest, request: Request, response: Response):
+    """Reconcile configured image directories without deleting files or references."""
+    try:
+        job = await get_image_reconciliation_service(request.app).start(**options.model_dump())
+    except ImageLockBusy:
+        raise ControllerError("Image synchronization is already running or shutting down")
+    response.headers["Location"] = str(request.url_for("get_image_sync_job", job_id=job["job_id"]))
+    return job
+
+
+@router.get("/sync/jobs/{job_id}", response_model=schemas.ImageSyncJob,
+            dependencies=[Depends(has_privilege("Image.Audit"))])
+async def get_image_sync_job(job_id: str, request: Request,
+                             offset: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=1000)):
+    """Get progress and a page of errors (first 1,000 errors retained per job)."""
+    job = await get_image_reconciliation_service(request.app).get_job(job_id, offset, limit)
+    if job is None:
+        raise ControllerNotFoundError(f"Image synchronization job '{job_id}' not found")
+    return job
+
 @router.post(
     "/qemu/{image_path:path}",
     response_model=schemas.Image,
@@ -76,33 +111,32 @@ async def create_qemu_image(
     disk_image_path = urllib.parse.unquote(image_path)
     image_dir, image_name = os.path.split(disk_image_path)
     # check if the path is within the default images directory
-    base_images_directory = os.path.expanduser(Config.instance().settings.Server.images_path)
-    full_path = os.path.abspath(os.path.join(base_images_directory, image_dir, image_name))
-    if os.path.commonprefix([base_images_directory, full_path]) != base_images_directory:
-        raise ControllerForbiddenError(f"Cannot write disk image, '{disk_image_path}' is forbidden")
+    disk_image_path = image_destination(disk_image_path)
 
     if not image_dir:
         # put the image in the default images directory for Qemu
         directory = default_images_directory(image_type="qemu")
         os.makedirs(directory, exist_ok=True)
-        disk_image_path = os.path.abspath(os.path.join(directory, disk_image_path))
+        disk_image_path = image_destination(os.path.join(directory, image_name))
 
-    if await images_repo.get_image(disk_image_path):
-        raise ControllerBadRequestError(f"Disk image '{disk_image_path}' already exists")
-
-    options = jsonable_encoder(image_data, exclude_unset=True)
-    # FIXME: should we have the create_disk_image in the compute code since
-    # this code is used to create images on the controller?
-    await Qemu.instance().create_disk_image(disk_image_path, options)
-
-    image_info = await read_image_info(disk_image_path, "qemu")
-
-    image = await images_repo.get_image(disk_image_path)
-    if image:
-        # the image has already been added to the database
-        return image
-    else:
-        return await images_repo.add_image(**image_info)
+    async with image_lock(disk_image_path):
+        if os.path.lexists(disk_image_path):
+            raise ControllerBadRequestError(f"Disk image '{disk_image_path}' already exists")
+        os.makedirs(os.path.dirname(disk_image_path), exist_ok=True)
+        fd, temporary = tempfile.mkstemp(prefix=".gns3-create-", suffix=".tmp", dir=os.path.dirname(disk_image_path))
+        os.close(fd)
+        try:
+            options = jsonable_encoder(image_data, exclude_unset=True)
+            await Qemu.instance().create_disk_image(temporary, options)
+            image_info = await read_image_info(temporary, "qemu", allow_raw_image=allow_raw_image)
+            publish_image(temporary, disk_image_path)
+            image_info.update(path=disk_image_path, image_name=image_name, file_fingerprint=fingerprint(disk_image_path))
+            return await images_repo.save_verified_image(image_info)
+        except (OSError, InvalidImageError, SQLAlchemyError) as e:
+            raise ControllerError(f"Could not create disk image '{disk_image_path}': {e}") from e
+        finally:
+            if os.path.exists(temporary):
+                os.unlink(temporary)
 
 @router.get(
     "",
@@ -111,7 +145,8 @@ async def create_qemu_image(
 )
 async def get_images(
         images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)),
-        image_type: Optional[schemas.ImageType] = None
+        image_type: Optional[schemas.ImageType] = None,
+        availability: Optional[Literal["unknown", "available", "missing", "unavailable", "invalid"]] = None
 ) -> List[schemas.Image]:
     """
     Return all images.
@@ -119,7 +154,7 @@ async def get_images(
     Required privilege: Image.Audit
     """
 
-    return await images_repo.get_images(image_type)
+    return await images_repo.get_images(image_type, availability)
 
 
 @router.post(
@@ -149,10 +184,7 @@ async def upload_image(
     image_path = urllib.parse.unquote(image_path)
     image_dir, image_name = os.path.split(image_path)
     # check if the path is within the default images directory
-    base_images_directory = os.path.expanduser(Config.instance().settings.Server.images_path)
-    full_path = os.path.abspath(os.path.join(base_images_directory, image_dir, image_name))
-    if os.path.commonprefix([base_images_directory, full_path]) != base_images_directory:
-        raise ControllerForbiddenError(f"Cannot write image, '{image_path}' is forbidden")
+    full_path = image_destination(image_path)
 
     # If the client sends X-MD5-Checksum, check for a duplicate before consuming the upload stream
     checksum_header = request.headers.get("X-MD5-Checksum")
@@ -168,7 +200,10 @@ async def upload_image(
     try:
         allow_raw_image = Config.instance().settings.Server.allow_raw_images
         image = await write_image(image_path, full_path, request.stream(), images_repo, allow_raw_image=allow_raw_image)
-    except (OSError, InvalidImageError, ClientDisconnect) as e:
+    except (OSError, InvalidImageError, ClientDisconnect, SQLAlchemyError) as e:
+        service = getattr(request.app.state, "image_reconciliation", None)
+        if service:
+            service.dirty.set()
         raise ControllerError(f"Could not save image '{image_path}': {e}")
 
     if install_appliances:
@@ -206,7 +241,9 @@ async def prune_images(
     # a single pass over all projects' node properties protects every
     # referenced file name at once
     referenced_filenames = Controller.instance().collect_referenced_image_filenames()
-    await images_repo.prune_images(list(skip_images) + list(referenced_filenames))
+    await images_repo.prune_images(
+        list(skip_images) + list(referenced_filenames),
+        is_in_use=Controller.instance().find_projects_using_image)
 
 
 @router.post(
@@ -232,6 +269,9 @@ async def install_images(
     skip_images = get_builtin_disks()
     images = await images_repo.get_images()
     for image in images:
+        if not await images_repo.is_usable(image):
+            skipped.append({"name": image.filename, "reason": "image is missing, changed or unreadable; synchronize images first"})
+            continue
         if skip_images and image.filename in skip_images:
             log.debug(f"Skipping image '{image.path}' for image installation")
             continue
@@ -318,11 +358,20 @@ async def delete_image(
     if project_names:
         raise ControllerError(f"Image '{image_path}' is used by one or more projects: {', '.join(project_names)}")
 
-    try:
-        os.remove(image.path)
-    except OSError:
-        log.warning(f"Could not delete image file {image.path}")
-
-    success = await images_repo.delete_image(image_path)
-    if not success:
-        raise ControllerError(f"Image '{image_path}' could not be deleted")
+    path = image.path
+    revision = (image.image_id, image.checksum, image.file_fingerprint)
+    async with image_lock(path):
+        image = await images_repo.get_image(path, refresh=True)
+        if image is None or (image.image_id, image.checksum, image.file_fingerprint) != revision:
+            raise ControllerError(f"Image '{image_path}' changed while waiting for deletion; refresh and retry")
+        # Recheck usage after waiting for a concurrent writer/scanner.
+        if await images_repo.get_image_templates(image.image_id) or Controller.instance().find_projects_using_image(image.filename):
+            raise ControllerError(f"Image '{image_path}' is in use")
+        try:
+            os.remove(image.path)
+        except FileNotFoundError:
+            pass
+        except OSError as e:
+            raise ControllerError(f"Could not delete image file '{image.path}': {e}") from e
+        if not await images_repo.delete_image_exact(image.image_id):
+            raise ControllerError(f"Image '{image_path}' could not be deleted")