fix(images): reconcile the image inventory with filesystem changes

gns3server/db/models/__init__.py

@@ -21,7 +21,7 @@ from .users import User, UserGroup
 from .roles import Role
 from .privileges import Privilege
 from .computes import Compute
-from .images import Image
+from .images import Image, ImageSyncJob
 from .pools import Resource, ResourcePool
 from .llm_model_configs import LLMModelConfig
 from .api_keys import ApiKey

gns3server/db/models/images.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from sqlalchemy import Table, Column, String, ForeignKey, BigInteger, Integer
+from sqlalchemy import Table, Column, String, ForeignKey, BigInteger, Integer, DateTime, JSON, Boolean
 from sqlalchemy.orm import relationship
 
 from .base import Base, BaseTable, GUID
@@ -40,4 +40,21 @@ class Image(BaseTable):
     image_size = Column(BigInteger)
     checksum = Column(String, index=True)
     checksum_algorithm = Column(String)
+    availability = Column(String, nullable=False, default="unknown", server_default="unknown")
+    file_fingerprint = Column(String)
+    last_seen_at = Column(DateTime)
+    last_verified_at = Column(DateTime)
+    last_error = Column(String)
     templates = relationship("Template", secondary=image_template_map, back_populates="images")
+
+
+class ImageSyncJob(BaseTable):
+    __tablename__ = "image_sync_jobs"
+
+    job_id = Column(String, primary_key=True)
+    status = Column(String, nullable=False)
+    dry_run = Column(Boolean, nullable=False)
+    force_checksum = Column(Boolean, nullable=False)
+    finished_at = Column(DateTime)
+    counts = Column(JSON, nullable=False)
+    errors = Column(JSON, nullable=False)

gns3server/db/repositories/images.py

@@ -16,10 +16,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import asyncio
+from datetime import datetime, timezone
 
-from typing import Optional, List
+from typing import Optional, List, Callable
 from sqlalchemy import select, delete, update
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.exc import IntegrityError
+from gns3server.utils.image_inventory import fingerprint, image_lock, normalized_path
 
 from .base import BaseRepository
 
@@ -36,18 +40,21 @@ class ImagesRepository(BaseRepository):
 
         super().__init__(db_session)
 
-    async def get_image(self, image_path: str) -> Optional[models.Image]:
+    async def get_image(self, image_path: str, *, refresh: bool = False) -> Optional[models.Image]:
         """
         Get an image by its path.
         """
 
         image_dir, image_name = os.path.split(image_path)
-        if image_dir:
+        if os.path.isabs(image_path):
+            query = select(models.Image).where(models.Image.path == image_path)
+        elif image_dir:
             query = select(models.Image).\
-                where(models.Image.filename == image_name, models.Image.path.endswith(image_path))
+                where(models.Image.filename == image_name,
+                      models.Image.path.endswith(os.sep + image_path, autoescape=True))
         else:
             query = select(models.Image).where(models.Image.filename == image_name)
-        result = await self._db_session.execute(query)
+        result = await self._db_session.execute(query.execution_options(populate_existing=refresh))
         return result.scalars().one_or_none()
 
     async def get_image_by_checksum(self, checksum: str, image_dir: str = None) -> Optional[models.Image]:
@@ -55,17 +62,28 @@ class ImagesRepository(BaseRepository):
         Get an image by its checksum.
         """
 
-        if image_dir:
-            query = select(models.Image).\
-                where(models.Image.checksum == checksum, models.Image.path.startswith(image_dir))
-            result = await self._db_session.execute(query)
-            return result.scalars().one_or_none()
-        else:
-            query = select(models.Image).where(models.Image.checksum == checksum)
-            result = await self._db_session.execute(query)
-            return result.scalars().first()
+        query = select(models.Image).where(models.Image.checksum == checksum).order_by(models.Image.image_id)
+        result = await self._db_session.execute(query)
+        for image in result.scalars().all():
+            if image_dir and normalized_path(os.path.dirname(image.path)) != normalized_path(image_dir):
+                continue
+            if await self.is_usable(image):
+                return image
+        return None
 
-    async def get_images(self, image_type=None) -> List[models.Image]:
+    async def is_usable(self, image: models.Image) -> bool:
+        """Validate a checksum candidate without trusting stale catalog/sidecar data."""
+        from gns3server.utils.images import inspect_image_file, InvalidImageError
+        try:
+            current = await asyncio.to_thread(fingerprint, image.path)
+            if image.availability == "available" and image.file_fingerprint == current:
+                return True
+            info = await asyncio.to_thread(inspect_image_file, image.path, image.image_type, True)
+            return info["checksum"] == image.checksum and info["image_size"] == image.image_size
+        except (OSError, InvalidImageError):
+            return False
+
+    async def get_images(self, image_type=None, availability=None) -> List[models.Image]:
         """
         Get all images.
         """
@@ -74,6 +92,8 @@ class ImagesRepository(BaseRepository):
             query = select(models.Image).where(models.Image.image_type == image_type)
         else:
             query = select(models.Image)
+        if availability:
+            query = query.where(models.Image.availability == availability)
         result = await self._db_session.execute(query)
         return result.scalars().all()
 
@@ -89,7 +109,8 @@ class ImagesRepository(BaseRepository):
         result = await self._db_session.execute(query)
         return result.scalars().all()
 
-    async def add_image(self, image_name, image_type, image_size, path, checksum, checksum_algorithm) -> models.Image:
+    async def add_image(self, image_name, image_type, image_size, path, checksum, checksum_algorithm,
+                        file_fingerprint=None) -> models.Image:
         """
         Create a new image.
         """
@@ -101,11 +122,17 @@ class ImagesRepository(BaseRepository):
             image_size=image_size,
             path=path,
             checksum=checksum,
-            checksum_algorithm=checksum_algorithm
+            checksum_algorithm=checksum_algorithm,
+            file_fingerprint=file_fingerprint,
+            availability="available" if file_fingerprint else "unknown",
         )
 
         self._db_session.add(db_image)
-        await self._db_session.commit()
+        try:
+            await self._db_session.commit()
+        except Exception:
+            await self._db_session.rollback()
+            raise
         await self._db_session.refresh(db_image)
         return db_image
 
@@ -125,15 +152,69 @@ class ImagesRepository(BaseRepository):
             await self._db_session.refresh(image_db)  # force refresh of updated_at value
         return image_db
 
+    async def save_verified_image(self, info: dict) -> models.Image:
+        """Upsert an exact path, preserving template associations and the image ID.
+
+        Callers coordinate publication/inspection with image_lock(). Each commit
+        is short, and a concurrent insert from a legacy caller is retried safely.
+        """
+        values = dict(info)
+        values["filename"] = values.pop("image_name")
+        values.update(availability="available", last_error=None,
+                      last_seen_at=datetime.now(timezone.utc).replace(tzinfo=None),
+                      last_verified_at=datetime.now(timezone.utc).replace(tzinfo=None))
+        for attempt in range(2):
+            try:
+                image = (await self._db_session.execute(
+                    select(models.Image).where(models.Image.path == info["path"])
+                    .execution_options(populate_existing=True))).scalar_one_or_none()
+                if image is None:
+                    # A legacy spelling (e.g. /images/QEMU/./disk.qcow2) can
+                    # refer to the same destination restored by an API upload.
+                    candidates = (await self._db_session.execute(select(models.Image).where(
+                        models.Image.filename == values["filename"]))).scalars().all()
+                    aliases = [candidate for candidate in candidates
+                               if normalized_path(candidate.path) == normalized_path(info["path"])]
+                    if len(aliases) > 1:
+                        from sqlalchemy.exc import MultipleResultsFound
+                        raise MultipleResultsFound("Ambiguous image path aliases; manual review required")
+                    if aliases:
+                        image = aliases[0]
+                        values["path"] = image.path
+                if image is None:
+                    image = models.Image(**values)
+                    self._db_session.add(image)
+                else:
+                    for key, value in values.items():
+                        setattr(image, key, value)
+                await self._db_session.commit()
+                await self._db_session.refresh(image)
+                return image
+            except IntegrityError:
+                await self._db_session.rollback()
+                if attempt:
+                    raise
+            except Exception:
+                await self._db_session.rollback()
+                raise
+
+    async def delete_image_exact(self, image_id: int) -> bool:
+        result = await self._db_session.execute(delete(models.Image).where(models.Image.image_id == image_id))
+        await self._db_session.commit()
+        return result.rowcount > 0
+
     async def delete_image(self, image_path: str) -> bool:
         """
         Delete an image.
         """
 
         image_dir, image_name = os.path.split(image_path)
-        if image_dir:
+        if os.path.isabs(image_path):
+            query = delete(models.Image).where(models.Image.path == image_path)
+        elif image_dir:
             query = delete(models.Image).\
-                where(models.Image.filename == image_name, models.Image.path.endswith(image_path)).\
+                where(models.Image.filename == image_name,
+                      models.Image.path.endswith(os.sep + image_path, autoescape=True)).\
                 execution_options(synchronize_session=False)
         else:
             query = delete(models.Image).where(models.Image.filename == image_name)
@@ -141,7 +222,8 @@ class ImagesRepository(BaseRepository):
         await self._db_session.commit()
         return result.rowcount > 0
 
-    async def prune_images(self, skip_images: list[str] = None) -> int:
+    async def prune_images(self, skip_images: list[str] = None,
+                           is_in_use: Optional[Callable] = None) -> int:
         """
         Prune images not attached to any template.
         """
@@ -149,18 +231,32 @@ class ImagesRepository(BaseRepository):
         query = select(models.Image).\
             filter(~models.Image.templates.any())
         result = await self._db_session.execute(query)
-        images = result.scalars().all()
+        # Snapshot scalar values; commits can expire ORM instances.
+        images = [(image.image_id, image.filename, image.path, image.checksum, image.file_fingerprint)
+                  for image in result.scalars().all()]
         images_deleted = 0
-        for image in images:
-            if skip_images and image.filename in skip_images:
-                log.debug(f"Skipping image '{image.path}' for pruning")
+        errors = []
+        for image_id, filename, path, checksum, file_fingerprint in images:
+            if skip_images and filename in skip_images:
                 continue
-            try:
-                log.debug(f"Deleting image '{image.path}'")
-                os.remove(image.path)
-            except OSError:
-                log.warning(f"Could not delete image file {image.path}")
-            if await self.delete_image(image.path):
-                images_deleted += 1
+            async with image_lock(path):
+                current = await self.get_image(path, refresh=True)
+                if current is None or (current.image_id, current.checksum, current.file_fingerprint) != (
+                        image_id, checksum, file_fingerprint):
+                    continue  # A concurrent request removed or replaced this image.
+                if await self.get_image_templates(image_id) or (is_in_use and is_in_use(filename)):
+                    continue
+                try:
+                    os.remove(path)
+                except FileNotFoundError:
+                    pass
+                except OSError as e:
+                    errors.append(f"{path}: {e}")
+                    continue
+                if await self.delete_image_exact(image_id):
+                    images_deleted += 1
         log.info(f"{images_deleted} image(s) have been deleted")
+        if errors:
+            from gns3server.controller.controller_error import ControllerError
+            raise ControllerError("Could not delete image files: " + "; ".join(errors))
         return images_deleted

gns3server/db/repositories/templates.py

@@ -29,6 +29,8 @@ from .base import BaseRepository
 
 import gns3server.db.models as models
 from gns3server import schemas
+from gns3server.utils.image_inventory import image_lock
+from gns3server.controller.controller_error import ControllerNotFoundError
 
 log = logging.getLogger(__name__)
 
@@ -129,17 +131,22 @@ class TemplatesRepository(BaseRepository):
             await self._db_session.refresh(db_template)
         return db_template
 
-    async def get_image(self, image_path: str) -> Optional[models.Image]:
+    async def get_image(self, image_path: str, *, include_unavailable: bool = False) -> Optional[models.Image]:
         """
         Get an image by its path.
         """
 
         image_dir, image_name = os.path.split(image_path)
-        if image_dir:
+        if os.path.isabs(image_path):
+            query = select(models.Image).where(models.Image.path == image_path)
+        elif image_dir:
             query = select(models.Image).\
-                where(models.Image.filename == image_name, models.Image.path.endswith(image_path))
+                where(models.Image.filename == image_name,
+                      models.Image.path.endswith(os.sep + image_path, autoescape=True))
         else:
             query = select(models.Image).where(models.Image.filename == image_name)
+        if not include_unavailable:
+            query = query.where(models.Image.availability.in_(["unknown", "available"]))
         query = query.order_by(models.Image.image_id)
         result = await self._db_session.execute(query)
         images = result.scalars().all()
@@ -160,18 +167,23 @@ class TemplatesRepository(BaseRepository):
         Add an image to template.
         """
 
-        query = select(models.Template).\
-            options(selectinload(models.Template.images)).\
-            where(models.Template.template_id == template_id)
-        result = await self._db_session.execute(query)
-        template_in_db = result.scalars().first()
-        if not template_in_db:
-            return None
+        async with image_lock(image.path):
+            exists = (await self._db_session.execute(select(models.Image.image_id).where(
+                models.Image.image_id == image.image_id))).scalar_one_or_none()
+            if exists is None:
+                raise ControllerNotFoundError(f"Image '{image.path}' was removed while creating the template")
+            query = select(models.Template).\
+                options(selectinload(models.Template.images)).\
+                where(models.Template.template_id == template_id)
+            result = await self._db_session.execute(query)
+            template_in_db = result.scalars().first()
+            if not template_in_db:
+                return None
 
-        template_in_db.images.append(image)
-        await self._db_session.commit()
-        await self._db_session.refresh(template_in_db)
-        return template_in_db
+            template_in_db.images.append(image)
+            await self._db_session.commit()
+            await self._db_session.refresh(template_in_db)
+            return template_in_db
 
     async def remove_image_from_template(
             self,

gns3server/db/tasks.py

@@ -15,8 +15,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import asyncio
-import time
 import os
 
 from fastapi import FastAPI
@@ -31,13 +29,10 @@ from alembic import command, config
 from alembic.script import ScriptDirectory
 from alembic.runtime.migration import MigrationContext
 from alembic.util.exc import CommandError
-from watchdog.observers import Observer
-from watchdog.events import FileSystemEvent, PatternMatchingEventHandler
 
 from gns3server.db.repositories.computes import ComputesRepository
 from gns3server.db.repositories.images import ImagesRepository
-from gns3server.utils.images import md5sum, discover_images, read_image_info, InvalidImageError
-from gns3server.utils.asyncio import wait_run_in_executor
+from gns3server.utils.images import read_image_info
 from gns3server import schemas
 
 from .models import Base
@@ -138,7 +133,14 @@ async def connect_to_db(app: FastAPI) -> None:
                     log.info("Created new database and stamped to head revision")
                 elif db_state == 'new_with_llm_configs':
                     # Database already has llm_model_configs table (from Base.metadata.create_all)
-                    # Just stamp the version
+                    # Older unversioned metadata databases lack inventory fields.
+                    # Ensure the additive schema before stamping the current head.
+                    def upgrade_inventory(connection):
+                        from alembic.operations import Operations
+                        from gns3server.db_migrations.versions.d9e8a2b7c401_image_inventory_reconciliation import upgrade
+                        with Operations.context(MigrationContext.configure(connection)):
+                            upgrade()
+                    await conn.run_sync(upgrade_inventory)
                     await conn.run_sync(run_stamp, alembic_cfg)
                     await conn.commit()
                     log.info("Database has llm_model_configs table, stamped to head revision")
@@ -182,144 +184,24 @@ async def get_computes(app: FastAPI) -> List[dict]:
     return computes
 
 
-async def discover_images_on_filesystem(app: FastAPI) -> None:
-
-    async with AsyncSession(app.state._db_engine) as db_session:
-        images_repository = ImagesRepository(db_session)
-        db_images = await images_repository.get_images()
-        existing_image_paths = []
-        for db_image in db_images:
-            try:
-                image = schemas.Image.model_validate(db_image)
-                existing_image_paths.append(image.path)
-            except ValidationError as e:
-                log.error(f"Could not load image '{db_image.filename}' from database: {e}")
-                continue
-        for image_type in ("qemu", "ios", "iou"):
-            discovered_images = await discover_images(image_type, existing_image_paths)
-            for image in discovered_images:
-                log.info(f"Adding discovered image '{image['path']}' to the database")
-                try:
-                    await images_repository.add_image(**image)
-                except SQLAlchemyError as e:
-                    log.warning(f"Error while adding image '{image['path']}' to the database: {e}")
-
-    # monitor if images have been manually added
-    asyncio.create_task(monitor_images_on_filesystem(app))
-
-
 async def update_disk_checksums(updated_disks: List[str]) -> None:
-    """
-    Update the checksum of a list of disks in the database.
-
-    :param updated_disks: list of updated disks
-    """
-
+    """Refresh complete metadata after a server-managed disk modification."""
     from gns3server.api.server import app
-    async with AsyncSession(app.state._db_engine) as db_session:
-        images_repository = ImagesRepository(db_session)
-        for path in updated_disks:
-            image = await images_repository.get_image(path)
-            if image:
-                log.info(f"Updating image '{path}' in the database")
-                checksum = await wait_run_in_executor(md5sum, path, cache_to_md5file=False)
-                if image.checksum != checksum:
-                    await images_repository.update_image(path, checksum, "md5")
-
-class EventHandler(PatternMatchingEventHandler):
-    """
-    Watchdog event handler.
-    """
-
-    def __init__(self, queue: asyncio.Queue, loop: asyncio.BaseEventLoop, **kwargs):
-
-        self._loop = loop
-        self._queue = queue
-
-        # ignore temporary files, md5sum files, hidden files and directories
-        super().__init__(ignore_patterns=["*.tmp", "*.md5sum", ".*"], ignore_directories = True, **kwargs)
-
-    def on_closed(self, event: FileSystemEvent) -> None:
-        # monitor for closed files (e.g. when a file has finished to be copied)
-        if "/lib/" in event.src_path or "/lib64/" in event.src_path:
-            return  # ignore custom IOU libraries
-        self._loop.call_soon_threadsafe(self._queue.put_nowait, event)
-
-class EventIterator(object):
-    """
-    Watchdog Event iterator.
-    """
-
-    def __init__(self, queue: asyncio.Queue):
-        self.queue = queue
-
-    def __aiter__(self):
-        return self
-
-    async def __anext__(self):
-
-        item = await self.queue.get()
-        if item is None:
-            raise StopAsyncIteration
-        return item
-
-async def monitor_images_on_filesystem(app: FastAPI):
-
-    def watchdog(
-            path: str,
-            queue: asyncio.Queue,
-            loop: asyncio.BaseEventLoop,
-            app: FastAPI, recursive: bool = False
-    ) -> None:
-        """
-        Thread to monitor a directory for new images.
-        """
-
-        handler = EventHandler(queue, loop)
-        observer = Observer()
-        observer.schedule(handler, str(path), recursive=recursive)
-        observer.start()
-        log.info(f"Monitoring for new images in '{path}'")
-        while True:
-            time.sleep(1)
-            # stop when the app is exiting
-            if app.state.exiting:
-                observer.stop()
-                observer.join(10)
-                log.info(f"Stopping monitoring for new images in '{path}'")
-                loop.call_soon_threadsafe(queue.put_nowait, None)
-                break
-
-    queue = asyncio.Queue()
-    loop = asyncio.get_event_loop()
-    server_config = Config.instance().settings.Server
-    image_dir = os.path.expanduser(server_config.images_path)
-    asyncio.get_event_loop().run_in_executor(None, watchdog,image_dir, queue, loop, app, True)
-
-    async for filesystem_event in EventIterator(queue):
-        # read the file system event from the queue
-        image_path = filesystem_event.src_path
-        expected_image_type = None
-        if "IOU" in image_path:
-            expected_image_type = "iou"
-        elif "QEMU" in image_path:
-            expected_image_type = "qemu"
-        elif "IOS" in image_path:
-            expected_image_type = "ios"
-        async with AsyncSession(app.state._db_engine) as db_session:
-            images_repository = ImagesRepository(db_session)
-            try:
-                image = await read_image_info(image_path, expected_image_type)
-            except InvalidImageError as e:
-                log.warning(str(e))
-                continue
-            try:
-                if await images_repository.get_image(image_path):
-                    continue
-                await images_repository.add_image(**image)
-                log.info(f"Discovered image '{image_path}' has been added to the database")
-            except SQLAlchemyError as e:
-                log.warning(f"Error while adding image '{image_path}' to the database: {e}")
+    from gns3server.utils.image_inventory import image_lock
+    for path in updated_disks:
+        async with image_lock(path):
+            async with AsyncSession(app.state._db_engine, expire_on_commit=False) as session:
+                repository = ImagesRepository(session)
+                image = await repository.get_image(path)
+                if image:
+                    info = await read_image_info(path, image.image_type, allow_raw_image=True)
+                    try:
+                        os.unlink(path + ".md5sum")
+                    except FileNotFoundError:
+                        pass
+                    except OSError as e:
+                        log.warning("Could not invalidate checksum cache for '%s': %s", path, e)
+                    await repository.save_verified_image(info)
 
 
 async def get_user_llm_config_full(user_id: str, app: FastAPI) -> Optional[dict]:
@@ -415,4 +297,3 @@ async def get_user_llm_config_full(user_id: str, app: FastAPI) -> Optional[dict]
     except Exception as e:
         log.error(f"Failed to retrieve LLM config for user {user_id}: {e}", exc_info=True)
         return None
-