Detect new images added to the default image directory.

* Images can be present before the server starts or while it is running
* Images are recorded in the database

gns3server/db/tasks.py

@@ -15,11 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import asyncio
+import signal
 import os
 
 from fastapi import FastAPI
-from fastapi.encoders import jsonable_encoder
 from pydantic import ValidationError
+from watchfiles import awatch, Change
 
 from typing import List
 from sqlalchemy import event
@@ -27,6 +29,8 @@ from sqlalchemy.engine import Engine
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
 from gns3server.db.repositories.computes import ComputesRepository
+from gns3server.db.repositories.images import ImagesRepository
+from gns3server.utils.images import discover_images, check_valid_image_header, read_image_info, InvalidImageError
 from gns3server import schemas
 
 from .models import Base
@@ -74,3 +78,94 @@ async def get_computes(app: FastAPI) -> List[dict]:
                 continue
             computes.append(compute)
     return computes
+
+
+def image_filter(change: Change, path: str) -> bool:
+
+    if change == Change.added:
+        header_magic_len = 7
+        with open(path, "rb") as f:
+            image_header = f.read(header_magic_len)  # read the first 7 bytes of the file
+            if len(image_header) >= header_magic_len:
+                try:
+                    check_valid_image_header(image_header)
+                except InvalidImageError as e:
+                    log.debug(f"New image '{path}' added: {e}")
+                    return False
+            else:
+                log.debug(f"New image '{path}' added: size is too small to be valid")
+                return False
+        return True
+    # FIXME: should we support image deletion?
+    # elif change == Change.deleted:
+    #     return True
+    return False
+
+
+async def monitor_images_on_filesystem(app: FastAPI):
+
+    server_config = Config.instance().settings.Server
+    images_dir = os.path.expanduser(server_config.images_path)
+
+    try:
+        async for changes in awatch(
+                images_dir,
+                watch_filter=image_filter,
+                raise_interrupt=True
+        ):
+            async with AsyncSession(app.state._db_engine) as db_session:
+                images_repository = ImagesRepository(db_session)
+                for change in changes:
+                    change_type, image_path = change
+                    if change_type == Change.added:
+                        try:
+                            image = await read_image_info(image_path)
+                        except InvalidImageError as e:
+                            log.warning(str(e))
+                            continue
+                        try:
+                            if await images_repository.get_image(image_path):
+                                continue
+                            await images_repository.add_image(**image)
+                            log.info(f"Discovered image '{image_path}' has been added to the database")
+                        except SQLAlchemyError as e:
+                            log.warning(f"Error while adding image '{image_path}' to the database: {e}")
+                    # if change_type == Change.deleted:
+                    #     try:
+                    #         if await images_repository.get_image(image_path):
+                    #             success = await images_repository.delete_image(image_path)
+                    #             if not success:
+                    #                 log.warning(f"Could not delete image '{image_path}' from the database")
+                    #             else:
+                    #                 log.info(f"Image '{image_path}' has been deleted from the database")
+                    #     except SQLAlchemyError as e:
+                    #         log.warning(f"Error while deleting image '{image_path}' from the database: {e}")
+    except KeyboardInterrupt:
+        # send SIGTERM to the server PID so uvicorn can shutdown the process
+        os.kill(os.getpid(), signal.SIGTERM)
+
+
+async def discover_images_on_filesystem(app: FastAPI):
+
+    async with AsyncSession(app.state._db_engine) as db_session:
+        images_repository = ImagesRepository(db_session)
+        db_images = await images_repository.get_images()
+        existing_image_paths = []
+        for db_image in db_images:
+            try:
+                image = schemas.Image.from_orm(db_image)
+                existing_image_paths.append(image.path)
+            except ValidationError as e:
+                log.error(f"Could not load image '{db_image.filename}' from database: {e}")
+                continue
+        for image_type in ("qemu", "ios", "iou"):
+            discovered_images = await discover_images(image_type, existing_image_paths)
+            for image in discovered_images:
+                log.info(f"Adding discovered image '{image['path']}' to the database")
+                try:
+                    await images_repository.add_image(**image)
+                except SQLAlchemyError as e:
+                    log.warning(f"Error while adding image '{image['path']}' to the database: {e}")
+
+    # monitor if images have been manually added
+    asyncio.create_task(monitor_images_on_filesystem(app))