diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py
index 40e2c007..a96fe2e7 100644
--- a/gns3server/compute/base_manager.py
+++ b/gns3server/compute/base_manager.py
@@ -506,7 +506,7 @@ class BaseManager:
"""
try:
- return list_images(self._NODE_TYPE)
+ return await list_images(self._NODE_TYPE)
except OSError as e:
raise ComputeError(f"Can not list images {e}")
diff --git a/gns3server/core/tasks.py b/gns3server/core/tasks.py
index 85df5791..3bef5774 100644
--- a/gns3server/core/tasks.py
+++ b/gns3server/core/tasks.py
@@ -15,7 +15,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-import sys
import asyncio
from typing import Callable
@@ -25,7 +24,7 @@ from gns3server.controller import Controller
from gns3server.compute import MODULES
from gns3server.compute.port_manager import PortManager
from gns3server.utils.http_client import HTTPClient
-from gns3server.db.tasks import connect_to_db, get_computes
+from gns3server.db.tasks import connect_to_db, get_computes, discover_images_on_filesystem
import logging
@@ -60,7 +59,9 @@ def create_startup_handler(app: FastAPI) -> Callable:
# computing with server start
from gns3server.compute.qemu import Qemu
- asyncio.ensure_future(Qemu.instance().list_images())
+ # Start the discovering new images on file system 5 seconds after the server has started
+ # to give it a chance to process API requests
+ loop.call_later(5, asyncio.create_task, discover_images_on_filesystem(app))
for module in MODULES:
log.debug(f"Loading module {module.__name__}")
diff --git a/gns3server/db/tasks.py b/gns3server/db/tasks.py
index 7c40a6fb..7f89459e 100644
--- a/gns3server/db/tasks.py
+++ b/gns3server/db/tasks.py
@@ -15,11 +15,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
+import asyncio
+import signal
import os
from fastapi import FastAPI
-from fastapi.encoders import jsonable_encoder
from pydantic import ValidationError
+from watchfiles import awatch, Change
from typing import List
from sqlalchemy import event
@@ -27,6 +29,8 @@ from sqlalchemy.engine import Engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from gns3server.db.repositories.computes import ComputesRepository
+from gns3server.db.repositories.images import ImagesRepository
+from gns3server.utils.images import discover_images, check_valid_image_header, read_image_info, InvalidImageError
from gns3server import schemas
from .models import Base
@@ -74,3 +78,94 @@ async def get_computes(app: FastAPI) -> List[dict]:
continue
computes.append(compute)
return computes
+
+
+def image_filter(change: Change, path: str) -> bool:
+
+ if change == Change.added:
+ header_magic_len = 7
+ with open(path, "rb") as f:
+ image_header = f.read(header_magic_len) # read the first 7 bytes of the file
+ if len(image_header) >= header_magic_len:
+ try:
+ check_valid_image_header(image_header)
+ except InvalidImageError as e:
+ log.debug(f"New image '{path}' added: {e}")
+ return False
+ else:
+ log.debug(f"New image '{path}' added: size is too small to be valid")
+ return False
+ return True
+ # FIXME: should we support image deletion?
+ # elif change == Change.deleted:
+ # return True
+ return False
+
+
+async def monitor_images_on_filesystem(app: FastAPI):
+
+ server_config = Config.instance().settings.Server
+ images_dir = os.path.expanduser(server_config.images_path)
+
+ try:
+ async for changes in awatch(
+ images_dir,
+ watch_filter=image_filter,
+ raise_interrupt=True
+ ):
+ async with AsyncSession(app.state._db_engine) as db_session:
+ images_repository = ImagesRepository(db_session)
+ for change in changes:
+ change_type, image_path = change
+ if change_type == Change.added:
+ try:
+ image = await read_image_info(image_path)
+ except InvalidImageError as e:
+ log.warning(str(e))
+ continue
+ try:
+ if await images_repository.get_image(image_path):
+ continue
+ await images_repository.add_image(**image)
+ log.info(f"Discovered image '{image_path}' has been added to the database")
+ except SQLAlchemyError as e:
+ log.warning(f"Error while adding image '{image_path}' to the database: {e}")
+ # if change_type == Change.deleted:
+ # try:
+ # if await images_repository.get_image(image_path):
+ # success = await images_repository.delete_image(image_path)
+ # if not success:
+ # log.warning(f"Could not delete image '{image_path}' from the database")
+ # else:
+ # log.info(f"Image '{image_path}' has been deleted from the database")
+ # except SQLAlchemyError as e:
+ # log.warning(f"Error while deleting image '{image_path}' from the database: {e}")
+ except KeyboardInterrupt:
+ # send SIGTERM to the server PID so uvicorn can shutdown the process
+ os.kill(os.getpid(), signal.SIGTERM)
+
+
+async def discover_images_on_filesystem(app: FastAPI):
+
+ async with AsyncSession(app.state._db_engine) as db_session:
+ images_repository = ImagesRepository(db_session)
+ db_images = await images_repository.get_images()
+ existing_image_paths = []
+ for db_image in db_images:
+ try:
+ image = schemas.Image.from_orm(db_image)
+ existing_image_paths.append(image.path)
+ except ValidationError as e:
+ log.error(f"Could not load image '{db_image.filename}' from database: {e}")
+ continue
+ for image_type in ("qemu", "ios", "iou"):
+ discovered_images = await discover_images(image_type, existing_image_paths)
+ for image in discovered_images:
+ log.info(f"Adding discovered image '{image['path']}' to the database")
+ try:
+ await images_repository.add_image(**image)
+ except SQLAlchemyError as e:
+ log.warning(f"Error while adding image '{image['path']}' to the database: {e}")
+
+ # monitor if images have been manually added
+ asyncio.create_task(monitor_images_on_filesystem(app))
diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py
index d1d9eecd..1ef85bc9 100644
--- a/gns3server/utils/images.py
+++ b/gns3server/utils/images.py
@@ -20,19 +20,20 @@ import stat
import aiofiles
import shutil
-from typing import AsyncGenerator
+from typing import List, AsyncGenerator
from ..config import Config
from . import force_unix_path
import gns3server.db.models as models
from gns3server.db.repositories.images import ImagesRepository
+from gns3server.utils.asyncio import wait_run_in_executor
import logging
log = logging.getLogger(__name__)
-def list_images(image_type):
+async def list_images(image_type):
"""
Scan directories for available image for a given type.
@@ -59,7 +60,6 @@ def list_images(image_type):
directory = os.path.normpath(directory)
for root, _, filenames in _os_walk(directory, recurse=recurse):
for filename in filenames:
- path = os.path.join(root, filename)
if filename not in files:
if filename.endswith(".md5sum") or filename.startswith("."):
continue
@@ -92,7 +92,7 @@ def list_images(image_type):
{
"filename": filename,
"path": force_unix_path(path),
- "md5sum": md5sum(os.path.join(root, filename)),
+ "md5sum": await wait_run_in_executor(md5sum, os.path.join(root, filename)),
"filesize": os.stat(os.path.join(root, filename)).st_size,
}
)
@@ -101,6 +101,59 @@ def list_images(image_type):
return images
+async def read_image_info(path: str, expected_image_type: str = None) -> dict:
+
+ header_magic_len = 7
+ try:
+ async with aiofiles.open(path, "rb") as f:
+ image_header = await f.read(header_magic_len) # read the first 7 bytes of the file
+ if len(image_header) >= header_magic_len:
+ detected_image_type = check_valid_image_header(image_header)
+ if expected_image_type and detected_image_type != expected_image_type:
+ raise InvalidImageError(f"Detected image type for '{path}' is {detected_image_type}, "
+ f"expected type is {expected_image_type}")
+ else:
+ raise InvalidImageError(f"Image '{path}' is too small to be valid")
+ except OSError as e:
+ raise InvalidImageError(f"Cannot read image '{path}': {e}")
+
+ image_info = {
+ "image_name": os.path.basename(path),
+ "image_type": detected_image_type,
+ "image_size": os.stat(path).st_size,
+ "path": path,
+ "checksum": await wait_run_in_executor(md5sum, path),
+ "checksum_algorithm": "md5",
+ }
+ return image_info
+
+
+async def discover_images(image_type: str, skip_image_paths: list = None) -> List[dict]:
+ """
+ Scan directories for available images
+ """
+
+ files = set()
+ images = []
+
+ for directory in images_directories(image_type):
+ for root, _, filenames in os.walk(os.path.normpath(directory)):
+ for filename in filenames:
+ if filename.endswith(".md5sum") or filename.startswith("."):
+ continue
+ path = os.path.join(root, filename)
+ if not os.path.isfile(path) or skip_image_paths and path in skip_image_paths or path in files:
+ continue
+ files.add(path)
+
+ try:
+ images.append(await read_image_info(path, image_type))
+ except InvalidImageError as e:
+ #log.warning(f"{e}")
+ continue
+ return images
+
+
def _os_walk(directory, recurse=True, **kwargs):
"""
Work like os.walk but if recurse is False just list current directory
@@ -133,18 +186,18 @@ def default_images_directory(image_type):
raise NotImplementedError(f"%s node type is not supported", image_type)
-def images_directories(type):
+def images_directories(image_type):
"""
Return all directories where we will look for images
by priority
- :param type: Type of emulator
+ :param image_type: Type of emulator
"""
server_config = Config.instance().settings.Server
paths = []
img_dir = os.path.expanduser(server_config.images_path)
- type_img_directory = default_images_directory(type)
+ type_img_directory = default_images_directory(image_type)
try:
os.makedirs(type_img_directory, exist_ok=True)
paths.append(type_img_directory)
@@ -193,7 +246,7 @@ def md5sum(path, working_dir=None, stopped_event=None):
if stopped_event is not None and stopped_event.is_set():
log.error(f"MD5 sum calculation of `{path}` has stopped due to cancellation")
return
- buf = f.read(128)
+ buf = f.read(1024)
if not buf:
break
m.update(buf)
@@ -237,10 +290,11 @@ def check_valid_image_header(data: bytes) -> str:
# for IOS images: file must start with the ELF magic number, be 32-bit, big endian and have an ELF version of 1
return "ios"
elif data[:7] == b'\x7fELF\x01\x01\x01' or data[:7] == b'\x7fELF\x02\x01\x01':
- # for IOU images file must start with the ELF magic number, be 32-bit or 64-bit, little endian and
+ # for IOU images: file must start with the ELF magic number, be 32-bit or 64-bit, little endian and
# have an ELF version of 1 (normal IOS images are big endian!)
return "iou"
- elif data[:4] != b'QFI\xfb' or data[:4] != b'KDMV':
+ elif data[:4] == b'QFI\xfb' or data[:4] == b'KDMV':
+ # for Qemy images: file must be QCOW2 or VMDK
return "qemu"
else:
raise InvalidImageError("Could not detect image type, please make sure it is a valid image")
diff --git a/requirements.txt b/requirements.txt
index 7bc97804..92e08028 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,4 +15,5 @@ aiosqlite===0.17.0
passlib[bcrypt]==1.7.4
python-jose==3.3.0
email-validator==1.1.3
+watchfiles==0.13
setuptools==60.6.0 # don't upgrade because of https://github.com/pypa/setuptools/issues/3084