Merge pull request #1911 from GNS3/image-management-refactoring

Images management refactoring

gns3server/controller/appliance.py

@@ -56,10 +56,28 @@ class Appliance:
     def name(self):
         return self._data.get("name")
 
+    @property
+    def images(self):
+        return self._data.get("images")
+
+    @property
+    def versions(self):
+        return self._data.get("versions")
+
     @symbol.setter
     def symbol(self, new_symbol):
         self._data["symbol"] = new_symbol
 
+    @property
+    def type(self):
+
+        if "iou" in self._data:
+            return "iou"
+        elif "dynamips" in self._data:
+            return "dynamips"
+        else:
+            return "qemu"
+
     def asdict(self):
         """
         Appliance data (a hash)

gns3server/controller/appliance_manager.py

@@ -16,10 +16,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import shutil
 import json
 import uuid
 import asyncio
+import aiofiles
+
+from aiohttp.client_exceptions import ClientError
 
 from .appliance import Appliance
 from ..config import Config
@@ -27,6 +29,9 @@ from ..utils.asyncio import locking
 from ..utils.get_resource import get_resource
 from ..utils.http_client import HTTPClient
 from .controller_error import ControllerError
+from .appliance_to_template import ApplianceToTemplate
+from ..utils.images import InvalidImageError, write_image, md5sum
+from ..utils.asyncio import wait_run_in_executor
 
 import logging
 
@@ -77,6 +82,90 @@ class ApplianceManager:
         os.makedirs(appliances_path, exist_ok=True)
         return appliances_path
 
+    def _find_appliance_from_image_checksum(self, image_checksum):
+        """
+        Find an appliance and version that matches an image checksum.
+        """
+
+        for appliance in self._appliances.values():
+            if appliance.images:
+                for image in appliance.images:
+                    if image.get("md5sum") == image_checksum:
+                        return appliance, image.get("version")
+
+    async def _download_image(self, image_dir, image_name, image_type, image_url, images_repo):
+        """
+        Download an image.
+        """
+
+        log.info(f"Downloading image '{image_name}' from '{image_url}'")
+        image_path = os.path.join(image_dir, image_name)
+        try:
+            async with HTTPClient.get(image_url) as response:
+                if response.status != 200:
+                    raise ControllerError(f"Could not download '{image_name}' due to HTTP error code {response.status}")
+                await write_image(image_name, image_type, image_path, response.content.iter_any(), images_repo)
+        except (OSError, InvalidImageError) as e:
+            raise ControllerError(f"Could not save {image_type} image '{image_path}': {e}")
+        except ClientError as e:
+            raise ControllerError(f"Could not connect to download '{image_name}': {e}")
+        except asyncio.TimeoutError:
+            raise ControllerError(f"Timeout while downloading '{image_name}' from '{image_url}'")
+
+    async def _find_appliance_version_images(self, appliance, version, images_repo, image_dir):
+        """
+        Find all the images belonging to a specific appliance version.
+        """
+
+        version_images = version.get("images")
+        if version_images:
+            for appliance_key, appliance_file in version_images.items():
+                for image in appliance.images:
+                    if appliance_file == image.get("filename"):
+                        image_checksum = image.get("md5sum")
+                        image_in_db = await images_repo.get_image_by_checksum(image_checksum)
+                        if image_in_db:
+                            version_images[appliance_key] = image_in_db.filename
+                        else:
+                            # check if the image is on disk
+                            image_path = os.path.join(image_dir, appliance_file)
+                            if os.path.exists(image_path) and await wait_run_in_executor(md5sum, image_path) == image_checksum:
+                                async with aiofiles.open(image_path, "rb") as f:
+                                    await write_image(appliance_file, appliance.type, image_path, f, images_repo)
+                            else:
+                                # download the image if there is a direct download URL
+                                direct_download_url = image.get("direct_download_url")
+                                if direct_download_url:
+                                    await self._download_image(
+                                        image_dir,
+                                        appliance_file,
+                                        appliance.type,
+                                        direct_download_url,
+                                        images_repo)
+                                else:
+                                    raise ControllerError(f"Could not find '{appliance_file}'")
+
+    async def install_appliance_from_image(self, image_checksum, images_repo, image_dir):
+        """
+        Find the image checksum in appliance files
+        """
+
+        from . import Controller
+
+        appliance_info = self._find_appliance_from_image_checksum(image_checksum)
+        if appliance_info:
+            appliance, image_version = appliance_info
+            if appliance.versions:
+                for version in appliance.versions:
+                    if version.get("name") == image_version:
+                        await self._find_appliance_version_images(appliance, version, images_repo, image_dir)
+                        # downloading missing custom symbol for this appliance
+                        if appliance.symbol and not appliance.symbol.startswith(":/symbols/"):
+                            destination_path = os.path.join(Controller.instance().symbols.symbols_path(), appliance.symbol)
+                            if not os.path.exists(destination_path):
+                                await self._download_symbol(appliance.symbol, destination_path)
+                        return ApplianceToTemplate().new_template(appliance.asdict(), version, "local")  # FIXME: "local"
+
     def load_appliances(self, symbol_theme="Classic"):
         """
         Loads appliance files from disk.
@@ -98,15 +187,17 @@ class ApplianceManager:
                     if not file.endswith(".gns3a") and not file.endswith(".gns3appliance"):
                         continue
                     path = os.path.join(directory, file)
-                    appliance_id = uuid.uuid3(
-                        uuid.NAMESPACE_URL, path
-                    )  # Generate UUID from path to avoid change between reboots
+                    # Generate UUID from path to avoid change between reboots
+                    appliance_id = uuid.uuid5(
+                        uuid.NAMESPACE_X500,
+                        path
+                    )
                     try:
                         with open(path, encoding="utf-8") as f:
                             appliance = Appliance(appliance_id, json.load(f), builtin=builtin)
                             json_data = appliance.asdict()  # Check if loaded without error
                             if appliance.status != "broken":
-                                self._appliances[appliance.id] = appliance
+                               self._appliances[appliance.id] = appliance
                             if not appliance.symbol or appliance.symbol.startswith(":/symbols/"):
                                 # apply a default symbol if the appliance has none or a default symbol
                                 default_symbol = self._get_default_symbol(json_data, symbol_theme)
@@ -157,6 +248,7 @@ class ApplianceManager:
         """
 
         symbol_url = f"https://raw.githubusercontent.com/GNS3/gns3-registry/master/symbols/{symbol}"
+        log.info(f"Downloading symbol '{symbol}'")
         async with HTTPClient.get(symbol_url) as response:
             if response.status != 200:
                 log.warning(

gns3server/controller/appliance_to_template.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class ApplianceToTemplate:
+    """
+    Appliance installation.
+    """
+
+    def new_template(self, appliance_config, version, server):
+        """
+        Creates a new template from an appliance.
+        """
+
+        new_template = {
+            "compute_id": server,
+            "name": appliance_config["name"],
+            "version": version.get("name")
+        }
+
+        if "usage" in appliance_config:
+            new_template["usage"] = appliance_config["usage"]
+
+        if appliance_config["category"] == "multilayer_switch":
+            new_template["category"] = "switch"
+        else:
+            new_template["category"] = appliance_config["category"]
+
+        if "symbol" in appliance_config:
+            new_template["symbol"] = appliance_config.get("symbol")
+
+        if new_template.get("symbol") is None:
+            if appliance_config["category"] == "guest":
+                if "docker" in appliance_config:
+                    new_template["symbol"] = ":/symbols/docker_guest.svg"
+                else:
+                    new_template["symbol"] = ":/symbols/qemu_guest.svg"
+            elif appliance_config["category"] == "router":
+                new_template["symbol"] = ":/symbols/router.svg"
+            elif appliance_config["category"] == "switch":
+                new_template["symbol"] = ":/symbols/ethernet_switch.svg"
+            elif appliance_config["category"] == "multilayer_switch":
+                new_template["symbol"] = ":/symbols/multilayer_switch.svg"
+            elif appliance_config["category"] == "firewall":
+                new_template["symbol"] = ":/symbols/firewall.svg"
+
+        if "qemu" in appliance_config:
+            new_template["template_type"] = "qemu"
+            self._add_qemu_config(new_template, appliance_config, version)
+        elif "iou" in appliance_config:
+            new_template["template_type"] = "iou"
+            self._add_iou_config(new_template, appliance_config, version)
+        elif "dynamips" in appliance_config:
+            new_template["template_type"] = "dynamips"
+            self._add_dynamips_config(new_template, appliance_config, version)
+        elif "docker" in appliance_config:
+            new_template["template_type"] = "docker"
+            self._add_docker_config(new_template, appliance_config)
+
+        return new_template
+
+    def _add_qemu_config(self, new_config, appliance_config, version):
+
+        new_config.update(appliance_config["qemu"])
+
+        # the following properties are not valid for a template
+        new_config.pop("kvm", None)
+        new_config.pop("path", None)
+        new_config.pop("arch", None)
+
+        options = appliance_config["qemu"].get("options", "")
+        if appliance_config["qemu"].get("kvm", "allow") == "disable" and "-machine accel=tcg" not in options:
+            options += " -machine accel=tcg"
+        new_config["options"] = options.strip()
+        new_config.update(version.get("images"))
+
+        if "path" in appliance_config["qemu"]:
+            new_config["qemu_path"] = appliance_config["qemu"]["path"]
+        else:
+            new_config["qemu_path"] = "qemu-system-{}".format(appliance_config["qemu"]["arch"])
+
+        if "first_port_name" in appliance_config:
+            new_config["first_port_name"] = appliance_config["first_port_name"]
+
+        if "port_name_format" in appliance_config:
+            new_config["port_name_format"] = appliance_config["port_name_format"]
+
+        if "port_segment_size" in appliance_config:
+            new_config["port_segment_size"] = appliance_config["port_segment_size"]
+
+        if "custom_adapters" in appliance_config:
+            new_config["custom_adapters"] = appliance_config["custom_adapters"]
+
+        if "linked_clone" in appliance_config:
+            new_config["linked_clone"] = appliance_config["linked_clone"]
+
+    def _add_docker_config(self, new_config, appliance_config):
+
+        new_config.update(appliance_config["docker"])
+
+        if "custom_adapters" in appliance_config:
+            new_config["custom_adapters"] = appliance_config["custom_adapters"]
+
+    def _add_dynamips_config(self, new_config, appliance_config, version):
+
+        new_config.update(appliance_config["dynamips"])
+        new_config["idlepc"] = version.get("idlepc", "")
+        new_config["image"] = version.get("images").get("image")
+
+    def _add_iou_config(self, new_config, appliance_config, version):
+
+        new_config.update(appliance_config["iou"])
+        new_config["path"] = version.get("images").get("image")

gns3server/controller/symbols.py

@@ -121,6 +121,10 @@ class Symbols:
                 return None
         return directory
 
+    def has_symbol(self, symbol_id):
+
+        return self._symbols_path.get(symbol_id)
+
     def get_path(self, symbol_id):
         try:
             return self._symbols_path[symbol_id]