diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 828e94b2..bca490cc 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -27,10 +27,13 @@ import re import subprocess from ...utils.asyncio import subprocess_check_output +from ...utils.get_resource import get_resource from ..base_manager import BaseManager +from ..error import NodeError, ImageMissingError from .qemu_error import QemuError from .qemu_vm import QemuVM from .utils.guest_cid import get_next_guest_cid +from .utils.ziputils import unpack_zip import logging log = logging.getLogger(__name__) @@ -45,6 +48,8 @@ class Qemu(BaseManager): super().__init__() self._guest_cid_lock = asyncio.Lock() + self.config_disk = "config.img" + self._init_config_disk() async def create_node(self, *args, **kwargs): """ @@ -343,3 +348,21 @@ class Qemu(BaseManager): log.info("Qemu disk '{}' extended by {} MB".format(path, extend)) except (OSError, subprocess.SubprocessError) as e: raise QemuError("Could not update disk image {}:{}".format(path, e)) + + def _init_config_disk(self): + """ + Initialize the default config disk + """ + + try: + self.get_abs_image_path(self.config_disk) + except (NodeError, ImageMissingError): + config_disk_zip = get_resource("compute/qemu/resources/{}.zip".format(self.config_disk)) + if config_disk_zip and os.path.exists(config_disk_zip): + directory = self.get_images_directory() + try: + unpack_zip(config_disk_zip, directory) + except OSError as e: + log.warning("Config disk creation: {}".format(e)) + else: + log.warning("Config disk: image '{}' missing".format(self.config_disk)) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 287ee188..4993a8e6 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -26,6 +26,7 @@ import re import shlex import math import shutil +import struct import asyncio import socket import gns3server @@ -37,7 +38,9 @@ from gns3server.utils import parse_version, shlex_quote from gns3server.utils.asyncio import subprocess_check_output, cancellable_wait_run_in_executor from .qemu_error import QemuError from .utils.qcow2 import Qcow2, Qcow2Error +from .utils.ziputils import pack_zip, unpack_zip from ..adapters.ethernet_adapter import EthernetAdapter +from ..error import NodeError, ImageMissingError from ..nios.nio_udp import NIOUDP from ..nios.nio_tap import NIOTAP from ..base_node import BaseNode @@ -100,10 +103,10 @@ class QemuVM(BaseNode): self._hdb_disk_image = "" self._hdc_disk_image = "" self._hdd_disk_image = "" - self._hda_disk_interface = "ide" - self._hdb_disk_interface = "ide" - self._hdc_disk_interface = "ide" - self._hdd_disk_interface = "ide" + self._hda_disk_interface = "none" + self._hdb_disk_interface = "none" + self._hdc_disk_interface = "none" + self._hdd_disk_interface = "none" self._cdrom_image = "" self._bios_image = "" self._boot_priority = "c" @@ -119,12 +122,28 @@ class QemuVM(BaseNode): self._kernel_command_line = "" self._legacy_networking = False self._replicate_network_connection_state = True + self._create_config_disk = False self._on_close = "power_off" self._cpu_throttling = 0 # means no CPU throttling self._process_priority = "low" self.mac_address = "" # this will generate a MAC address self.adapters = 1 # creates 1 adapter by default + + # config disk + self.config_disk_name = self.manager.config_disk + self.config_disk_image = "" + if self.config_disk_name: + if not shutil.which("mcopy"): + log.warning("Config disk: 'mtools' are not installed.") + self.config_disk_name = "" + else: + try: + self.config_disk_image = self.manager.get_abs_image_path(self.config_disk_name) + except (NodeError, ImageMissingError): + log.warning("Config disk: image '{}' missing".format(self.config_disk_name)) + self.config_disk_name = "" + log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id)) @property @@ -641,6 +660,30 @@ class QemuVM(BaseNode): log.info('QEMU VM "{name}" [{id}] has disabled network connection state replication'.format(name=self._name, id=self._id)) self._replicate_network_connection_state = replicate_network_connection_state + @property + def create_config_disk(self): + """ + Returns whether a config disk is automatically created on HDD disk interface (secondary slave) + + :returns: boolean + """ + + return self._create_config_disk + + @create_config_disk.setter + def create_config_disk(self, create_config_disk): + """ + Sets whether a config disk is automatically created on HDD disk interface (secondary slave) + + :param replicate_network_connection_state: boolean + """ + + if create_config_disk: + log.info('QEMU VM "{name}" [{id}] has enabled the config disk creation feature'.format(name=self._name, id=self._id)) + else: + log.info('QEMU VM "{name}" [{id}] has disabled the config disk creation feature'.format(name=self._name, id=self._id)) + self._create_config_disk = create_config_disk + @property def on_close(self): """ @@ -1124,6 +1167,7 @@ class QemuVM(BaseNode): self._stop_cpulimit() if self.on_close != "save_vm_state": await self._clear_save_vm_stated() + await self._export_config() await super().stop() async def _open_qemu_monitor_connection_vm(self, timeout=10): @@ -1661,6 +1705,105 @@ class QemuVM(BaseNode): log.info("{} returned with {}".format(self._get_qemu_img(), retcode)) return retcode + async def _mcopy(self, image, *args): + try: + # read offset of first partition from MBR + with open(image, "rb") as img_file: + mbr = img_file.read(512) + part_type, offset, signature = struct.unpack("<450xB3xL52xH", mbr) + if signature != 0xAA55: + raise OSError("mcopy failure: {}: invalid MBR".format(image)) + if part_type not in (1, 4, 6, 11, 12, 14): + raise OSError("mcopy failure: {}: invalid partition type {:02X}" + .format(image, part_type)) + part_image = image + "@@{}S".format(offset) + + process = await asyncio.create_subprocess_exec( + "mcopy", "-i", part_image, *args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=self.working_dir) + (stdout, _) = await process.communicate() + retcode = process.returncode + except (OSError, subprocess.SubprocessError) as e: + raise OSError("mcopy failure: {}".format(e)) + if retcode != 0: + stdout = stdout.decode("utf-8").rstrip() + if stdout: + raise OSError("mcopy failure: {}".format(stdout)) + else: + raise OSError("mcopy failure: return code {}".format(retcode)) + + async def _export_config(self): + disk_name = getattr(self, "config_disk_name") + if not disk_name: + return + disk = os.path.join(self.working_dir, disk_name) + if not os.path.exists(disk): + return + config_dir = os.path.join(self.working_dir, "configs") + zip_file = os.path.join(self.working_dir, "config.zip") + try: + os.mkdir(config_dir) + await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir) + if os.path.exists(zip_file): + os.remove(zip_file) + pack_zip(zip_file, config_dir) + except OSError as e: + log.warning("Can't export config: {}".format(e)) + self.project.emit("log.warning", {"message": "{}: Can't export config: {}".format(self._name, e)}) + shutil.rmtree(config_dir, ignore_errors=True) + + async def _import_config(self): + disk_name = getattr(self, "config_disk_name") + zip_file = os.path.join(self.working_dir, "config.zip") + if not disk_name or not os.path.exists(zip_file): + return + config_dir = os.path.join(self.working_dir, "configs") + disk = os.path.join(self.working_dir, disk_name) + disk_tmp = disk + ".tmp" + try: + os.mkdir(config_dir) + shutil.copyfile(getattr(self, "config_disk_image"), disk_tmp) + unpack_zip(zip_file, config_dir) + config_files = [os.path.join(config_dir, fname) + for fname in os.listdir(config_dir)] + if config_files: + await self._mcopy(disk_tmp, "-s", "-m", "-o", "--", *config_files, "::/") + os.replace(disk_tmp, disk) + except OSError as e: + log.warning("Can't import config: {}".format(e)) + self.project.emit("log.warning", {"message": "{}: Can't import config: {}".format(self._name, e)}) + if os.path.exists(disk_tmp): + os.remove(disk_tmp) + os.remove(zip_file) + shutil.rmtree(config_dir, ignore_errors=True) + + def _disk_interface_options(self, disk, disk_index, interface, format=None): + options = [] + extra_drive_options = "" + if format: + extra_drive_options += ",format={}".format(format) + + if interface == "sata": + # special case, sata controller doesn't exist in Qemu + options.extend(["-device", 'ahci,id=ahci{}'.format(disk_index)]) + options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + options.extend(["-device", 'ide-drive,drive=drive{},bus=ahci{}.0,id=drive{}'.format(disk_index, disk_index, disk_index)]) + elif interface == "nvme": + options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + options.extend(["-device", 'nvme,drive=drive{},serial={}'.format(disk_index, disk_index)]) + elif interface == "scsi": + options.extend(["-device", 'virtio-scsi-pci,id=scsi{}'.format(disk_index)]) + options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + options.extend(["-device", 'scsi-hd,drive=drive{}'.format(disk_index)]) + #elif interface == "sd": + # options.extend(["-drive", 'file={},id=drive{},index={}{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + # options.extend(["-device", 'sd-card,drive=drive{},id=drive{}'.format(disk_index, disk_index, disk_index)]) + else: + options.extend(["-drive", 'file={},if={},index={},media=disk,id=drive{}{}'.format(disk, interface, disk_index, disk_index, extra_drive_options)]) + return options + async def _disk_options(self): options = [] qemu_img_path = self._get_qemu_img() @@ -1669,13 +1812,16 @@ class QemuVM(BaseNode): for disk_index, drive in enumerate(drives): disk_image = getattr(self, "_hd{}_disk_image".format(drive)) - interface = getattr(self, "hd{}_disk_interface".format(drive)) - if not disk_image: continue - disk_name = "hd" + drive + interface = getattr(self, "hd{}_disk_interface".format(drive)) + # fail-safe: use "ide" if there is a disk image and no interface type has been explicitly configured + if interface == "none": + interface = "ide" + setattr(self, "hd{}_disk_interface".format(drive), interface) + disk_name = "hd" + drive if not os.path.isfile(disk_image) or not os.path.exists(disk_image): if os.path.islink(disk_image): raise QemuError("{} disk image '{}' linked to '{}' is not accessible".format(disk_name, disk_image, os.path.realpath(disk_image))) @@ -1725,23 +1871,29 @@ class QemuVM(BaseNode): else: disk = disk_image - if interface == "sata": - # special case, sata controller doesn't exist in Qemu - options.extend(["-device", 'ahci,id=ahci{}'.format(disk_index)]) - options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)]) - options.extend(["-device", 'ide-drive,drive=drive{},bus=ahci{}.0,id=drive{}'.format(disk_index, disk_index, disk_index)]) - elif interface == "nvme": - options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)]) - options.extend(["-device", 'nvme,drive=drive{},serial={}'.format(disk_index, disk_index)]) - elif interface == "scsi": - options.extend(["-device", 'virtio-scsi-pci,id=scsi{}'.format(disk_index)]) - options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)]) - options.extend(["-device", 'scsi-hd,drive=drive{}'.format(disk_index)]) - #elif interface == "sd": - # options.extend(["-drive", 'file={},id=drive{},index={}'.format(disk, disk_index, disk_index)]) - # options.extend(["-device", 'sd-card,drive=drive{},id=drive{}'.format(disk_index, disk_index, disk_index)]) + options.extend(self._disk_interface_options(disk, disk_index, interface)) + + # config disk + disk_image = getattr(self, "config_disk_image") + if disk_image and self._create_config_disk: + if getattr(self, "_hdd_disk_image"): + log.warning("Config disk: blocked by disk image 'hdd'") else: - options.extend(["-drive", 'file={},if={},index={},media=disk,id=drive{}'.format(disk, interface, disk_index, disk_index)]) + disk_name = getattr(self, "config_disk_name") + disk = os.path.join(self.working_dir, disk_name) + if self.hdd_disk_interface == "none": + # use the HDA interface type if none has been configured for HDD + self.hdd_disk_interface = getattr(self, "hda_disk_interface", "none") + await self._import_config() + disk_exists = os.path.exists(disk) + if not disk_exists: + try: + shutil.copyfile(disk_image, disk) + disk_exists = True + except OSError as e: + log.warning("Could not create '{}' disk image: {}".format(disk_name, e)) + if disk_exists: + options.extend(self._disk_interface_options(disk, 3, self.hdd_disk_interface, "raw")) return options diff --git a/gns3server/compute/qemu/resources/config.img.zip b/gns3server/compute/qemu/resources/config.img.zip new file mode 100644 index 00000000..7ba43f9e Binary files /dev/null and b/gns3server/compute/qemu/resources/config.img.zip differ diff --git a/gns3server/compute/qemu/utils/ziputils.py b/gns3server/compute/qemu/utils/ziputils.py new file mode 100644 index 00000000..3ff8c999 --- /dev/null +++ b/gns3server/compute/qemu/utils/ziputils.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import time +import shutil +import zipfile + +def pack_zip(filename, root_dir=None, base_dir=None): + """Create a zip archive""" + + if filename[-4:].lower() == ".zip": + filename = filename[:-4] + shutil.make_archive(filename, 'zip', root_dir, base_dir) + +def unpack_zip(filename, extract_dir=None): + """Unpack a zip archive""" + + dirs = [] + if not extract_dir: + extract_dir = os.getcwd() + + try: + with zipfile.ZipFile(filename, 'r') as zfile: + for zinfo in zfile.infolist(): + fname = os.path.join(extract_dir, zinfo.filename) + date_time = time.mktime(zinfo.date_time + (0, 0, -1)) + zfile.extract(zinfo, extract_dir) + + # update timestamp + if zinfo.is_dir(): + dirs.append((fname, date_time)) + else: + os.utime(fname, (date_time, date_time)) + # update timestamp of directories + for fname, date_time in reversed(dirs): + os.utime(fname, (date_time, date_time)) + except zipfile.BadZipFile: + raise shutil.ReadError("%s is not a zip file" % filename) diff --git a/gns3server/schemas/qemu.py b/gns3server/schemas/qemu.py index a029e40c..41a79d50 100644 --- a/gns3server/schemas/qemu.py +++ b/gns3server/schemas/qemu.py @@ -206,6 +206,10 @@ QEMU_CREATE_SCHEMA = { "description": "Replicate the network connection state for links in Qemu", "type": ["boolean", "null"], }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": ["boolean", "null"], + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], @@ -412,6 +416,10 @@ QEMU_UPDATE_SCHEMA = { "description": "Replicate the network connection state for links in Qemu", "type": ["boolean", "null"], }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": ["boolean", "null"], + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], @@ -631,6 +639,10 @@ QEMU_OBJECT_SCHEMA = { "description": "Replicate the network connection state for links in Qemu", "type": "boolean", }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": ["boolean", "null"], + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], @@ -704,6 +716,7 @@ QEMU_OBJECT_SCHEMA = { "kernel_command_line", "legacy_networking", "replicate_network_connection_state", + "create_config_disk", "on_close", "cpu_throttling", "process_priority", diff --git a/gns3server/schemas/qemu_template.py b/gns3server/schemas/qemu_template.py index eb3c2235..9d12e42e 100644 --- a/gns3server/schemas/qemu_template.py +++ b/gns3server/schemas/qemu_template.py @@ -123,7 +123,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hda_disk_interface": { "description": "QEMU hda interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "hdb_disk_image": { "description": "QEMU hdb disk image path", @@ -133,7 +133,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hdb_disk_interface": { "description": "QEMU hdb interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "hdc_disk_image": { "description": "QEMU hdc disk image path", @@ -143,7 +143,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hdc_disk_interface": { "description": "QEMU hdc interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "hdd_disk_image": { "description": "QEMU hdd disk image path", @@ -153,7 +153,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hdd_disk_interface": { "description": "QEMU hdd interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "cdrom_image": { "description": "QEMU cdrom image path", @@ -190,6 +190,11 @@ QEMU_TEMPLATE_PROPERTIES = { "type": "boolean", "default": True }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": "boolean", + "default": True + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], diff --git a/tests/compute/qemu/test_qemu_vm.py b/tests/compute/qemu/test_qemu_vm.py index 51e53468..c52d3334 100644 --- a/tests/compute/qemu/test_qemu_vm.py +++ b/tests/compute/qemu/test_qemu_vm.py @@ -337,6 +337,7 @@ def test_set_qemu_path_kvm_binary(vm, fake_qemu_binary): async def test_set_platform(compute_project, manager): + manager.config_disk = None # avoids conflict with config.img support with patch("shutil.which", return_value="/bin/qemu-system-x86_64") as which_mock: with patch("gns3server.compute.qemu.QemuVM._check_qemu_path"): vm = QemuVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", compute_project, manager, platform="x86_64") diff --git a/tests/compute/test_manager.py b/tests/compute/test_manager.py index bceb9e20..e257e763 100644 --- a/tests/compute/test_manager.py +++ b/tests/compute/test_manager.py @@ -18,7 +18,7 @@ import uuid import os import pytest -from unittest.mock import patch +from unittest.mock import patch, MagicMock from tests.utils import asyncio_patch from gns3server.compute.vpcs import VPCS @@ -41,6 +41,7 @@ async def vpcs(loop, port_manager): async def qemu(loop, port_manager): Qemu._instance = None + Qemu._init_config_disk = MagicMock() # do not create the config.img image qemu = Qemu.instance() qemu.port_manager = port_manager return qemu diff --git a/tests/conftest.py b/tests/conftest.py index d94b4bc2..7311b7e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -117,8 +117,8 @@ def images_dir(config): path = config.get_section_config("Server").get("images_path") os.makedirs(path, exist_ok=True) - os.makedirs(os.path.join(path, "QEMU")) - os.makedirs(os.path.join(path, "IOU")) + os.makedirs(os.path.join(path, "QEMU"), exist_ok=True) + os.makedirs(os.path.join(path, "IOU"), exist_ok=True) return path diff --git a/tests/handlers/api/compute/test_qemu.py b/tests/handlers/api/compute/test_qemu.py index 3c349797..45636af9 100644 --- a/tests/handlers/api/compute/test_qemu.py +++ b/tests/handlers/api/compute/test_qemu.py @@ -277,7 +277,8 @@ async def test_images(compute_api, fake_qemu_vm): response = await compute_api.get("/qemu/images") assert response.status == 200 - assert response.json == [{"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}] + assert response.json == [{'filename': 'config.img', 'filesize': 1048576, 'md5sum': '0ab49056760ae1db6c25376446190b47', 'path': 'config.img'}, + {"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}] async def test_upload_image(compute_api, tmpdir): diff --git a/tests/handlers/api/controller/test_template.py b/tests/handlers/api/controller/test_template.py index c9eaf75a..2aeff91f 100644 --- a/tests/handlers/api/controller/test_template.py +++ b/tests/handlers/api/controller/test_template.py @@ -669,13 +669,13 @@ async def test_qemu_template_create(controller_api): "default_name_format": "{name}-{0}", "first_port_name": "", "hda_disk_image": "IOSvL2-15.2.4.0.55E.qcow2", - "hda_disk_interface": "ide", + "hda_disk_interface": "none", "hdb_disk_image": "", - "hdb_disk_interface": "ide", + "hdb_disk_interface": "none", "hdc_disk_image": "", - "hdc_disk_interface": "ide", + "hdc_disk_interface": "none", "hdd_disk_image": "", - "hdd_disk_interface": "ide", + "hdd_disk_interface": "none", "initrd": "", "kernel_command_line": "", "kernel_image": "",