# # Copyright (C) 2014 GNS3 Technologies Inc. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . import os import hashlib import stat import aiofiles import shutil from typing import AsyncGenerator from ..config import Config from . import force_unix_path import gns3server.db.models as models from gns3server.db.repositories.images import ImagesRepository import logging log = logging.getLogger(__name__) def list_images(image_type): """ Scan directories for available image for a given type. :param image_type: image type (dynamips, qemu, iou) """ files = set() images = [] server_config = Config.instance().settings.Server general_images_directory = os.path.expanduser(server_config.images_path) # Subfolder of the general_images_directory specific to this VM type default_directory = default_images_directory(image_type) for directory in images_directories(image_type): # We limit recursion to path outside the default images directory # the reason is in the default directory manage file organization and # it should be flatten to keep things simple recurse = True if os.path.commonprefix([directory, general_images_directory]) == general_images_directory: recurse = False directory = os.path.normpath(directory) for root, _, filenames in _os_walk(directory, recurse=recurse): for filename in filenames: path = os.path.join(root, filename) if filename not in files: if filename.endswith(".md5sum") or filename.startswith("."): continue elif ( ((filename.endswith(".image") or filename.endswith(".bin")) and image_type == "dynamips") or ((filename.endswith(".bin") or filename.startswith("i86bi")) and image_type == "iou") or (not filename.endswith(".bin") and not filename.endswith(".image") and image_type == "qemu") ): files.add(filename) # It the image is located in the standard directory the path is relative if os.path.commonprefix([root, default_directory]) != default_directory: path = os.path.join(root, filename) else: path = os.path.relpath(os.path.join(root, filename), default_directory) try: if image_type in ["dynamips", "iou"]: with open(os.path.join(root, filename), "rb") as f: # read the first 7 bytes of the file. elf_header_start = f.read(7) # valid IOS images must start with the ELF magic number, be 32-bit, big endian and have an ELF version of 1 if ( not elf_header_start == b"\x7fELF\x01\x02\x01" and not elf_header_start == b"\x7fELF\x01\x01\x01" ): continue images.append( { "filename": filename, "path": force_unix_path(path), "md5sum": md5sum(os.path.join(root, filename)), "filesize": os.stat(os.path.join(root, filename)).st_size, } ) except OSError as e: log.warning(f"Can't add image {path}: {str(e)}") return images def _os_walk(directory, recurse=True, **kwargs): """ Work like os.walk but if recurse is False just list current directory """ if recurse: for root, dirs, files in os.walk(directory, **kwargs): yield root, dirs, files else: files = [] for filename in os.listdir(directory): if os.path.isfile(os.path.join(directory, filename)): files.append(filename) yield directory, [], files def default_images_directory(image_type): """ :returns: Return the default directory for an image type. """ server_config = Config.instance().settings.Server img_dir = os.path.expanduser(server_config.images_path) if image_type == "qemu": return os.path.join(img_dir, "QEMU") elif image_type == "iou": return os.path.join(img_dir, "IOU") elif image_type == "dynamips" or image_type == "ios": return os.path.join(img_dir, "IOS") else: raise NotImplementedError(f"%s node type is not supported", image_type) def images_directories(type): """ Return all directories where we will look for images by priority :param type: Type of emulator """ server_config = Config.instance().settings.Server paths = [] img_dir = os.path.expanduser(server_config.images_path) type_img_directory = default_images_directory(type) try: os.makedirs(type_img_directory, exist_ok=True) paths.append(type_img_directory) except (OSError, PermissionError): pass for directory in server_config.additional_images_paths: paths.append(directory) # Compatibility with old topologies we look in parent directory paths.append(img_dir) # Return only the existing paths return [force_unix_path(p) for p in paths if os.path.exists(p)] def md5sum(path, stopped_event=None): """ Return the md5sum of an image and cache it on disk :param path: Path to the image :param stopped_event: In case you execute this function on thread and would like to have possibility to cancel operation pass the `threading.Event` :returns: Digest of the image """ if path is None or len(path) == 0 or not os.path.exists(path): return None try: with open(path + ".md5sum") as f: md5 = f.read().strip() if len(md5) == 32: return md5 # Unicode error is when user rename an image to .md5sum .... except (OSError, UnicodeDecodeError): pass try: m = hashlib.md5() with open(path, "rb") as f: while True: if stopped_event is not None and stopped_event.is_set(): log.error(f"MD5 sum calculation of `{path}` has stopped due to cancellation") return buf = f.read(128) if not buf: break m.update(buf) digest = m.hexdigest() except OSError as e: log.error("Can't create digest of %s: %s", path, str(e)) return None try: with open(f"{path}.md5sum", "w+") as f: f.write(digest) except OSError as e: log.error("Can't write digest of %s: %s", path, str(e)) return digest def remove_checksum(path): """ Remove the checksum of an image from cache if exists """ path = f"{path}.md5sum" if os.path.exists(path): os.remove(path) class InvalidImageError(Exception): def __init__(self, message: str): super().__init__() self._message = message def __str__(self): return self._message def check_valid_image_header(data: bytes) -> str: if data[:7] == b'\x7fELF\x01\x02\x01': # for IOS images: file must start with the ELF magic number, be 32-bit, big endian and have an ELF version of 1 return "ios" elif data[:7] == b'\x7fELF\x01\x01\x01' or data[:7] == b'\x7fELF\x02\x01\x01': # for IOU images file must start with the ELF magic number, be 32-bit or 64-bit, little endian and # have an ELF version of 1 (normal IOS images are big endian!) return "iou" elif data[:4] != b'QFI\xfb' or data[:4] != b'KDMV': return "qemu" else: raise InvalidImageError("Could not detect image type, please make sure it is a valid image") async def write_image( image_filename: str, image_path: str, stream: AsyncGenerator[bytes, None], images_repo: ImagesRepository, check_image_header=True ) -> models.Image: image_dir, image_name = os.path.split(image_filename) log.info(f"Writing image file to '{image_path}'") # Store the file under its final name only when the upload is completed tmp_path = image_path + ".tmp" os.makedirs(os.path.dirname(image_path), exist_ok=True) checksum = hashlib.md5() header_magic_len = 7 image_type = None try: async with aiofiles.open(tmp_path, "wb") as f: async for chunk in stream: if check_image_header and len(chunk) >= header_magic_len: check_image_header = False image_type = check_valid_image_header(chunk) await f.write(chunk) checksum.update(chunk) image_size = os.path.getsize(tmp_path) if not image_size or image_size < header_magic_len: raise InvalidImageError("The image content is empty or too small to be valid") checksum = checksum.hexdigest() duplicate_image = await images_repo.get_image_by_checksum(checksum) if duplicate_image and os.path.dirname(duplicate_image.path) == os.path.dirname(image_path): raise InvalidImageError(f"Image {duplicate_image.filename} with " f"same checksum already exists in the same directory") except InvalidImageError: os.remove(tmp_path) raise os.chmod(tmp_path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC) if not image_dir: directory = default_images_directory(image_type) os.makedirs(directory, exist_ok=True) image_path = os.path.abspath(os.path.join(directory, image_filename)) shutil.move(tmp_path, image_path) return await images_repo.add_image(image_name, image_type, image_size, image_path, checksum, checksum_algorithm="md5")