Merge branch '3.1' into fix/duplicate-device-configs-merge

gns3server/utils/image_inventory.py

@@ -3,7 +3,9 @@
 import asyncio
 import hashlib
 import os
+import re
 import stat
+import uuid
 
 from gns3server.config import Config
 
@@ -97,3 +99,144 @@ def publish_image(temporary, destination):
     """
     os.link(temporary, destination)
     os.unlink(temporary)
+
+
+def validate_image_subdirectory(value):
+    """A portable relative folder below the server-selected image type root."""
+    if not value:
+        return []
+    parts = value.split("/")
+    if len(value) > 512 or len(parts) > 8:
+        raise ValueError("Image subfolder is too long or has more than eight levels")
+    for part in parts:
+        if (
+            not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9 ._-]{0,63}", part)
+            or part.endswith((".", " ", ".tmp", ".md5sum"))
+            or part.lower() in ("lib", "lib64")
+            or re.fullmatch(r"(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?", part)
+        ):
+            raise ValueError(
+                "Use relative subfolders with letters, numbers, spaces, dots, hyphens or underscores; "
+                "hidden, reserved and traversal names are not allowed"
+            )
+    return parts
+
+
+class ImageUploadDirectory:
+    """Keep upload operations anchored to the authorized directory.
+
+    POSIX operations use directory descriptors and never follow descendant
+    symlinks. The portable fallback rejects symlinks/junctions and rechecks the
+    directory before each operation. The configured root is administrator-owned.
+    """
+
+    def __init__(self, root, path):
+        self.root = os.path.realpath(os.path.expanduser(root))
+        self.path = os.path.abspath(path)
+        if contained_path(self.path, normalized_path(root)):
+            self.path = os.path.join(self.root, os.path.relpath(self.path, normalized_path(root)))
+        self.fd = None
+        self.identity = None
+        self.anchored = (
+            all(operation in os.supports_dir_fd for operation in (os.open, os.mkdir, os.link, os.unlink))
+            and hasattr(os, "O_NOFOLLOW")
+            and hasattr(os, "O_DIRECTORY")
+        )
+
+    def __enter__(self):
+        if not contained_path(self.path, self.root):
+            raise OSError("Image destination is outside the configured image directory")
+        os.makedirs(self.root, exist_ok=True)
+        parts = os.path.relpath(self.path, self.root).split(os.sep)
+        if parts == ["."]:
+            parts = []
+        try:
+            current = self.root
+            if self.anchored:
+                self.fd = os.open(current, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+            for part in parts:
+                if self.anchored:
+                    try:
+                        os.mkdir(part, mode=0o755, dir_fd=self.fd)
+                    except FileExistsError:
+                        pass
+                    child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=self.fd)
+                    os.close(self.fd)
+                    self.fd = child
+                else:
+                    self._check_components(current)
+                    current = os.path.join(current, part)
+                    try:
+                        os.mkdir(current, mode=0o755)
+                    except FileExistsError:
+                        pass
+                    self._check_components(current)
+            info = os.fstat(self.fd) if self.anchored else os.stat(self.path, follow_symlinks=False)
+            self.identity = (info.st_dev, info.st_ino)
+            self.verify()
+            return self
+        except BaseException:
+            self.__exit__()
+            raise
+
+    def _check_components(self, path):
+        current = self.root
+        relative = os.path.relpath(path, self.root)
+        for part in [] if relative == "." else relative.split(os.sep):
+            current = os.path.join(current, part)
+            info = os.lstat(current)
+            reparse = getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
+            if stat.S_ISLNK(info.st_mode) or reparse:
+                raise OSError("Image upload folders must not be symlinks or junctions")
+
+    def verify(self):
+        self._check_components(self.path)
+        info = os.stat(self.path, follow_symlinks=False)
+        if (info.st_dev, info.st_ino) != self.identity or not contained_path(os.path.realpath(self.path), self.root):
+            raise OSError("Image upload directory changed during upload")
+
+    def temporary(self):
+        self.verify()
+        name = f".gns3-upload-{uuid.uuid4().hex}.tmp"
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
+        if self.anchored:
+            fd = os.open(name, flags | os.O_NOFOLLOW, 0o700, dir_fd=self.fd)
+        else:
+            fd = os.open(os.path.join(self.path, name), flags, 0o700)
+        return fd, name
+
+    def publish(self, temporary, filename):
+        self.verify()
+        if self.anchored:
+            os.link(temporary, filename, src_dir_fd=self.fd, dst_dir_fd=self.fd, follow_symlinks=False)
+        else:
+            os.link(os.path.join(self.path, temporary), os.path.join(self.path, filename))
+        self.remove(temporary)
+        self.verify()
+        return fingerprint(os.path.join(self.path, filename))
+
+    def remove(self, name):
+        if self.anchored:
+            os.unlink(name, dir_fd=self.fd)
+        else:
+            self.verify()
+            os.unlink(os.path.join(self.path, name))
+
+    def __exit__(self, *args):
+        if self.fd is not None:
+            os.close(self.fd)
+            self.fd = None
+
+
+def validate_image_upload_name(name):
+    if (
+        not name
+        or len(name) > 255
+        or name.startswith(".")
+        or name.endswith((".", " ", ".tmp", ".md5sum"))
+        or any(ord(c) < 32 or ord(c) == 127 or c in '/\\:%<>"|?*' for c in name)
+        or re.fullmatch(r"(?i)(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?", name)
+    ):
+        raise ValueError(
+            "Subfolder uploads require a plain, non-hidden image filename without path separators or reserved characters"
+        )

gns3server/utils/images.py

@@ -18,11 +18,18 @@ import asyncio
 import hashlib
 import os
 import stat
-import tempfile
 
 import aiofiles
 
-from gns3server.utils.image_inventory import contained_path, fingerprint, image_lock, publish_image, stat_fingerprint
+from gns3server.utils.image_inventory import (
+    ImageUploadDirectory,
+    contained_path,
+    fingerprint,
+    image_lock,
+    stat_fingerprint,
+    validate_image_subdirectory,
+    validate_image_upload_name,
+)
 
 try:
     import importlib_resources
@@ -365,9 +372,14 @@ async def write_image(
     images_repo: ImagesRepository,
     check_image_header=True,
     allow_raw_image=False,
+    subdirectory=None,
 ) -> models.Image:
 
     image_dir, image_name = os.path.split(image_filename)
+    subfolders = []
+    if subdirectory is not None:
+        validate_image_upload_name(image_filename)
+        subfolders = validate_image_subdirectory(subdirectory)
     # HTTP chunk boundaries need not align with the seven-byte image header.
     iterator = stream.__aiter__()
     prefix = bytearray()
@@ -379,55 +391,68 @@ async def write_image(
         raise InvalidImageError("The image content is empty or too small to be valid")
     image_type = check_valid_image_header(image_path, bytes(prefix), allow_raw_image or not check_image_header)
     if not image_dir:
-        image_path = os.path.abspath(os.path.join(default_images_directory(image_type), image_name))
+        image_path = os.path.abspath(os.path.join(default_images_directory(image_type), *subfolders, image_name))
         root = os.path.realpath(os.path.expanduser(Config.instance().settings.Server.images_path))
         if not contained_path(os.path.realpath(image_path), root):
             raise InvalidImageError(f"Image destination is outside the configured image directory: {image_path}")
-    os.makedirs(os.path.dirname(image_path), exist_ok=True)
-    descriptor, tmp_path = tempfile.mkstemp(prefix=".gns3-upload-", suffix=".tmp", dir=os.path.dirname(image_path))
-    os.close(descriptor)
-    checksum = hashlib.md5()
-    try:
-        async with aiofiles.open(tmp_path, "wb") as f:
-            await f.write(prefix)
-            checksum.update(prefix)
-            async for chunk in iterator:
-                await f.write(chunk)
-                checksum.update(chunk)
-        image_size = os.path.getsize(tmp_path)
-        async with image_lock(image_path):
-            if os.path.lexists(image_path):
-                raise InvalidImageError(
-                    f"File '{image_path}' already exists, please choose a different name or remove the existing image"
-                )
-            checksum_str: str = checksum.hexdigest()
-            duplicate_image = await images_repo.get_image_by_checksum(checksum_str, os.path.dirname(image_path))
-            if duplicate_image:
-                raise InvalidImageError(
-                    f"Image '{duplicate_image.filename}' with the same checksum "
-                    f"already exists in '{os.path.dirname(image_path)}'"
-                )
-            os.chmod(tmp_path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
-            publish_image(tmp_path, image_path)
-            # Complete files survive a database failure so the next scan can
-            # recover them. Never compensate by unlinking a published image.
-            image = await images_repo.save_verified_image(
-                dict(
-                    image_name=image_name,
-                    image_type=image_type,
-                    image_size=image_size,
-                    path=image_path,
-                    checksum=checksum_str,
-                    checksum_algorithm="md5",
-                    file_fingerprint=fingerprint(image_path),
-                )
-            )
-            if image is None:
-                raise InvalidImageError(f"Failed to save image '{image_name}' to database")
-            return image
-    finally:
+    root = Config.instance().settings.Server.images_path
+    with ImageUploadDirectory(root, os.path.dirname(image_path)) as directory:
+        descriptor, temporary = directory.temporary()
+        checksum = hashlib.md5()
         try:
-            if os.path.exists(tmp_path):
-                os.remove(tmp_path)
-        except OSError:
-            log.warning(f"Could not remove '{tmp_path}'")
+            # Own the descriptor explicitly so cancellation cannot leak it.
+            async with aiofiles.open(descriptor, "wb", closefd=False) as f:
+                await f.write(prefix)
+                checksum.update(prefix)
+                async for chunk in iterator:
+                    await f.write(chunk)
+                    checksum.update(chunk)
+            image_size = os.fstat(descriptor).st_size
+            # Preserve executable IOU permissions even under a restrictive umask.
+            permissions = stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC
+            if hasattr(os, "fchmod"):
+                os.fchmod(descriptor, permissions)
+            else:
+                directory.verify()
+                os.chmod(os.path.join(directory.path, temporary), permissions)
+            os.close(descriptor)
+            descriptor = None
+            async with image_lock(image_path):
+                directory.verify()
+                if os.path.lexists(image_path):
+                    raise InvalidImageError(
+                        f"File '{image_path}' already exists, "
+                        f"please choose a different name or remove the existing image"
+                    )
+                checksum_str = checksum.hexdigest()
+                duplicate_image = await images_repo.get_image_by_checksum(checksum_str, os.path.dirname(image_path))
+                if duplicate_image:
+                    raise InvalidImageError(
+                        f"Image '{duplicate_image.filename}' with the same checksum "
+                        f"already exists in '{os.path.dirname(image_path)}'"
+                    )
+                file_fingerprint = directory.publish(temporary, image_name)
+                # Complete files survive a database failure for reconciliation.
+                image = await images_repo.save_verified_image(
+                    dict(
+                        image_name=image_name,
+                        image_type=image_type,
+                        image_size=image_size,
+                        path=image_path,
+                        checksum=checksum_str,
+                        checksum_algorithm="md5",
+                        file_fingerprint=file_fingerprint,
+                    )
+                )
+                if image is None:
+                    raise InvalidImageError(f"Failed to save image '{image_name}' to database")
+                return image
+        finally:
+            if descriptor is not None:
+                os.close(descriptor)
+            try:
+                directory.remove(temporary)
+            except FileNotFoundError:
+                pass
+            except OSError:
+                log.warning("Could not remove temporary image '%s'", temporary)