docker: replace vendor SKIP_INIT exec volume bridge with create-time direct binds

The SKIP_INIT volume bridge replicated init.sh's seed + mount --bind script
via docker exec *after* the container started. That copied the mechanism but
not the invariant that makes init.sh safe — the entrypoint position, which
guarantees the volume is in place before the application runs. The exec runs
concurrently with the NOS boot, so whether the NOS loaded its persisted
config or the overlay's factory copy was a timing race:

- single node stop/start on an idle system won it (exec ~1s, SR Linux reads
  its startup config at ~2-4s) — the save/stop/start round-trip passed;
- a server restart + project reload lost it (concurrent node starts queue on
  the Docker API, delaying the exec by seconds) — SR Linux booted factory
  while the persisted config.json sat intact on the host;
- XRd was immune (systemd boots tens of seconds before XR touches
  /xr-storage), which is why the race was never observed on it.

Replace the bridge entirely:

- new DockerVM._prepare_volumes hook (no-op in the base class) runs in
  create() after the image is present, before the container is created;
  VendorDockerVM overrides it to seed each volume's host directory from the
  image (throwaway docker create container + docker cp -a, nothing
  executes). The .gns3_perms marker gates the seeding: a volume that ever
  started is never re-seeded, so saved configuration is never overwritten
  with factory content (also the upgrade path for existing nodes).
- VendorDockerVM._mount_binds now binds the volumes directly at their real
  in-container paths (/etc/opt/srlinux) instead of /gns3volumes aliases, so
  the persisted config is visible to the NOS from the very first process.
- _setup_skip_init_volumes and its start() call are gone; the container-side
  _fix_permissions targets the volume paths directly (the direct binds
  exist for the whole container lifetime, unlike the old bridge).

The volume-list computation (validation + overlap de-duplication) moves
into DockerVM._persistent_volume_list so create-time seeding and _mount_binds
cannot drift apart.

gns3server/compute/docker/docker_vm.py

@@ -379,6 +379,51 @@ class DockerVM(BaseNode):
         result = await self.manager.query("GET", f"images/{self._image}/json")
         return result
 
+    def _persistent_volume_list(self, image_info, include_network_config=True):
+        """
+        The in-container paths that get a persistent volume mount: GNS3's
+        /etc/network, every VOLUME declared by the image and the node's
+        extra_volumes. Overlapping paths are de-duplicated so that a path
+        covered by a more general volume is not mounted twice.
+
+        :param include_network_config: include GNS3's hardcoded /etc/network
+            volume (consumed by init.sh; subclasses that skip init.sh pass
+            False so the list matches the mounts they actually create).
+        """
+
+        for volume in self._extra_volumes:
+            if not volume.strip() or volume[0] != "/" or volume.find("..") >= 0:
+                raise DockerError(
+                    f"Persistent volume '{volume}' has invalid format. It must start with a '/' and not contain '..'."
+                )
+        volumes = []
+        if include_network_config:
+            volumes.append("/etc/network")
+        volumes.extend((image_info.get("Config", {}).get("Volumes") or {}).keys())
+        volumes.extend(self._extra_volumes)
+
+        deduped = []
+        # define lambdas for validation checks
+        nf = lambda x: re.sub(r"//+", "/", (x if x.endswith("/") else x + "/"))
+        generalises = lambda v1, v2: nf(v2).startswith(nf(v1))
+        for volume in volumes:
+            # remove any mount that is equal or more specific, then append this one
+            deduped = list(filter(lambda v: not generalises(volume, v), deduped))
+            # if there is nothing more general, append this mount
+            if not [v for v in deduped if generalises(v, volume)]:
+                deduped.append(volume)
+        return deduped
+
+    async def _prepare_volumes(self, image_info):
+        """
+        Hook: prepare persistent volumes before the container (and its
+        mounts) are created. The default implementation does nothing —
+        init.sh performs the first-copy seeding inside the container at
+        boot. Subclasses that skip init.sh override this to seed the host
+        directories from the image instead, so their mounts can be bound
+        directly at the real in-container paths from the very first process.
+        """
+
     def _mount_binds(self, image_info):
         """
         :returns: Return the path that we need to map to local folders
@@ -402,26 +447,7 @@ class DockerVM(BaseNode):
             self._create_network_config()
         except OSError as e:
             raise DockerError(f"Could not create network config in the container: {e}")
-        volumes = ["/etc/network"]
-
-        volumes.extend((image_info.get("Config", {}).get("Volumes") or {}).keys())
-        for volume in self._extra_volumes:
-            if not volume.strip() or volume[0] != "/" or volume.find("..") >= 0:
-                raise DockerError(
-                    f"Persistent volume '{volume}' has invalid format. It must start with a '/' and not contain '..'."
-                )
-        volumes.extend(self._extra_volumes)
-
-        self._volumes = []
-        # define lambdas for validation checks
-        nf = lambda x: re.sub(r"//+", "/", (x if x.endswith("/") else x + "/"))
-        generalises = lambda v1, v2: nf(v2).startswith(nf(v1))
-        for volume in volumes:
-            # remove any mount that is equal or more specific, then append this one
-            self._volumes = list(filter(lambda v: not generalises(volume, v), self._volumes))
-            # if there is nothing more general, append this mount
-            if not [v for v in self._volumes if generalises(v, volume)]:
-                self._volumes.append(volume)
+        self._volumes = self._persistent_volume_list(image_info)
 
         for volume in self._volumes:
             source = os.path.join(self.working_dir, os.path.relpath(volume, "/"))
@@ -544,6 +570,10 @@ class DockerVM(BaseNode):
                 f"(max available is {available_cpus} CPUs)"
             )
 
+        # Prepare persistent volume content before the container and its
+        # mounts are created (no-op for the init.sh path).
+        await self._prepare_volumes(image_infos)
+
         params = {
             "Hostname": self._name,
             "Image": self._image,

gns3server/compute/docker/vendor_docker_vm.py

@@ -48,9 +48,11 @@ class VendorDockerVM(DockerVM):
     (host-side only — GNS3_ entries are never forwarded into the container):
 
     * ``GNS3_SKIP_INIT=1`` — do not prepend /gns3/init.sh; the container runs
-      its own entrypoint (e.g. SR Linux's ``sr_linux``). Init.sh's volume
-      persistence (bind-mount /gns3volumes → target) is replicated via
-      ``docker exec`` after the container starts.
+      its own entrypoint (e.g. SR Linux's ``sr_linux``). Persistent volumes
+      are seeded host-side and bound directly at their real in-container
+      paths at create time (see ``_prepare_volumes`` / ``_mount_binds``), so
+      the NOS sees its saved configuration from the very first process —
+      no post-start mount pass that could race the NOS reading its config.
     * ``GNS3_INTERFACE_NAMES=mgmt0,e1-1,e1-2`` — rename injected interfaces
       (adapter order) instead of default ``eth{N}``.
     * ``GNS3_CONSOLE_CMD=/opt/srlinux/bin/sr_cli`` — command run inside the
@@ -127,6 +129,17 @@ class VendorDockerVM(DockerVM):
         Removes the bind, drops the volume from self._volumes (so
         GNS3_VOLUMES and the vendor passes stay consistent) and deletes the
         host-side skeleton directory the base class just created.
+
+        Additionally, the persistent volumes are bound directly at their
+        real in-container paths instead of /gns3volumes<volume>. With
+        init.sh skipped there is no in-container mount pass, so a volume
+        bound at /gns3volumes would only be moved into place by a post-start
+        ``docker exec`` — racing the NOS reading its startup configuration
+        (an SR Linux node booted factory whenever the exec lost that race,
+        e.g. on the concurrent node starts of a project reload). Binding at
+        the real path is safe because the content is seeded host-side before
+        the container is created (see _prepare_volumes): the image's files
+        are never shadowed by an empty mount.
         """
         binds = super()._mount_binds(image_info)
         if self._gns3_init:
@@ -136,7 +149,115 @@ class VendorDockerVM(DockerVM):
         shutil.rmtree(os.path.join(self.working_dir, "etc", "network"), ignore_errors=True)
         with contextlib.suppress(OSError):
             os.rmdir(os.path.join(self.working_dir, "etc"))
-        return binds
+
+        # Re-target the volume binds from /gns3volumes<volume> to <volume>.
+        retargeted = []
+        for bind in binds:
+            target = bind.get("Target", "")
+            if target.startswith("/gns3volumes"):
+                volume = target[len("/gns3volumes"):]
+                if volume in self._volumes:
+                    bind = {**bind, "Target": volume}
+            retargeted.append(bind)
+        return retargeted
+
+    async def _prepare_volumes(self, image_info):
+        """
+        Override: for SKIP_INIT containers, seed every persistent volume's
+        host directory with the image's original content *before* the
+        container is created. This is the host-side replacement of init.sh's
+        first-copy: because the volume is then bound directly at its real
+        in-container path (see _mount_binds), the seed must exist first or
+        the NOS would boot with an empty config directory.
+
+        ``.gns3_perms`` doubles as the seeded marker: a volume that has it
+        (every node that ever started, on any GNS3 version) is never
+        re-seeded — a re-seed would overwrite the node's saved
+        configuration with the factory image content.
+        """
+        if self._gns3_init:
+            return
+        volumes = self._persistent_volume_list(image_info, include_network_config=False)
+        to_seed = []
+        for volume in volumes:
+            host_dir = os.path.join(self.working_dir, os.path.relpath(volume, "/"))
+            os.makedirs(host_dir, exist_ok=True)
+            if not os.path.exists(os.path.join(host_dir, ".gns3_perms")):
+                to_seed.append((volume, host_dir))
+        if not to_seed:
+            return
+        seed_cid = await self._create_seed_container()
+        try:
+            for volume, host_dir in to_seed:
+                await self._seed_volume_from_container(seed_cid, volume, host_dir)
+                # Write the marker only after the copy attempt, mirroring
+                # init.sh: a volume without it is (re)seeded on the next
+                # create(), so a partial seed self-heals.
+                open(os.path.join(host_dir, ".gns3_perms"), "a").close()
+        finally:
+            await self._remove_seed_container(seed_cid)
+
+    async def _create_seed_container(self):
+        """
+        A throwaway ``docker create`` container (nothing executes) used as
+        the copy source for seeding persistent volumes with the image's
+        original content.
+        """
+
+        try:
+            process = await asyncio.subprocess.create_subprocess_exec(
+                "docker", "create", self._image,
+                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+            )
+        except OSError as e:
+            raise DockerError(f"Could not seed persistent volumes for '{self._name}': {e}")
+        stdout, stderr = await process.communicate()
+        if process.returncode != 0:
+            raise DockerError(
+                f"Could not create a seeding container for image '{self._image}': "
+                f"{stderr.decode(errors='replace').strip()}"
+            )
+        return stdout.decode().strip()
+
+    async def _seed_volume_from_container(self, seed_cid, volume, host_dir):
+        """
+        Copy one volume's original content from the seeding container to its
+        host directory with ``docker cp -a`` (preserves modes/ownership; no
+        dependency on tools inside the image).
+        """
+
+        try:
+            process = await asyncio.subprocess.create_subprocess_exec(
+                "docker", "cp", "-a", f"{seed_cid}:{volume}/.", host_dir + "/",
+                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+            )
+        except OSError as e:
+            raise DockerError(f"Could not seed persistent volume '{volume}' for '{self._name}': {e}")
+        _, stderr = await process.communicate()
+        if process.returncode != 0:
+            # A path the image does not contain (e.g. XRd's /xr-storage-shadow)
+            # is not an error: the volume starts empty. Same tolerance as
+            # init.sh's first copy (cp -a ... 2>/dev/null).
+            log.info(
+                "Persistent volume '%s' on '%s' not seedable from image '%s' (%s); starting empty",
+                volume, self._name, self._image, stderr.decode(errors="replace").strip(),
+            )
+            return
+        log.info("Seeded persistent volume '%s' for '%s' from image '%s'", volume, self._name, self._image)
+
+    async def _remove_seed_container(self, seed_cid):
+        """
+        Best-effort removal of the seeding container.
+        """
+
+        try:
+            process = await asyncio.subprocess.create_subprocess_exec(
+                "docker", "rm", "-f", seed_cid,
+                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
+            )
+        except OSError:
+            return
+        await process.communicate()
 
     def _prepare_init_and_interface_env(self, params):
         """
@@ -209,23 +330,22 @@ class VendorDockerVM(DockerVM):
     async def start(self):
         await super().start()
         if self.status == "started" and not self._gns3_init:
-            await self._setup_skip_init_volumes()
-            # Fix host-side ownership of the seeded volume right away so the
-            # controller can read project files while the node runs. Reset the
-            # "fixed" flag afterwards: files written by the container during
-            # runtime still need the stop-time pass.
+            # Persistent volumes are seeded and bound directly at create time
+            # (see _prepare_volumes / _mount_binds), so there is no post-start
+            # bridge to run. Fix host-side ownership right away so the
+            # controller can read project files while the node runs, and reset
+            # the "fixed" flag: files written by the container during runtime
+            # still need the stop-time pass.
             await self._fix_permissions()
             self._permissions_fixed = False
 
     async def _fix_permissions(self):
         """
         Container-side override of DockerVM._fix_permissions for vendor NOS
-        containers. It targets the Docker bind-mount paths
-        (`/gns3volumes<volume>`) directly instead of the in-container paths:
-        the in-container paths only resolve to persistent storage while the
-        `mount --bind` bridge from _setup_skip_init_volumes is up, and after a
-        container restart the bridge is gone — the base implementation would
-        then chown the overlay copy instead of the host files.
+        containers. The persistent volumes are Docker bind mounts created
+        with the container (see _mount_binds), so the in-container paths
+        resolve to the host-side files for the container's whole lifetime —
+        no /gns3volumes aliasing is needed.
 
         The busybox script runs inside the container as root (a host-side
         GNS3 process may be unprivileged and cannot chown root-owned files).
@@ -248,7 +368,7 @@ class VendorDockerVM(DockerVM):
 
         uid, gid = os.getuid(), os.getgid()
         for volume in self._volumes:
-            target = f"/gns3volumes{volume}"
+            target = volume
             log.debug("Docker container '%s' fix ownership on %s", self._name, target)
             try:
                 # chown prefers the container's own coreutils over /gns3/bin/busybox:
@@ -284,61 +404,6 @@ class VendorDockerVM(DockerVM):
             else:
                 self._permissions_fixed = True
 
-    async def _setup_skip_init_volumes(self):
-        """
-        Replicate the volume-persistence portion of init.sh (lines 35–52) for
-        containers that skip init.sh (GNS3_SKIP_INIT=1).
-
-        On first start the container's original files are seeded into the
-        persistent host directory; on subsequent starts the persisted data
-        is bind-mounted over the in-container path so writes land on the host.
-        Permission-changes recorded by _fix_permissions at the previous
-        stop are restored (best-effort).
-        """
-        for volume in self._volumes:
-            vol_target = f"/gns3volumes{volume}"
-            # fmt: off
-            script = (
-                f'mkdir -p "{volume}" && '
-                f'if [ ! -f "{vol_target}/.gns3_perms" ]; then '
-                f'  /gns3/bin/busybox cp -a "{volume}/." "{vol_target}/" 2>/dev/null; '
-                f'  /gns3/bin/busybox touch "{vol_target}/.gns3_perms"; '
-                f'fi && '
-                f'/gns3/bin/busybox mount --bind "{vol_target}" "{volume}" && '
-                f'while IFS=: read -r PERMS OWNER GROUP FILE; do '
-                f'  [ -L "$FILE" ] || /gns3/bin/busybox chmod "$PERMS" "$FILE" 2>/dev/null; '
-                # chown: prefer the container's coreutils, fall back to busybox
-                # (see _fix_permissions -- static busybox chown aborts on
-                # mismatched-glibc NOS images like XRd).
-                f'  ( command -v chown >/dev/null 2>&1 && chown -h "$OWNER:$GROUP" "$FILE" || /gns3/bin/busybox chown -h "$OWNER:$GROUP" "$FILE" ) 2>/dev/null; '
-                f'done < "{volume}/.gns3_perms"'
-            )
-            # fmt: on
-            try:
-                process = await asyncio.subprocess.create_subprocess_exec(
-                    "docker",
-                    "exec",
-                    self._cid,
-                    "sh",
-                    "-c",
-                    script,
-                    stdout=asyncio.subprocess.PIPE,
-                    stderr=asyncio.subprocess.PIPE,
-                )
-                stdout, stderr = await process.communicate()
-                if process.returncode != 0:
-                    err = stderr.decode(errors="replace").strip()
-                    log.warning(
-                        "Volume setup for '%s' on container '%s' returned %d: %s",
-                        volume, self._name, process.returncode, err,
-                    )
-                else:
-                    log.info("Volume '%s' bound to persistent storage for '%s'", volume, self._name)
-            except OSError as e:
-                log.warning(
-                    "Could not setup volume '%s' for container '%s': %s", volume, self._name, e
-                )
-
     async def _start_console_server(self):
         """
         Override: add the ``docker_exec`` console type alongside the