Change directory layout for dynamips.

This create a directory by node to allow feature like save as
to work when running on multiple compute. Benefits now dynamips work
like other nodes. Old topologies are migrated.

Ref https://github.com/GNS3/gns3-gui/issues/1761

gns3server/compute/dynamips/__init__.py

@@ -219,10 +219,12 @@ class Dynamips(BaseManager):
         project_dir = project.module_working_path(self.module_name.lower())
 
         files = glob.glob(os.path.join(glob.escape(project_dir), "*.ghost"))
-        files += glob.glob(os.path.join(glob.escape(project_dir), "*_lock"))
+        files += glob.glob(os.path.join(glob.escape(project_dir), "*", "*_lock"))
+        files += glob.glob(os.path.join(glob.escape(project_dir), "*_log.txt"))
+        files += glob.glob(os.path.join(glob.escape(project_dir), "*_stdout.txt"))
         files += glob.glob(os.path.join(glob.escape(project_dir), "ilt_*"))
-        files += glob.glob(os.path.join(glob.escape(project_dir), "c[0-9][0-9][0-9][0-9]_i[0-9]*_rommon_vars"))
-        files += glob.glob(os.path.join(glob.escape(project_dir), "c[0-9][0-9][0-9][0-9]_i[0-9]*_log.txt"))
+        files += glob.glob(os.path.join(glob.escape(project_dir), "*", "c[0-9][0-9][0-9][0-9]_i[0-9]*_rommon_vars"))
+        files += glob.glob(os.path.join(glob.escape(project_dir), "*", "c[0-9][0-9][0-9][0-9]_i[0-9]*_log.txt"))
         for file in files:
             try:
                 log.debug("Deleting file {}".format(file))
@@ -409,7 +411,9 @@ class Dynamips(BaseManager):
             return
 
         ghost_file = vm.formatted_ghost_file()
-        ghost_file_path = os.path.join(vm.hypervisor.working_dir, ghost_file)
+
+        module_workdir = vm.project.module_working_directory(self.module_name.lower())
+        ghost_file_path = os.path.join(module_workdir, ghost_file)
         if ghost_file_path not in self._ghost_files:
             # create a new ghost IOS instance
             ghost_id = str(uuid4())
@@ -418,7 +422,7 @@ class Dynamips(BaseManager):
                 yield from ghost.create()
                 yield from ghost.set_image(vm.image)
                 yield from ghost.set_ghost_status(1)
-                yield from ghost.set_ghost_file(ghost_file)
+                yield from ghost.set_ghost_file(ghost_file_path)
                 yield from ghost.set_ram(vm.ram)
                 try:
                     yield from ghost.start()
@@ -434,7 +438,7 @@ class Dynamips(BaseManager):
         if vm.ghost_file != ghost_file and os.path.isfile(ghost_file_path):
             # set the ghost file to the router
             yield from vm.set_ghost_status(2)
-            yield from vm.set_ghost_file(ghost_file)
+            yield from vm.set_ghost_file(ghost_file_path)
 
     @asyncio.coroutine
     def update_vm_settings(self, vm, settings):
@@ -508,8 +512,8 @@ class Dynamips(BaseManager):
         """
 
         module_workdir = vm.project.module_working_directory(self.module_name.lower())
-        default_startup_config_path = os.path.join(module_workdir, "configs", "i{}_startup-config.cfg".format(vm.dynamips_id))
-        default_private_config_path = os.path.join(module_workdir, "configs", "i{}_private-config.cfg".format(vm.dynamips_id))
+        default_startup_config_path = os.path.join(module_workdir, vm.id, "configs", "i{}_startup-config.cfg".format(vm.dynamips_id))
+        default_private_config_path = os.path.join(module_workdir, vm.id, "configs", "i{}_private-config.cfg".format(vm.dynamips_id))
 
         startup_config_path = settings.get("startup_config")
         startup_config_content = settings.get("startup_config_content")

gns3server/compute/dynamips/nodes/router.py

@@ -25,10 +25,12 @@ import time
 import sys
 import os
 import glob
+import shlex
 import base64
+import shutil
 import binascii
-
 import logging
+
 log = logging.getLogger(__name__)
 
 from ...base_node import BaseNode
@@ -113,11 +115,14 @@ class Router(BaseNode):
             self._dynamips_id = 0
             self._name = "Ghost"
 
+        self._working_directory = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()), self.id)
+        os.makedirs(os.path.join(self._working_directory, "configs"), exist_ok=True)
+
     def __json__(self):
 
         router_info = {"name": self.name,
                        "node_id": self.id,
-                       "node_directory": os.path.join(self.project.module_working_directory(self.manager.module_name.lower())),
+                       "node_directory": os.path.join(self._working_directory),
                        "project_id": self.project.id,
                        "dynamips_id": self._dynamips_id,
                        "platform": self._platform,
@@ -187,8 +192,10 @@ class Router(BaseNode):
     def create(self):
 
         if not self._hypervisor:
-            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
-            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
+            # We start the hypervisor is the dynamips folder and next we change to node dir
+            # this allow the creation of common files in the dynamips folder
+            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=self.project.module_working_directory(self.manager.module_name.lower()))
+            yield from self._hypervisor.set_working_dir(self._working_directory)
 
         yield from self._hypervisor.send('vm create "{name}" {id} {platform}'.format(name=self._name,
                                                                                      id=self._dynamips_id,
@@ -365,14 +372,13 @@ class Router(BaseNode):
 
         if self._auto_delete_disks:
             # delete nvram and disk files
-            project_dir = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()))
-            files = glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_disk[0-1]".format(self.platform, self.dynamips_id)))
-            files += glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_slot[0-1]".format(self.platform, self.dynamips_id)))
-            files += glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_nvram".format(self.platform, self.dynamips_id)))
-            files += glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_flash[0-1]".format(self.platform, self.dynamips_id)))
-            files += glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_rom".format(self.platform, self.dynamips_id)))
-            files += glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_bootflash".format(self.platform, self.dynamips_id)))
-            files += glob.glob(os.path.join(glob.escape(project_dir), "{}_i{}_ssa".format(self.platform, self.dynamips_id)))
+            files = glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_disk[0-1]".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_slot[0-1]".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_nvram".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_flash[0-1]".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_rom".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_bootflash".format(self.platform, self.dynamips_id)))
+            files += glob.glob(os.path.join(glob.escape(self._working_directory), "{}_i{}_ssa".format(self.platform, self.dynamips_id)))
             for file in files:
                 try:
                     log.debug("Deleting file {}".format(file))
@@ -760,7 +766,7 @@ class Router(BaseNode):
         """
 
         yield from self._hypervisor.send('vm set_ghost_file "{name}" {ghost_file}'.format(name=self._name,
-                                                                                          ghost_file=ghost_file))
+                                                                                          ghost_file=shlex.quote(ghost_file)))
 
         log.info('Router "{name}" [{id}]: ghost file set to {ghost_file}'.format(name=self._name,
                                                                                  id=self._id,
@@ -1453,10 +1459,9 @@ class Router(BaseNode):
         :param new_name: new name string
         """
 
-        module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
         if self._startup_config:
             # change the hostname in the startup-config
-            startup_config_path = os.path.join(module_workdir, "configs", "i{}_startup-config.cfg".format(self._dynamips_id))
+            startup_config_path = os.path.join(self._working_directory, "configs", "i{}_startup-config.cfg".format(self._dynamips_id))
             if os.path.isfile(startup_config_path):
                 try:
                     with open(startup_config_path, "r+", encoding="utf-8", errors="replace") as f:
@@ -1470,7 +1475,7 @@ class Router(BaseNode):
 
         if self._private_config:
             # change the hostname in the private-config
-            private_config_path = os.path.join(module_workdir, "configs", "i{}_private-config.cfg".format(self._dynamips_id))
+            private_config_path = os.path.join(self._working_directory, "configs", "i{}_private-config.cfg".format(self._dynamips_id))
             if os.path.isfile(private_config_path):
                 try:
                     with open(private_config_path, "r+", encoding="utf-8", errors="replace") as f:
@@ -1504,9 +1509,8 @@ class Router(BaseNode):
             self._startup_config = startup_config
             self._private_config = private_config
 
-            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
             if private_config:
-                private_config_path = os.path.join(module_workdir, private_config)
+                private_config_path = os.path.join(self._working_directory, private_config)
                 try:
                     if not os.path.getsize(private_config_path):
                         # an empty private-config can prevent a router to boot.
@@ -1519,7 +1523,7 @@ class Router(BaseNode):
                     raise DynamipsError("Cannot access the private-config {}: {}".format(private_config_path, e))
 
             try:
-                startup_config_path = os.path.join(module_workdir, startup_config)
+                startup_config_path = os.path.join(self._working_directory, startup_config)
                 with open(startup_config_path) as f:
                     self._startup_config_content = f.read()
             except OSError as e:
@@ -1565,9 +1569,8 @@ class Router(BaseNode):
 
         if self.startup_config or self.private_config:
 
-            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
             try:
-                config_path = os.path.join(module_workdir, "configs")
+                config_path = os.path.join(self._working_directory, "configs")
                 os.makedirs(config_path, exist_ok=True)
             except OSError as e:
                 raise DynamipsError("Could could not create configuration directory {}: {}".format(config_path, e))
@@ -1579,7 +1582,7 @@ class Router(BaseNode):
                 try:
                     config = base64.b64decode(startup_config_base64).decode("utf-8", errors="replace")
                     config = "!\n" + config.replace("\r", "")
-                    config_path = os.path.join(module_workdir, self.startup_config)
+                    config_path = os.path.join(self._working_directory, self.startup_config)
                     with open(config_path, "wb") as f:
                         log.info("saving startup-config to {}".format(self.startup_config))
                         self._startup_config_content = config
@@ -1592,7 +1595,7 @@ class Router(BaseNode):
                     self._private_config = os.path.join("configs", "i{}_private-config.cfg".format(self._dynamips_id))
                 try:
                     config = base64.b64decode(private_config_base64).decode("utf-8", errors="replace")
-                    config_path = os.path.join(module_workdir, self.private_config)
+                    config_path = os.path.join(self._working_directory, self.private_config)
                     with open(config_path, "wb") as f:
                         log.info("saving private-config to {}".format(self.private_config))
                         self._private_config_content = config
@@ -1604,31 +1607,10 @@ class Router(BaseNode):
         """
         Delete this VM (including all its files).
         """
-
-        # delete the VM files
-        project_dir = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()))
-        files = glob.glob(os.path.join(project_dir, "{}_i{}*".format(self._platform, self._dynamips_id)))
-
-        module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
-        # delete the startup-config
-        if self._startup_config:
-            startup_config_path = os.path.join(module_workdir, "configs", "i{}_startup-config.cfg".format(self._dynamips_id))
-            if os.path.isfile(startup_config_path):
-                files.append(startup_config_path)
-
-        # delete the private-config
-        if self._private_config:
-            private_config_path = os.path.join(module_workdir, "configs", "i{}_private-config.cfg".format(self._dynamips_id))
-            if os.path.isfile(private_config_path):
-                files.append(private_config_path)
-
-        for file in files:
-            try:
-                log.debug("Deleting file {}".format(file))
-                yield from wait_run_in_executor(os.remove, file)
-            except OSError as e:
-                log.warn("Could not delete file {}: {}".format(file, e))
-                continue
+        try:
+            yield from wait_run_in_executor(shutil.rmtree, self._working_directory)
+        except OSError as e:
+            log.warn("Could not delete file {}".format(e))
 
         self.manager.release_dynamips_id(self._project.id, self._dynamips_id)
 
@@ -1640,11 +1622,14 @@ class Router(BaseNode):
 
         yield from self._hypervisor.send('vm clean_delete "{}"'.format(self._name))
         self._hypervisor.devices.remove(self)
+        try:
+            yield from wait_run_in_executor(shutil.rmtree, self._working_directory)
+        except OSError as e:
+            log.warn("Could not delete file {}".format(e))
         log.info('Router "{name}" [{id}] has been deleted (including associated files)'.format(name=self._name, id=self._id))
 
     def _memory_files(self):
-        project_dir = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()))
         return [
-            os.path.join(project_dir, "{}_i{}_rom".format(self.platform, self.dynamips_id)),
-            os.path.join(project_dir, "{}_i{}_nvram".format(self.platform, self.dynamips_id))
+            os.path.join(self._working_directory, "{}_i{}_rom".format(self.platform, self.dynamips_id)),
+            os.path.join(self._working_directory, "{}_i{}_nvram".format(self.platform, self.dynamips_id))
         ]

gns3server/controller/topology.py

@@ -19,6 +19,7 @@ import os
 import json
 import copy
 import uuid
+import glob
 import shutil
 import zipfile
 import aiohttp
@@ -36,7 +37,7 @@ import logging
 log = logging.getLogger(__name__)
 
 
-GNS3_FILE_FORMAT_REVISION = 6
+GNS3_FILE_FORMAT_REVISION = 7
 
 
 def _check_topology_schema(topo):
@@ -131,12 +132,45 @@ def load_topology(path):
         with open(path, "w+", encoding="utf-8") as f:
             json.dump(topo, f, indent=4, sort_keys=True)
 
+    # Version before GNS3 2.0 beta 3
+    if topo["revision"] < 7:
+        shutil.copy(path, path + ".backup{}".format(topo.get("revision", 0)))
+        topo = _convert_2_0_0_beta_2(topo, path)
+        _check_topology_schema(topo)
+        with open(path, "w+", encoding="utf-8") as f:
+            json.dump(topo, f, indent=4, sort_keys=True)
+
     if topo["revision"] > GNS3_FILE_FORMAT_REVISION:
         raise aiohttp.web.HTTPConflict(text="This project is designed for a more recent version of GNS3 please update GNS3 to version {} or later".format(topo["version"]))
     _check_topology_schema(topo)
     return topo
 
 
+def _convert_2_0_0_beta_2(topo, topo_path):
+    """
+    Convert topologies from GNS3 2.0.0 beta 2 to beta 3.
+
+    Changes:
+     * Node id folders for dynamips
+    """
+    topo_dir = os.path.dirname(topo_path)
+    topo["revision"] = 7
+
+    for node in topo.get("topology", {}).get("nodes", []):
+        if node["node_type"] == "dynamips":
+            node_id = node["node_id"]
+            dynamips_id = node["properties"]["dynamips_id"]
+
+            dynamips_dir = os.path.join(topo_dir, "project-files", "dynamips")
+            node_dir = os.path.join(dynamips_dir, node_id)
+            os.makedirs(os.path.join(node_dir, "configs"), exist_ok=True)
+            for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "*_i{}_*".format(dynamips_id))):
+                shutil.move(path, os.path.join(node_dir, os.path.basename(path)))
+            for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "configs", "i{}_*".format(dynamips_id))):
+                shutil.move(path, os.path.join(node_dir, "configs", os.path.basename(path)))
+    return topo
+
+
 def _convert_2_0_0_alpha(topo, topo_path):
     """
     Convert topologies from GNS3 2.0.0 alpha to 2.0.0 final.