Dynamips VM & device deletion and ghost support.

gns3server/handlers/dynamips_vm_handler.py

@@ -67,6 +67,7 @@ class DynamipsVMHandler:
                     else:
                         setter(value)
 
+        yield from dynamips_manager.ghost_ios_support(vm)
         response.set_status(201)
         response.json(vm)
 

gns3server/handlers/project_handler.py

@@ -18,6 +18,7 @@
 from ..web.route import Route
 from ..schemas.project import PROJECT_OBJECT_SCHEMA, PROJECT_CREATE_SCHEMA, PROJECT_UPDATE_SCHEMA
 from ..modules.project_manager import ProjectManager
+from ..modules import MODULES
 
 
 class ProjectHandler:
@@ -112,6 +113,8 @@ class ProjectHandler:
         pm = ProjectManager.instance()
         project = pm.get_project(request.match_info["project_id"])
         yield from project.close()
+        for module in MODULES:
+            yield from module.instance().project_closed(project.path)
         response.set_status(204)
 
     @classmethod

gns3server/modules/base_manager.py

@@ -205,6 +205,16 @@ class BaseManager:
             vm.close()
         return vm
 
+    @asyncio.coroutine
+    def project_closed(self, project_dir):
+        """
+        Called when a project is closed.
+
+        :param project_dir: project directory
+        """
+
+        pass
+
     @asyncio.coroutine
     def delete_vm(self, vm_id):
         """

gns3server/modules/base_vm.py

@@ -15,7 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import logging
+import aiohttp
+import shutil
+import asyncio
+
+from ..utils.asyncio import wait_run_in_executor
+
 log = logging.getLogger(__name__)
 
 
@@ -107,6 +114,19 @@ class BaseVM:
                                                           name=self.name,
                                                           id=self.id))
 
+    @asyncio.coroutine
+    def delete(self):
+        """
+        Delete the VM (including all its files).
+        """
+
+        directory = self.project.vm_working_dir(self)
+        if os.path.exists(directory):
+            try:
+                yield from wait_run_in_executor(shutil.rmtree, directory)
+            except OSError as e:
+                raise aiohttp.web.HTTPInternalServerError(text="Could not delete the VM working directory: {}".format(e))
+
     def start(self):
         """
         Starts the VM process.

gns3server/modules/dynamips/__init__.py

@@ -26,11 +26,14 @@ import shutil
 import socket
 import time
 import asyncio
+import tempfile
+import glob
 import logging
 
 log = logging.getLogger(__name__)
 
 from gns3server.utils.interfaces import get_windows_interfaces
+from gns3server.utils.asyncio import wait_run_in_executor
 from pkg_resources import parse_version
 from uuid import UUID, uuid4
 from ..base_manager import BaseManager
@@ -63,11 +66,9 @@ class Dynamips(BaseManager):
 
         super().__init__()
         self._devices = {}
+        self._ghost_files = set()
         self._dynamips_path = None
 
-        # FIXME: temporary
-        self._working_dir = "/tmp"
-
     @asyncio.coroutine
     def unload(self):
 
@@ -86,18 +87,43 @@ class Dynamips(BaseManager):
                     log.error("Could not stop device hypervisor {}".format(e), exc_info=1)
                     continue
 
-#         files = glob.glob(os.path.join(self._working_dir, "dynamips", "*.ghost"))
+    @asyncio.coroutine
-#         files += glob.glob(os.path.join(self._working_dir, "dynamips", "*_lock"))
+    def project_closed(self, project_dir):
-#         files += glob.glob(os.path.join(self._working_dir, "dynamips", "ilt_*"))
+        """
-#         files += glob.glob(os.path.join(self._working_dir, "dynamips", "c[0-9][0-9][0-9][0-9]_*_rommon_vars"))
+        Called when a project is closed.
-#         files += glob.glob(os.path.join(self._working_dir, "dynamips", "c[0-9][0-9][0-9][0-9]_*_ssa"))
+
-#         for file in files:
+        :param project_dir: project directory
-#             try:
+        """
-#                 log.debug("deleting file {}".format(file))
+
-#                 os.remove(file)
+        # delete the Dynamips devices
-#             except OSError as e:
+        tasks = []
-#                 log.warn("could not delete file {}: {}".format(file, e))
+        for device in self._devices.values():
-#                 continue
+            tasks.append(asyncio.async(device.delete()))
+
+        if tasks:
+            done, _ = yield from asyncio.wait(tasks)
+            for future in done:
+                try:
+                    future.result()
+                except Exception as e:
+                    log.error("Could not delete device {}".format(e), exc_info=1)
+
+        # delete useless files
+        project_dir = os.path.join(project_dir, 'project-files', self.module_name.lower())
+        files = glob.glob(os.path.join(project_dir, "*.ghost"))
+        files += glob.glob(os.path.join(project_dir, "*_lock"))
+        files += glob.glob(os.path.join(project_dir, "ilt_*"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_*_rommon_vars"))
+        files += glob.glob(os.path.join(project_dir, "c[0-9][0-9][0-9][0-9]_*_ssa"))
+        for file in files:
+            try:
+                log.debug("Deleting file {}".format(file))
+                if file in self._ghost_files:
+                    self._ghost_files.remove(file)
+                yield from wait_run_in_executor(os.remove, file)
+            except OSError as e:
+                log.warn("Could not delete file {}: {}".format(file, e))
+                continue
 
     @property
     def dynamips_path(self):
@@ -126,7 +152,6 @@ class Dynamips(BaseManager):
         device = self._DEVICE_CLASS(name, device_id, project, self, device_type, *args, **kwargs)
         yield from device.create()
         self._devices[device.id] = device
-        project.add_device(device)
         return device
 
     def get_device(self, device_id, project_id=None):
@@ -170,7 +195,6 @@ class Dynamips(BaseManager):
 
         device = self.get_device(device_id)
         yield from device.delete()
-        device.project.remove_device(device)
         del self._devices[device.id]
         return device
 
@@ -220,16 +244,21 @@ class Dynamips(BaseManager):
             log.info("Dynamips server ready after {:.4f} seconds".format(time.time() - begin))
 
     @asyncio.coroutine
-    def start_new_hypervisor(self):
+    def start_new_hypervisor(self, working_dir=None):
         """
         Creates a new Dynamips process and start it.
 
+        :param working_dir: working directory
+
         :returns: the new hypervisor instance
         """
 
         if not self._dynamips_path:
             self.find_dynamips()
 
+        if not working_dir:
+            working_dir = tempfile.gettempdir()
+
         try:
             # let the OS find an unused port for the Dynamips hypervisor
             with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
@@ -238,9 +267,9 @@ class Dynamips(BaseManager):
         except OSError as e:
             raise DynamipsError("Could not find free port for the Dynamips hypervisor: {}".format(e))
 
-        hypervisor = Hypervisor(self._dynamips_path, self._working_dir, "127.0.0.1", port)
+        hypervisor = Hypervisor(self._dynamips_path, working_dir, "127.0.0.1", port)
 
-        log.info("Ceating new hypervisor {}:{} with working directory {}".format(hypervisor.host, hypervisor.port, self._working_dir))
+        log.info("Ceating new hypervisor {}:{} with working directory {}".format(hypervisor.host, hypervisor.port, working_dir))
         yield from hypervisor.start()
 
         yield from self._wait_for_hypervisor("127.0.0.1", port)
@@ -252,6 +281,13 @@ class Dynamips(BaseManager):
 
         return hypervisor
 
+    @asyncio.coroutine
+    def ghost_ios_support(self, vm):
+
+        ghost_ios_support = self.config.get_section_config("Dynamips").get("ghost_ios_support", True)
+        if ghost_ios_support:
+            yield from self._set_ghost_ios(vm)
+
     @asyncio.coroutine
     def create_nio(self, node, nio_settings):
         """
@@ -317,45 +353,44 @@ class Dynamips(BaseManager):
         yield from nio.create()
         return nio
 
-#     def set_ghost_ios(self, router):
+    @asyncio.coroutine
-#         """
+    def _set_ghost_ios(self, vm):
-#         Manages Ghost IOS support.
+        """
-#
+        Manages Ghost IOS support.
-#         :param router: Router instance
+
-#         """
+        :param vm: VM instance
-#
+        """
-#         if not router.mmap:
+
-#             raise DynamipsError("mmap support is required to enable ghost IOS support")
+        if not vm.mmap:
-#
+            raise DynamipsError("mmap support is required to enable ghost IOS support")
-#         ghost_instance = router.formatted_ghost_file()
+
-#         all_ghosts = []
+        ghost_file = vm.formatted_ghost_file()
-#
+        ghost_file_path = os.path.join(vm.hypervisor.working_dir, ghost_file)
-#         # search of an existing ghost instance across all hypervisors
+        if ghost_file_path not in self._ghost_files:
-#         for hypervisor in self._hypervisor_manager.hypervisors:
+            # create a new ghost IOS instance
-#             all_ghosts.extend(hypervisor.ghosts)
+            ghost_id = str(uuid4())
-#
+            ghost = Router("ghost-" + ghost_file, ghost_id, vm.project, vm.manager, platform=vm.platform, hypervisor=vm.hypervisor, ghost_flag=True)
-#         if ghost_instance not in all_ghosts:
+            yield from ghost.create()
-#             # create a new ghost IOS instance
+            yield from ghost.set_image(vm.image)
-#             ghost = Router(router.hypervisor, "ghost-" + ghost_instance, router.platform, ghost_flag=True)
+            # for 7200s, the NPE must be set when using an NPE-G2.
-#             ghost.image = router.image
+            if vm.platform == "c7200":
-#             # for 7200s, the NPE must be set when using an NPE-G2.
+                yield from ghost.set_npe(vm.npe)
-#             if router.platform == "c7200":
+            yield from ghost.set_ghost_status(1)
-#                 ghost.npe = router.npe
+            yield from ghost.set_ghost_file(ghost_file)
-#             ghost.ghost_status = 1
+            yield from ghost.set_ram(vm.ram)
-#             ghost.ghost_file = ghost_instance
+            try:
-#             ghost.ram = router.ram
+                yield from ghost.start()
-#             try:
+                yield from ghost.stop()
-#                 ghost.start()
+                self._ghost_files.add(ghost_file_path)
-#                 ghost.stop()
+            except DynamipsError:
-#             except DynamipsError:
+                raise
-#                 raise
+            finally:
-#             finally:
+                yield from ghost.clean_delete()
-#                 ghost.clean_delete()
+
-#
+        if vm.ghost_file != ghost_file:
-#         if router.ghost_file != ghost_instance:
+            # set the ghost file to the router
-#             # set the ghost file to the router
+            yield from vm.set_ghost_status(2)
-#             router.ghost_status = 2
+            yield from vm.set_ghost_file(ghost_file)
-#             router.ghost_file = ghost_instance
 #
 #     def create_config_from_file(self, local_base_config, router, destination_config_path):
 #         """

gns3server/modules/dynamips/dynamips_hypervisor.py

@@ -20,7 +20,6 @@ Interface for Dynamips hypervisor management module ("hypervisor")
 http://github.com/GNS3/dynamips/blob/master/README.hypervisor#L46
 """
 
-import socket
 import re
 import logging
 import asyncio
@@ -53,15 +52,7 @@ class DynamipsHypervisor:
         self._port = port
 
         self._devices = []
-        self._ghosts = {}
-        self._jitsharing_groups = {}
         self._working_dir = working_dir
-        # self._console_start_port_range = 2001
-        # self._console_end_port_range = 2500
-        # self._aux_start_port_range = 2501
-        # self._aux_end_port_range = 3000
-        # self._udp_start_port_range = 10001
-        # self._udp_end_port_range = 20000
         self._nio_udp_auto_instances = {}
         self._version = "N/A"
         self._timeout = timeout
@@ -193,26 +184,6 @@ class DynamipsHypervisor:
 
         return self._devices
 
-    @property
-    def ghosts(self):
-        """
-        Returns a list of the ghosts hosted by this hypervisor.
-
-        :returns: Ghosts dict (image_name -> device)
-        """
-
-        return self._ghosts
-
-    def add_ghost(self, image_name, router):
-        """
-        Adds a ghost name to the list of ghosts created on this hypervisor.
-
-        :param image_name: name of the ghost image
-        :param router: Router instance
-        """
-
-        self._ghosts[image_name] = router
-
     @property
     def port(self):
         """

gns3server/modules/dynamips/nodes/atm_switch.py

@@ -58,7 +58,8 @@ class ATMSwitch(Device):
     def create(self):
 
         if self._hypervisor is None:
-            self._hypervisor = yield from self.manager.start_new_hypervisor()
+            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
+            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
 
         yield from self._hypervisor.send('atmsw create "{}"'.format(self._name))
         log.info('ATM switch "{name}" [{id}] has been created'.format(name=self._name, id=self._id))

gns3server/modules/dynamips/nodes/bridge.py

@@ -44,7 +44,8 @@ class Bridge(Device):
     def create(self):
 
         if self._hypervisor is None:
-            self._hypervisor = yield from self.manager.start_new_hypervisor()
+            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
+            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
 
         yield from self._hypervisor.send('nio_bridge create "{}"'.format(self._name))
         self._hypervisor.devices.append(self)

gns3server/modules/dynamips/nodes/ethernet_switch.py

@@ -66,7 +66,8 @@ class EthernetSwitch(Device):
     def create(self):
 
         if self._hypervisor is None:
-            self._hypervisor = yield from self.manager.start_new_hypervisor()
+            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
+            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
 
         yield from self._hypervisor.send('ethsw create "{}"'.format(self._name))
         log.info('Ethernet switch "{name}" [{id}] has been created'.format(name=self._name, id=self._id))

gns3server/modules/dynamips/nodes/frame_relay_switch.py

@@ -57,7 +57,8 @@ class FrameRelaySwitch(Device):
     def create(self):
 
         if self._hypervisor is None:
-            self._hypervisor = yield from self.manager.start_new_hypervisor()
+            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
+            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
 
         yield from self._hypervisor.send('frsw create "{}"'.format(self._name))
         log.info('Frame Relay switch "{name}" [{id}] has been created'.format(name=self._name, id=self._id))

gns3server/modules/dynamips/nodes/router.py

@@ -20,17 +20,19 @@ Interface for Dynamips virtual Machine module ("vm")
 http://github.com/GNS3/dynamips/blob/master/README.hypervisor#L77
 """
 
-from ...base_vm import BaseVM
-from ..dynamips_error import DynamipsError
-
 import asyncio
 import time
 import sys
 import os
+import glob
 
 import logging
 log = logging.getLogger(__name__)
 
+from ...base_vm import BaseVM
+from ..dynamips_error import DynamipsError
+from gns3server.utils.asyncio import wait_run_in_executor
+
 
 class Router(BaseVM):
 
@@ -51,11 +53,11 @@ class Router(BaseVM):
                2: "running",
                3: "suspended"}
 
-    def __init__(self, name, vm_id, project, manager, dynamips_id=None, platform="c7200", ghost_flag=False):
+    def __init__(self, name, vm_id, project, manager, dynamips_id=None, platform="c7200", hypervisor=None, ghost_flag=False):
 
         super().__init__(name, vm_id, project, manager)
 
-        self._hypervisor = None
+        self._hypervisor = hypervisor
         self._dynamips_id = dynamips_id
         self._closed = False
         self._name = name
@@ -113,7 +115,7 @@ class Router(BaseVM):
             else:
                 self._aux = self._manager.port_manager.get_free_console_port()
         else:
-            log.info("creating a new ghost IOS file")
+            log.info("Creating a new ghost IOS instance")
             self._dynamips_id = 0
             self._name = "Ghost"
 
@@ -166,10 +168,22 @@ class Router(BaseVM):
 
         cls._dynamips_ids.clear()
 
+    @property
+    def dynamips_id(self):
+        """
+        Returns the Dynamips VM ID.
+
+        :return: Dynamips VM identifier
+        """
+
+        return self._dynamips_id
+
     @asyncio.coroutine
     def create(self):
 
-        self._hypervisor = yield from self.manager.start_new_hypervisor()
+        if not self._hypervisor:
+            module_workdir = self.project.module_working_directory(self.manager.module_name.lower())
+            self._hypervisor = yield from self.manager.start_new_hypervisor(working_dir=module_workdir)
 
         yield from self._hypervisor.send('vm create "{name}" {id} {platform}'.format(name=self._name,
                                                                                      id=self._dynamips_id,
@@ -300,6 +314,7 @@ class Router(BaseVM):
                 yield from self.stop()
             except DynamipsError:
                 pass
+            yield from self._hypervisor.send('vm delete "{}"'.format(self._name))
             yield from self.hypervisor.stop()
 
         if self._console:
@@ -315,17 +330,6 @@ class Router(BaseVM):
 
         self._closed = True
 
-    @asyncio.coroutine
-    def delete(self):
-        """
-        Deletes this router.
-        """
-
-        yield from self.close()
-        yield from self._hypervisor.send('vm delete "{}"'.format(self._name))
-        self._hypervisor.devices.remove(self)
-        log.info('Router "{name}" [{id}] has been deleted'.format(name=self._name, id=self._id))
-
     @property
     def platform(self):
         """
@@ -398,7 +402,6 @@ class Router(BaseVM):
         :param image: path to IOS image file
         """
 
-        # encase image in quotes to protect spaces in the path
         yield from self._hypervisor.send('vm set_ios "{name}" "{image}"'.format(name=self._name, image=image))
 
         log.info('Router "{name}" [{id}]: has a new IOS image set: "{image}"'.format(name=self._name,
@@ -707,10 +710,6 @@ class Router(BaseVM):
 
         self._ghost_file = ghost_file
 
-        # this is a ghost instance, track this as a hosted ghost instance by this hypervisor
-        if self.ghost_status == 1:
-            self._hypervisor.add_ghost(ghost_file, self)
-
     def formatted_ghost_file(self):
         """
         Returns a properly formatted ghost file name.
@@ -1548,14 +1547,31 @@ class Router(BaseVM):
     #             except OSError as e:
     #                 raise DynamipsError("Could not save the private configuration {}: {}".format(config_path, e))
 
-    # def clean_delete(self):
+    def delete(self):
-    #     """
+        """
-    #     Deletes this router & associated files (nvram, disks etc.)
+        Delete the VM (including all its files).
-    #     """
+        """
-    #
+
-    #     self._hypervisor.send("vm clean_delete {}".format(self._name))
+        # delete the VM files
-    #     self._hypervisor.devices.remove(self)
+        project_dir = os.path.join(self.project.module_working_directory(self.manager.module_name.lower()))
-    #
+        files = glob.glob(os.path.join(project_dir, "{}_i{}*".format(self._platform, self._dynamips_id)))
+        for file in files:
+            try:
+                log.debug("Deleting file {}".format(file))
+                yield from wait_run_in_executor(os.remove, file)
+            except OSError as e:
+                log.warn("Could not delete file {}: {}".format(file, e))
+                continue
+
+    @asyncio.coroutine
+    def clean_delete(self, stop_hypervisor=False):
+        """
+        Deletes this router & associated files (nvram, disks etc.)
+        """
+
+        yield from self._hypervisor.send('vm clean_delete "{}"'.format(self._name))
+        self._hypervisor.devices.remove(self)
+
         # if self._startup_config:
         #     # delete the startup-config
         #     startup_config_path = os.path.join(self.hypervisor.working_dir, "configs", "{}.cfg".format(self.name))
@@ -1567,5 +1583,5 @@ class Router(BaseVM):
         #     private_config_path = os.path.join(self.hypervisor.working_dir, "configs", "{}-private.cfg".format(self.name))
         #     if os.path.isfile(private_config_path):
         #         os.remove(private_config_path)
-    #
+
-    #     log.info("router {name} [id={id}] has been deleted (including associated files)".format(name=self._name, id=self._id))
+        log.info('Router "{name}" [{id}] has been deleted (including associated files)'.format(name=self._name, id=self._id))

gns3server/modules/project.py

@@ -19,8 +19,8 @@ import aiohttp
 import os
 import shutil
 import asyncio
-from uuid import UUID, uuid4
 
+from uuid import UUID, uuid4
 from ..config import Config
 from ..utils.asyncio import wait_run_in_executor
 
@@ -59,8 +59,6 @@ class Project:
 
         self._vms = set()
         self._vms_to_destroy = set()
-        self._devices = set()
-
         self.temporary = temporary
 
         if path is None:
@@ -73,6 +71,15 @@ class Project:
 
         log.debug("Create project {id} in directory {path}".format(path=self._path, id=self._id))
 
+    def __json__(self):
+
+        return {
+            "project_id": self._id,
+            "location": self._location,
+            "temporary": self._temporary,
+            "path": self._path,
+        }
+
     def _config(self):
 
         return Config.instance().get_section_config("Server")
@@ -129,11 +136,6 @@ class Project:
 
         return self._vms
 
-    @property
-    def devices(self):
-
-        return self._devices
-
     @property
     def temporary(self):
 
@@ -167,13 +169,29 @@ class Project:
             if os.path.exists(os.path.join(self._path, ".gns3_temporary")):
                 os.remove(os.path.join(self._path, ".gns3_temporary"))
 
+    def module_working_directory(self, module_name):
+        """
+        Return a working directory for the module
+        If the directory doesn't exist, the directory is created.
+
+        :param module_name: name for the module
+        :returns: working directory
+        """
+
+        workdir = os.path.join(self._path, 'project-files', module_name)
+        try:
+            os.makedirs(workdir, exist_ok=True)
+        except OSError as e:
+            raise aiohttp.web.HTTPInternalServerError(text="Could not create module working directory: {}".format(e))
+        return workdir
+
     def vm_working_directory(self, vm):
         """
         Return a working directory for a specific VM.
         If the directory doesn't exist, the directory is created.
 
-        :param vm: An instance of VM
+        :param vm: VM instance
-        :returns: A string with a VM working directory
+        :returns: VM working directory
         """
 
         workdir = os.path.join(self._path, 'project-files', vm.manager.module_name.lower(), vm.id)
@@ -205,15 +223,6 @@ class Project:
         self.remove_vm(vm)
         self._vms_to_destroy.add(vm)
 
-    def __json__(self):
-
-        return {
-            "project_id": self._id,
-            "location": self._location,
-            "temporary": self._temporary,
-            "path": self._path,
-        }
-
     def add_vm(self, vm):
         """
         Add a VM to the project.
@@ -235,27 +244,6 @@ class Project:
         if vm in self._vms:
             self._vms.remove(vm)
 
-    def add_device(self, device):
-        """
-        Add a device to the project.
-        In theory this should be called by the VM manager.
-
-        :param device: Device instance
-        """
-
-        self._devices.add(device)
-
-    def remove_device(self, device):
-        """
-        Remove a device from the project.
-        In theory this should be called by the VM manager.
-
-        :param device: Device instance
-        """
-
-        if device in self._devices:
-            self._devices.remove(device)
-
     @asyncio.coroutine
     def close(self):
         """Close the project, but keep information on disk"""
@@ -277,9 +265,6 @@ class Project:
             else:
                 vm.close()
 
-        for device in self._devices:
-            tasks.append(asyncio.async(device.delete()))
-
         if tasks:
             done, _ = yield from asyncio.wait(tasks)
             for future in done:
@@ -300,12 +285,7 @@ class Project:
 
         while self._vms_to_destroy:
             vm = self._vms_to_destroy.pop()
-            directory = self.vm_working_directory(vm)
+            yield from vm.delete()
-            if os.path.exists(directory):
-                try:
-                    yield from wait_run_in_executor(shutil.rmtree, directory)
-                except OSError as e:
-                    raise aiohttp.web.HTTPInternalServerError(text="Could not delete the project directory: {}".format(e))
             self.remove_vm(vm)
 
     @asyncio.coroutine