Move endpoints to routes & preparations to use a database.

gns3server/compute/base_node.py

@@ -31,8 +31,8 @@ from gns3server.compute.compute_error import ComputeError
 from ..compute.port_manager import PortManager
 from ..utils.asyncio import wait_run_in_executor, locking
 from ..utils.asyncio.telnet_server import AsyncioTelnetServer
-from ..ubridge.hypervisor import Hypervisor
-from ..ubridge.ubridge_error import UbridgeError
+from gns3server.compute.ubridge.hypervisor import Hypervisor
+from gns3server.compute.ubridge.ubridge_error import UbridgeError
 from .nios.nio_udp import NIOUDP
 from .error import NodeError
 

gns3server/compute/builtin/nodes/cloud.py

@@ -21,7 +21,7 @@ import subprocess
 from ...error import NodeError
 from ...base_node import BaseNode
 from ...nios.nio_udp import NIOUDP
-from ....ubridge.ubridge_error import UbridgeError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError
 
 import gns3server.utils.interfaces
 import gns3server.utils.asyncio

gns3server/compute/docker/docker_vm.py

@@ -35,7 +35,7 @@ from gns3server.utils.asyncio import wait_for_file_creation
 from gns3server.utils.asyncio import monitor_process
 from gns3server.utils.get_resource import get_resource
 
-from gns3server.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError
 from ..base_node import BaseNode
 
 from ..adapters.ethernet_adapter import EthernetAdapter

gns3server/compute/iou/iou_vm.py

@@ -40,7 +40,7 @@ from ..nios.nio_udp import NIOUDP
 from ..base_node import BaseNode
 from .utils.iou_import import nvram_import
 from .utils.iou_export import nvram_export
-from gns3server.ubridge.ubridge_error import UbridgeError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError
 from gns3server.utils.file_watcher import FileWatcher
 from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer
 from gns3server.utils.asyncio import locking

gns3server/compute/notification_manager.py

@@ -17,7 +17,7 @@
 
 
 from contextlib import contextmanager
-from ..notification_queue import NotificationQueue
+from gns3server.utils.notification_queue import NotificationQueue
 
 
 class NotificationManager:

gns3server/compute/ubridge/hypervisor.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Represents a uBridge hypervisor and starts/stops the associated uBridge process.
+"""
+
+import sys
+import os
+import subprocess
+import asyncio
+import socket
+import re
+
+from gns3server.utils import parse_version
+from gns3server.utils.asyncio import wait_for_process_termination
+from gns3server.utils.asyncio import monitor_process
+from gns3server.utils.asyncio import subprocess_check_output
+from .ubridge_hypervisor import UBridgeHypervisor
+from .ubridge_error import UbridgeError
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class Hypervisor(UBridgeHypervisor):
+
+    """
+    Hypervisor.
+
+    :param project: Project instance
+    :param path: path to uBridge executable
+    :param working_dir: working directory
+    :param host: host/address for this hypervisor
+    :param port: port for this hypervisor
+    """
+
+    _instance_count = 1
+
+    def __init__(self, project, path, working_dir, host, port=None):
+
+        if port is None:
+            try:
+                port = None
+                info = socket.getaddrinfo(host, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
+                if not info:
+                    raise UbridgeError("getaddrinfo returns an empty list on {}".format(host))
+                for res in info:
+                    af, socktype, proto, _, sa = res
+                    # let the OS find an unused port for the uBridge hypervisor
+                    with socket.socket(af, socktype, proto) as sock:
+                        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+                        sock.bind(sa)
+                        port = sock.getsockname()[1]
+                        break
+            except OSError as e:
+                raise UbridgeError("Could not find free port for the uBridge hypervisor: {}".format(e))
+
+        super().__init__(host, port)
+        self._project = project
+        self._path = path
+        self._working_dir = working_dir
+        self._command = []
+        self._process = None
+        self._stdout_file = ""
+        self._started = False
+        self._version = ""
+
+    @property
+    def process(self):
+        """
+        Returns the subprocess of the Hypervisor
+
+        :returns: subprocess
+        """
+
+        return self._process
+
+    @property
+    def started(self):
+        """
+        Returns either this hypervisor has been started or not.
+
+        :returns: boolean
+        """
+
+        return self._started
+
+    @property
+    def path(self):
+        """
+        Returns the path to the uBridge executable.
+
+        :returns: path to uBridge
+        """
+
+        return self._path
+
+    @path.setter
+    def path(self, path):
+        """
+        Sets the path to the uBridge executable.
+
+        :param path: path to uBridge
+        """
+
+        self._path = path
+
+    @property
+    def version(self):
+        """
+        Returns the uBridge version.
+
+        :returns: string
+        """
+
+        return self._version
+
+    async def _check_ubridge_version(self, env=None):
+        """
+        Checks if the ubridge executable version
+        """
+        try:
+            output = await subprocess_check_output(self._path, "-v", cwd=self._working_dir, env=env)
+            match = re.search(r"ubridge version ([0-9a-z\.]+)", output)
+            if match:
+                self._version = match.group(1)
+                if sys.platform.startswith("win") or sys.platform.startswith("darwin"):
+                    minimum_required_version = "0.9.12"
+                else:
+                    # uBridge version 0.9.14 is required for packet filters 
+                    # to work for IOU nodes.
+                    minimum_required_version = "0.9.14"
+                if parse_version(self._version) < parse_version(minimum_required_version):
+                    raise UbridgeError("uBridge executable version must be >= {}".format(minimum_required_version))
+            else:
+                raise UbridgeError("Could not determine uBridge version for {}".format(self._path))
+        except (OSError, subprocess.SubprocessError) as e:
+            raise UbridgeError("Error while looking for uBridge version: {}".format(e))
+
+    async def start(self):
+        """
+        Starts the uBridge hypervisor process.
+        """
+
+        env = os.environ.copy()
+        if sys.platform.startswith("win"):
+            # add the Npcap directory to $PATH to force uBridge to use npcap DLL instead of Winpcap (if installed)
+            system_root = os.path.join(os.path.expandvars("%SystemRoot%"), "System32", "Npcap")
+            if os.path.isdir(system_root):
+                env["PATH"] = system_root + ';' + env["PATH"]
+        await self._check_ubridge_version(env)
+        try:
+            command = self._build_command()
+            log.info("starting ubridge: {}".format(command))
+            self._stdout_file = os.path.join(self._working_dir, "ubridge.log")
+            log.info("logging to {}".format(self._stdout_file))
+            with open(self._stdout_file, "w", encoding="utf-8") as fd:
+                self._process = await asyncio.create_subprocess_exec(*command,
+                                                                          stdout=fd,
+                                                                          stderr=subprocess.STDOUT,
+                                                                          cwd=self._working_dir,
+                                                                          env=env)
+
+            log.info("ubridge started PID={}".format(self._process.pid))
+            # recv: Bad address is received by uBridge when a docker image stops by itself
+            # see https://github.com/GNS3/gns3-gui/issues/2957
+            #monitor_process(self._process, self._termination_callback)
+        except (OSError, subprocess.SubprocessError) as e:
+            ubridge_stdout = self.read_stdout()
+            log.error("Could not start ubridge: {}\n{}".format(e, ubridge_stdout))
+            raise UbridgeError("Could not start ubridge: {}\n{}".format(e, ubridge_stdout))
+
+    def _termination_callback(self, returncode):
+        """
+        Called when the process has stopped.
+
+        :param returncode: Process returncode
+        """
+
+        if returncode != 0:
+            error_msg = "uBridge process has stopped, return code: {}\n{}\n".format(returncode, self.read_stdout())
+            log.error(error_msg)
+            self._project.emit("log.error", {"message": error_msg})
+        else:
+            log.info("uBridge process has stopped, return code: %d", returncode)
+
+    async def stop(self):
+        """
+        Stops the uBridge hypervisor process.
+        """
+
+        if self.is_running():
+            log.info("Stopping uBridge process PID={}".format(self._process.pid))
+            await UBridgeHypervisor.stop(self)
+            try:
+                await wait_for_process_termination(self._process, timeout=3)
+            except asyncio.TimeoutError:
+                if self._process and self._process.returncode is None:
+                    log.warning("uBridge process {} is still running... killing it".format(self._process.pid))
+                    try:
+                        self._process.kill()
+                    except ProcessLookupError:
+                        pass
+
+        if self._stdout_file and os.access(self._stdout_file, os.W_OK):
+            try:
+                os.remove(self._stdout_file)
+            except OSError as e:
+                log.warning("could not delete temporary uBridge log file: {}".format(e))
+        self._process = None
+        self._started = False
+
+    def read_stdout(self):
+        """
+        Reads the standard output of the uBridge process.
+        Only use when the process has been stopped or has crashed.
+        """
+
+        output = ""
+        if self._stdout_file and os.access(self._stdout_file, os.R_OK):
+            try:
+                with open(self._stdout_file, "rb") as file:
+                    output = file.read().decode("utf-8", errors="replace")
+            except OSError as e:
+                log.warning("could not read {}: {}".format(self._stdout_file, e))
+        return output
+
+    def is_running(self):
+        """
+        Checks if the process is running
+
+        :returns: True or False
+        """
+
+        if self._process and self._process.returncode is None:
+            return True
+        return False
+
+    def _build_command(self):
+        """
+        Command to start the uBridge hypervisor process.
+        (to be passed to subprocess.Popen())
+        """
+
+        command = [self._path]
+        command.extend(["-H", "{}:{}".format(self._host, self._port)])
+        if log.getEffectiveLevel() == logging.DEBUG:
+            command.extend(["-d", "1"])
+        return command

gns3server/compute/ubridge/ubridge_error.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Custom exceptions for the ubridge.
+"""
+
+
+class UbridgeError(Exception):
+
+    def __init__(self, message):
+        Exception.__init__(self, message)
+
+
+class UbridgeNamespaceError(Exception):
+    """
+    Raised if ubridge can not move a container to a namespace
+    """
+    pass

gns3server/compute/ubridge/ubridge_hypervisor.py

@@ -0,0 +1,274 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import time
+import logging
+import asyncio
+
+from gns3server.utils.asyncio import locking
+from .ubridge_error import UbridgeError
+
+log = logging.getLogger(__name__)
+
+
+class UBridgeHypervisor:
+
+    """
+    Creates a new connection to uBridge hypervisor.
+
+    :param host: the hostname or ip address string of the uBridge hypervisor
+    :param port: the tcp port integer
+    :param timeout: timeout integer for how long to wait for a response to commands sent to the
+    hypervisor (defaults to 30 seconds)
+    """
+
+    # Used to parse Ubridge response codes
+    error_re = re.compile(r"""^2[0-9]{2}-""")
+    success_re = re.compile(r"""^1[0-9]{2}\s{1}""")
+
+    def __init__(self, host, port, timeout=30.0):
+
+        self._host = host
+        self._port = port
+        self._version = "N/A"
+        self._timeout = timeout
+        self._reader = None
+        self._writer = None
+
+    async def connect(self, timeout=10):
+        """
+        Connects to the hypervisor.
+        """
+
+        # connect to a local address by default
+        # if listening to all addresses (IPv4 or IPv6)
+        if self._host == "0.0.0.0":
+            host = "127.0.0.1"
+        elif self._host == "::":
+            host = "::1"
+        else:
+            host = self._host
+
+        begin = time.time()
+        connection_success = False
+        last_exception = None
+        while time.time() - begin < timeout:
+            await asyncio.sleep(0.1)
+            try:
+                self._reader, self._writer = await asyncio.open_connection(host, self._port)
+            except OSError as e:
+                last_exception = e
+                continue
+            connection_success = True
+            break
+
+        if not connection_success:
+            raise UbridgeError("Couldn't connect to hypervisor on {}:{} :{}".format(host, self._port, last_exception))
+        else:
+            log.info("Connected to uBridge hypervisor on {}:{} after {:.4f} seconds".format(host, self._port, time.time() - begin))
+
+        try:
+            await asyncio.sleep(0.1)
+            version = await self.send("hypervisor version")
+            self._version = version[0].split("-", 1)[0]
+        except IndexError:
+            self._version = "Unknown"
+
+    @property
+    def version(self):
+        """
+        Returns uBridge version.
+
+        :returns: version string
+        """
+
+        return self._version
+
+    async def close(self):
+        """
+        Closes the connection to this hypervisor (but leave it running).
+        """
+
+        await self.send("hypervisor close")
+        self._writer.close()
+        self._reader, self._writer = None
+
+    async def stop(self):
+        """
+        Stops this hypervisor (will no longer run).
+        """
+
+        try:
+            # try to properly stop the hypervisor
+            await self.send("hypervisor stop")
+        except UbridgeError:
+            pass
+        try:
+            if self._writer is not None:
+                await self._writer.drain()
+                self._writer.close()
+        except OSError as e:
+            log.debug("Stopping hypervisor {}:{} {}".format(self._host, self._port, e))
+        self._reader = self._writer = None
+
+    async def reset(self):
+        """
+        Resets this hypervisor (used to get an empty configuration).
+        """
+
+        await self.send("hypervisor reset")
+
+    @property
+    def port(self):
+        """
+        Returns the port used to start the hypervisor.
+
+        :returns: port number (integer)
+        """
+
+        return self._port
+
+    @port.setter
+    def port(self, port):
+        """
+        Sets the port used to start the hypervisor.
+
+        :param port: port number (integer)
+        """
+
+        self._port = port
+
+    @property
+    def host(self):
+        """
+        Returns the host (binding) used to start the hypervisor.
+
+        :returns: host/address (string)
+        """
+
+        return self._host
+
+    @host.setter
+    def host(self, host):
+        """
+        Sets the host (binding) used to start the hypervisor.
+
+        :param host: host/address (string)
+        """
+
+        self._host = host
+
+    @locking
+    async def send(self, command):
+        """
+        Sends commands to this hypervisor.
+
+        :param command: a uBridge hypervisor command
+
+        :returns: results as a list
+        """
+
+        # uBridge responses are of the form:
+        #   1xx yyyyyy\r\n
+        #   1xx yyyyyy\r\n
+        #   ...
+        #   100-yyyy\r\n
+        # or
+        #   2xx-yyyy\r\n
+        #
+        # Where 1xx is a code from 100-199 for a success or 200-299 for an error
+        # The result might be multiple lines and might be less than the buffer size
+        # but still have more data. The only thing we know for sure is the last line
+        # will begin with '100-' or a '2xx-' and end with '\r\n'
+
+        if self._writer is None or self._reader is None:
+            raise UbridgeError("Not connected")
+
+        try:
+            command = command.strip() + '\n'
+            log.debug("sending {}".format(command))
+            self._writer.write(command.encode())
+            await self._writer.drain()
+        except OSError as e:
+            raise UbridgeError("Lost communication with {host}:{port} when sending command '{command}': {error}, uBridge process running: {run}"
+                               .format(host=self._host, port=self._port, command=command, error=e, run=self.is_running()))
+
+        # Now retrieve the result
+        data = []
+        buf = ''
+        retries = 0
+        max_retries = 10
+        while True:
+            try:
+                try:
+                    chunk = await self._reader.read(1024)
+                except asyncio.CancelledError:
+                    # task has been canceled but continue to read
+                    # any remaining data sent by the hypervisor
+                    continue
+                except ConnectionResetError as e:
+                    # Sometimes WinError 64 (ERROR_NETNAME_DELETED) is returned here on Windows.
+                    # These happen if connection reset is received before IOCP could complete
+                    # a previous operation. Ignore and try again....
+                    log.warning("Connection reset received while reading uBridge response: {}".format(e))
+                    continue
+                if not chunk:
+                    if retries > max_retries:
+                        raise UbridgeError("No data returned from {host}:{port} after sending command '{command}', uBridge process running: {run}"
+                                            .format(host=self._host, port=self._port, command=command, run=self.is_running()))
+                    else:
+                        retries += 1
+                        await asyncio.sleep(0.5)
+                        continue
+                retries = 0
+                buf += chunk.decode("utf-8")
+            except OSError as e:
+                raise UbridgeError("Lost communication with {host}:{port} after sending command '{command}': {error}, uBridge process running: {run}"
+                                   .format(host=self._host, port=self._port, command=command, error=e, run=self.is_running()))
+
+            # If the buffer doesn't end in '\n' then we can't be done
+            try:
+                if buf[-1] != '\n':
+                    continue
+            except IndexError:
+                raise UbridgeError("Could not communicate with {host}:{port} after sending command '{command}', uBridge process running: {run}"
+                                   .format(host=self._host, port=self._port, command=command, run=self.is_running()))
+
+            data += buf.split('\r\n')
+            if data[-1] == '':
+                data.pop()
+            buf = ''
+
+            # Does it contain an error code?
+            if self.error_re.search(data[-1]):
+                raise UbridgeError(data[-1][4:])
+
+            # Or does the last line begin with '100-'? Then we are done!
+            if data[-1][:4] == '100-':
+                data[-1] = data[-1][4:]
+                if data[-1] == 'OK':
+                    data.pop()
+                break
+
+        # Remove success responses codes
+        for index in range(len(data)):
+            if self.success_re.search(data[index]):
+                data[index] = data[index][4:]
+
+        log.debug("returned result {}".format(data))
+        return data