Merge branch '2.1' into 2.2

# Conflicts:
#	gns3server/compute/docker/docker_vm.py
#	gns3server/controller/compute.py
#	gns3server/controller/gns3vm/__init__.py
#	gns3server/controller/link.py
#	gns3server/controller/project.py
#	gns3server/handlers/api/controller/project_handler.py
#	gns3server/handlers/api/controller/server_handler.py
#	gns3server/utils/asyncio/__init__.py
#	gns3server/utils/asyncio/telnet_server.py
#	gns3server/version.py
#	gns3server/web/web_server.py
#	tests/compute/qemu/test_qemu_vm.py

gns3server/controller/compute.py

@@ -27,9 +27,9 @@ from operator import itemgetter
 
 from ..utils import parse_version
 from ..utils.images import list_images
-from ..utils.asyncio import locked_coroutine, asyncio_ensure_future
+from ..utils.asyncio import locking, asyncio_ensure_future
 from ..controller.controller_error import ControllerError
-from ..version import __version__
+from ..version import __version__, __version_info__
 
 
 import logging
@@ -95,6 +95,7 @@ class Compute:
         self._set_auth(user, password)
         self._cpu_usage_percent = None
         self._memory_usage_percent = None
+        self._last_error = None
         self._capabilities = {
             "version": None,
             "node_types": []
@@ -137,6 +138,14 @@ class Compute:
                 self._password = None
                 self._auth = aiohttp.BasicAuth(self._user, "")
 
+    def set_last_error(self, msg):
+        """
+        Set the last error message for this compute.
+
+        :param msg: message
+        """
+        self._last_error = msg
+
     @asyncio.coroutine
     def interfaces(self):
         """
@@ -301,7 +310,8 @@ class Compute:
             "connected": self._connected,
             "cpu_usage_percent": self._cpu_usage_percent,
             "memory_usage_percent": self._memory_usage_percent,
-            "capabilities": self._capabilities
+            "capabilities": self._capabilities,
+            "last_error": self._last_error
         }
 
     @asyncio.coroutine
@@ -398,7 +408,8 @@ class Compute:
         except aiohttp.web.HTTPConflict:
             pass
 
-    @locked_coroutine
+    @locking
+    @asyncio.coroutine
     def connect(self):
         """
         Check if remote server is accessible
@@ -420,7 +431,7 @@ class Compute:
                     if self._connection_failure == 10:
                         log.error("Could not connect to compute '{}' after multiple attempts: {}".format(self._id, e))
                         yield from self._controller.close_compute_projects(self)
-                    asyncio.get_event_loop().call_later(2, lambda: asyncio_ensure_future(self._try_reconnect()))
+                    asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self._try_reconnect()))
                 return
             except aiohttp.web.HTTPNotFound:
                 raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server or it's a 1.X server".format(self._id))
@@ -432,16 +443,36 @@ class Compute:
                 raise aiohttp.web.HTTPConflict(text="Invalid server url for server {}".format(self._id))
 
             if "version" not in response.json:
+                msg = "The server {} is not a GNS3 server".format(self._id)
+                log.error(msg)
                 self._http_session.close()
-                raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server".format(self._id))
+                raise aiohttp.web.HTTPConflict(text=msg)
             self._capabilities = response.json
-            if parse_version(__version__)[:2] != parse_version(response.json["version"])[:2]:
-                self._http_session.close()
-                raise aiohttp.web.HTTPConflict(text="The server {} versions are not compatible {} != {}".format(self._id, __version__, response.json["version"]))
+
+            if response.json["version"].split("-")[0] != __version__.split("-")[0]:
+                msg = "GNS3 controller version {} is not the same as compute server {} version {}".format(__version__,
+                                                                                                          self._name,
+                                                                                                          response.json["version"])
+                if __version_info__[3] == 0:
+                    # Stable release
+                    log.error(msg)
+                    self._http_session.close()
+                    self._last_error = msg
+                    raise aiohttp.web.HTTPConflict(text=msg)
+                elif parse_version(__version__)[:2] != parse_version(response.json["version"])[:2]:
+                    # We don't allow different major version to interact even with dev build
+                    log.error(msg)
+                    self._http_session.close()
+                    self._last_error = msg
+                    raise aiohttp.web.HTTPConflict(text=msg)
+                else:
+                    msg = "{}\nUsing different versions may result in unexpected problems. Please use at your own risk.".format(msg)
+                    self._controller.notification.emit("log.warning", {"message": msg})
 
             self._notifications = asyncio.gather(self._connect_notification())
             self._connected = True
             self._connection_failure = 0
+            self._last_error = None
             self._controller.notification.controller_emit("compute.updated", self.__json__())
 
     @asyncio.coroutine

gns3server/controller/export_project.py

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import sys
 import json
 import asyncio
 import aiohttp
@@ -23,6 +24,7 @@ import zipfile
 import tempfile
 import zipstream
 
+from datetime import datetime
 
 import logging
 log = logging.getLogger(__name__)
@@ -77,6 +79,7 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id
             # ignore the .gns3 file
             if file.endswith(".gns3"):
                 continue
+            _patch_mtime(path)
             zstream.write(path, os.path.relpath(path, project._path), compress_type=zipfile.ZIP_DEFLATED)
 
     # Export files from remote computes
@@ -99,12 +102,29 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id
                         f.write(data)
                     response.close()
                     f.close()
+                    _patch_mtime(temp_path)
                     zstream.write(temp_path, arcname=compute_file["path"], compress_type=zipfile.ZIP_DEFLATED)
                     downloaded_files.add(compute_file['path'])
 
     return zstream
 
 
+def _patch_mtime(path):
+    """
+    Patch the file mtime because ZIP does not support timestamps before 1980
+
+    :param path: file path
+    """
+
+    if sys.platform.startswith("win"):
+        # only UNIX type platforms
+        return
+    st = os.stat(path)
+    file_date = datetime.fromtimestamp(st.st_mtime)
+    if file_date.year < 1980:
+        new_mtime = file_date.replace(year=1980).timestamp()
+        os.utime(path, (st.st_atime, new_mtime))
+
 def _is_exportable(path):
     """
     :returns: True if file should not be included in the final archive
@@ -228,6 +248,7 @@ def _export_local_image(image, zstream):
 
         if os.path.exists(path):
             arcname = os.path.join("images", directory, os.path.basename(image))
+            _patch_mtime(path)
             zstream.write(path, arcname)
             return
 

gns3server/controller/gns3vm/__init__.py

@@ -21,7 +21,7 @@ import asyncio
 import aiohttp
 import ipaddress
 
-from ...utils.asyncio import locked_coroutine, asyncio_ensure_future
+from ...utils.asyncio import locking, asyncio_ensure_future
 from .vmware_gns3_vm import VMwareGNS3VM
 from .virtualbox_gns3_vm import VirtualBoxGNS3VM
 from .hyperv_gns3_vm import HyperVGNS3VM
@@ -278,13 +278,15 @@ class GNS3VM:
             except GNS3VMError as e:
                 # User will receive the error later when they will try to use the node
                 try:
-                    yield from self._controller.add_compute(compute_id="vm",
-                                                            name="GNS3 VM ({})".format(self.current_engine().vmname),
-                                                            host=None,
-                                                            force=True)
+                    compute = yield from self._controller.add_compute(compute_id="vm",
+                                                                      name="GNS3 VM ({})".format(self.current_engine().vmname),
+                                                                      host=None,
+                                                                      force=True)
+                    compute.set_last_error(str(e))
+
                 except aiohttp.web.HTTPConflict:
                     pass
-                log.error("Can't start the GNS3 VM: %s", str(e))
+                log.error("Cannot start the GNS3 VM: {}".format(e))
 
     @asyncio.coroutine
     def exit_vm(self):
@@ -298,7 +300,8 @@ class GNS3VM:
             except GNS3VMError as e:
                 log.warning(str(e))
 
-    @locked_coroutine
+    @locking
+    @asyncio.coroutine
     def start(self):
         """
         Start the GNS3 VM
@@ -323,8 +326,9 @@ class GNS3VM:
                 yield from engine.start()
             except Exception as e:
                 yield from self._controller.delete_compute("vm")
-                log.error("Can't start the GNS3 VM: {}".format(str(e)))
+                log.error("Cannot start the GNS3 VM: {}".format(str(e)))
                 yield from compute.update(name="GNS3 VM ({})".format(engine.vmname))
+                compute.set_last_error(str(e))
                 raise e
             yield from compute.connect()  # we can connect now that the VM has started
             yield from compute.update(name="GNS3 VM ({})".format(engine.vmname),
@@ -370,8 +374,11 @@ class GNS3VM:
                                 self._controller.notification.controller_emit("log.warning", {"message": msg})
         except ComputeError as e:
             log.warning("Could not check the VM is in the same subnet as the local server: {}".format(e))
+        except aiohttp.web.HTTPConflict as e:
+            log.warning("Could not check the VM is in the same subnet as the local server: {}".format(e.text))
 
-    @locked_coroutine
+    @locking
+    @asyncio.coroutine
     def _suspend(self):
         """
         Suspend the GNS3 VM
@@ -383,7 +390,8 @@ class GNS3VM:
             log.info("Suspend the GNS3 VM")
             yield from engine.suspend()
 
-    @locked_coroutine
+    @locking
+    @asyncio.coroutine
     def _stop(self):
         """
         Stop the GNS3 VM

gns3server/controller/gns3vm/virtualbox_gns3_vm.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import sys
 import aiohttp
 import logging
 import asyncio
@@ -107,7 +108,6 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
         Check if the DHCP server associated with a vboxnet is enabled.
 
         :param vboxnet: vboxnet name
-
         :returns: boolean
         """
 
@@ -125,6 +125,41 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
                     return True
         return False
 
+    @asyncio.coroutine
+    def _check_vboxnet_exists(self, vboxnet):
+        """
+        Check if the vboxnet interface exists
+
+        :param vboxnet: vboxnet name
+        :returns: boolean
+        """
+
+        properties = yield from self._execute("list", ["hostonlyifs"])
+        for prop in properties.splitlines():
+            try:
+                name, value = prop.split(':', 1)
+            except ValueError:
+                continue
+            if name.strip() == "Name" and value.strip() == vboxnet:
+                return True
+        return False
+
+    @asyncio.coroutine
+    def _find_first_available_vboxnet(self):
+        """
+        Find the first available vboxnet.
+        """
+
+        properties = yield from self._execute("list", ["hostonlyifs"])
+        for prop in properties.splitlines():
+            try:
+                name, value = prop.split(':', 1)
+            except ValueError:
+                continue
+            if name.strip() == "Name":
+                return value.strip()
+        return None
+
     @asyncio.coroutine
     def _check_vbox_port_forwarding(self):
         """
@@ -158,18 +193,32 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
         # get a NAT interface number
         nat_interface_number = yield from self._look_for_interface("nat")
         if nat_interface_number < 0:
-            raise GNS3VMError("The GNS3 VM: {} must have a NAT interface configured in order to start".format(self.vmname))
+            raise GNS3VMError('VM "{}" must have a NAT interface configured in order to start'.format(self.vmname))
 
         hostonly_interface_number = yield from self._look_for_interface("hostonly")
         if hostonly_interface_number < 0:
-            raise GNS3VMError("The GNS3 VM: {} must have a host only interface configured in order to start".format(self.vmname))
+            raise GNS3VMError('VM "{}" must have a host-only interface configured in order to start'.format(self.vmname))
 
         vboxnet = yield from self._look_for_vboxnet(hostonly_interface_number)
         if vboxnet is None:
-            raise GNS3VMError("VirtualBox host-only network could not be found for interface {} on GNS3 VM".format(hostonly_interface_number))
+            raise GNS3VMError('A VirtualBox host-only network could not be found on network adapter {} for "{}"'.format(hostonly_interface_number, self._vmname))
+
+        if not (yield from self._check_vboxnet_exists(vboxnet)):
+            if sys.platform.startswith("win") and vboxnet == "vboxnet0":
+                # The GNS3 VM is configured with vboxnet0 by default which is not available
+                # on Windows. Try to patch this with the first available vboxnet we find.
+                first_available_vboxnet = yield from self._find_first_available_vboxnet()
+                if first_available_vboxnet is None:
+                    raise GNS3VMError('Please add a VirtualBox host-only network with DHCP enabled and attached it to network adapter {} for "{}"'.format(hostonly_interface_number, self._vmname))
+                yield from self.set_hostonly_network(hostonly_interface_number, first_available_vboxnet)
+                vboxnet = first_available_vboxnet
+            else:
+                raise GNS3VMError('VirtualBox host-only network "{}" does not exist, please make the sure the network adapter {} configuration is valid for "{}"'.format(vboxnet,
+                                                                                                                                                                         hostonly_interface_number,
+                                                                                                                                                                         self._vmname))
 
         if not (yield from self._check_dhcp_server(vboxnet)):
-            raise GNS3VMError("DHCP must be enabled on VirtualBox host-only network: {} for GNS3 VM".format(vboxnet))
+            raise GNS3VMError('DHCP must be enabled on VirtualBox host-only network "{}"'.format(vboxnet))
 
         vm_state = yield from self._get_state()
         log.info('"{}" state is {}'.format(self._vmname, vm_state))
@@ -311,3 +360,17 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
 
         yield from self._execute("modifyvm", [self._vmname, "--memory", str(ram)], timeout=3)
         log.info("GNS3 VM RAM amount set to {}".format(ram))
+
+    @asyncio.coroutine
+    def set_hostonly_network(self, adapter_number, hostonly_network_name):
+        """
+        Set a VirtualBox host-only network on a network adapter for the GNS3 VM.
+
+        :param adapter_number: network adapter number
+        :param hostonly_network_name: name of the VirtualBox host-only network
+        """
+
+        yield from self._execute("modifyvm", [self._vmname, "--hostonlyadapter{}".format(adapter_number), hostonly_network_name], timeout=3)
+        log.info('VirtualBox host-only network "{}" set on network adapter {} for "{}"'.format(hostonly_network_name,
+                                                                                               adapter_number,
+                                                                                               self._vmname))

gns3server/controller/ports/atm_port.py

@@ -33,8 +33,8 @@ class ATMPort(SerialPort):
         """
         return "ATM"
 
-    @property
-    def short_name_type(self):
+    @staticmethod
+    def short_name_type():
         """
         Returns the short name type for this port.
 
@@ -42,10 +42,6 @@ class ATMPort(SerialPort):
         """
         return "a"
 
-    @property
-    def short_name(self):
-        return "{}".format(self._port_number)
-
     @property
     def data_link_types(self):
         """

gns3server/controller/ports/ethernet_port.py

@@ -39,8 +39,8 @@ class EthernetPort(Port):
 
         return "Ethernet"
 
-    @property
-    def short_name_type(self):
+    @staticmethod
+    def short_name_type():
         """
         Returns the short name type for this port.
 

gns3server/controller/ports/fastethernet_port.py

@@ -34,8 +34,8 @@ class FastEthernetPort(Port):
 
         return "FastEthernet"
 
-    @property
-    def short_name_type(self):
+    @staticmethod
+    def short_name_type():
         """
         Returns the short name type for this port.
 

gns3server/controller/ports/frame_relay_port.py

@@ -34,10 +34,6 @@ class FrameRelayPort(SerialPort):
 
         return "FrameRelay"
 
-    @property
-    def short_name(self):
-        return "{}".format(self._port_number)
-
     @property
     def data_link_types(self):
         """

gns3server/controller/ports/gigabitethernet_port.py

@@ -34,8 +34,8 @@ class GigabitEthernetPort(Port):
 
         return "GigabitEthernet"
 
-    @property
-    def short_name_type(self):
+    @staticmethod
+    def short_name_type():
         """
         Returns the short name type for this port.
 

gns3server/controller/ports/port.py

@@ -85,9 +85,9 @@ class Port:
         if self._short_name:
             return self._short_name
         elif '/' in self._name:
-            return self._name.replace(self.long_name_type(), self.short_name_type)
+            return self._name.replace(self.long_name_type(), self.short_name_type())
         elif self._name.startswith("{}{}".format(self.long_name_type(), self._interface_number)):
-            return self.short_name_type + "{}".format(self._interface_number)
+            return self.short_name_type() + "{}".format(self._interface_number)
         return self._name
 
     @short_name.setter

gns3server/controller/ports/port_factory.py

@@ -209,7 +209,7 @@ class DynamipsPortFactory:
                     for port_number in range(0, cls.ADAPTER_MATRIX[properties[name]]["nb_ports"]):
                         name = "{}{}/{}".format(port_class.long_name_type(), adapter_number, port_number)
                         port = port_class(name, adapter_number, adapter_number, port_number)
-                        port.short_name = "{}{}/{}".format(port.short_name_type, adapter_number, port_number)
+                        port.short_name = "{}{}/{}".format(port_class.short_name_type(), adapter_number, port_number)
                         ports.append(port)
                 adapter_number += 1
             elif name.startswith("wic") and properties[name]:
@@ -218,7 +218,7 @@ class DynamipsPortFactory:
                     for port_number in range(0, cls.WIC_MATRIX[properties[name]]["nb_ports"]):
                         name = "{}{}/{}".format(port_class.long_name_type(), 0, display_wic_port_number)
                         port = port_class(name, 0, 0, wic_port_number)
-                        port.short_name = "{}{}/{}".format(port.short_name_type, 0, display_wic_port_number)
+                        port.short_name = "{}{}/{}".format(port_class.short_name_type(), 0, display_wic_port_number)
                         ports.append(port)
                         display_wic_port_number += 1
                         wic_port_number += 1

gns3server/controller/ports/pos_port.py

@@ -33,8 +33,8 @@ class POSPort(SerialPort):
         """
         return "POS"
 
-    @property
-    def short_name_type(self):
+    @staticmethod
+    def short_name_type():
         """
         Returns the short name type for this port.
 

gns3server/controller/ports/serial_port.py

@@ -34,8 +34,8 @@ class SerialPort(Port):
 
         return "Serial"
 
-    @property
-    def short_name_type(self):
+    @staticmethod
+    def short_name_type():
         """
         Returns the short name type for this port.
 

gns3server/controller/project.py

@@ -36,8 +36,9 @@ from .udp_link import UDPLink
 from ..config import Config
 from ..utils.path import check_path_allowed, get_default_project_directory
 from ..utils.asyncio.pool import Pool
-from ..utils.asyncio import locked_coroutine, asyncio_ensure_future
+from ..utils.asyncio import locking
 from ..utils.asyncio import wait_run_in_executor
+from ..utils.asyncio import asyncio_ensure_future
 from .export_project import export_project
 from .import_project import import_project
 
@@ -524,7 +525,8 @@ class Project:
             self.dump()
         return node
 
-    @locked_coroutine
+    @locking
+    @asyncio.coroutine
     def __delete_node_links(self, node):
         """
         Delete all link connected to this node.
@@ -814,7 +816,8 @@ class Project:
     def _topology_file(self):
         return os.path.join(self.path, self._filename)
 
-    @locked_coroutine
+    @locking
+    @asyncio.coroutine
     def open(self):
         """
         Load topology elements

gns3server/controller/snapshot.py

@@ -101,7 +101,7 @@ class Snapshot:
             with tempfile.TemporaryDirectory() as tmpdir:
                 zipstream = yield from export_project(self._project, tmpdir, keep_compute_id=True, allow_all_nodes=True)
                 yield from wait_run_in_executor(self._create_snapshot_file, zipstream)
-        except OSError as e:
+        except (ValueError, OSError, RuntimeError) as e:
             raise aiohttp.web.HTTPConflict(text="Could not create snapshot file '{}': {}".format(self.path, e))
 
     @asyncio.coroutine