Merge branch '2.1' into 2.2

# Conflicts:
#	gns3server/version.py
#	requirements.txt

gns3server/controller/__init__.py

@@ -123,14 +123,19 @@ class Controller:
                 if prop in ["enable_remote_console", "use_ubridge"]:
                     del vm[prop]
             vm.setdefault("appliance_id", str(uuid.uuid4()))
-            appliance = Appliance(vm["appliance_id"], vm)
-            self._appliances[appliance.id] = appliance
+            try:
+                appliance = Appliance(vm["appliance_id"], vm)
+                self._appliances[appliance.id] = appliance
+            except KeyError as e:
+                # appliance data is not complete (missing name or type)
+                log.warning("Could not load appliance template {} ('{}'): {}".format(vm["appliance_id"], vm.get("name", "unknown"), e))
+                continue
 
         # Add builtins
         builtins = []
         builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "cloud"), {"node_type": "cloud", "name": "Cloud", "category": 2, "symbol": ":/symbols/cloud.svg"}, builtin=True))
         builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "nat"), {"node_type": "nat", "name": "NAT", "category": 2, "symbol": ":/symbols/cloud.svg"}, builtin=True))
-        builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "vpcs"), {"node_type": "vpcs", "name": "VPCS", "category": 2, "symbol": ":/symbols/vpcs_guest.svg", "properties": {"base_script_file": "vpcs_base_config.txt"}}, builtin=True))
+        builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "vpcs"), {"node_type": "vpcs", "name": "VPCS", "default_name_format": "PC-{0}", "category": 2, "symbol": ":/symbols/vpcs_guest.svg", "properties": {"base_script_file": "vpcs_base_config.txt"}}, builtin=True))
         builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_switch"), {"node_type": "ethernet_switch", "name": "Ethernet switch", "category": 1, "symbol": ":/symbols/ethernet_switch.svg"}, builtin=True))
         builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_hub"), {"node_type": "ethernet_hub", "name": "Ethernet hub", "category": 1, "symbol": ":/symbols/hub.svg"}, builtin=True))
         builtins.append(Appliance(uuid.uuid3(uuid.NAMESPACE_DNS, "frame_relay_switch"), {"node_type": "frame_relay_switch", "name": "Frame Relay switch", "category": 1, "symbol": ":/symbols/frame_relay_switch.svg"}, builtin=True))
@@ -405,6 +410,7 @@ class Controller:
         :param connect: True connect to the compute immediately
         :param kwargs: See the documentation of Compute
         """
+
         if compute_id not in self._computes:
 
             # We disallow to create from the outside the local and VM server

gns3server/controller/compute.py

@@ -377,13 +377,13 @@ class Compute:
         """
         :param dont_connect: If true do not reconnect if not connected
         """
+
         if not self._connected and not dont_connect:
             if self._id == "vm" and not self._controller.gns3vm.running:
                 yield from self._controller.gns3vm.start()
-
             yield from self.connect()
         if not self._connected and not dont_connect:
-            raise ComputeError("Can't connect to {}".format(self._name))
+            raise ComputeError("Cannot connect to compute '{}' with request {} {}".format(self._name, method, path))
         response = yield from self._run_http_query(method, path, data=data, **kwargs)
         return response
 
@@ -402,20 +402,20 @@ class Compute:
         """
         Check if remote server is accessible
         """
+
         if not self._connected and not self._closed:
             try:
+                log.info("Connecting to compute '{}'".format(self._id))
                 response = yield from self._run_http_query("GET", "/capabilities")
-            except ComputeError:
+            except ComputeError as e:
                 # Try to reconnect after 2 seconds if server unavailable only if not during tests (otherwise we create a ressources usage bomb)
                 if not hasattr(sys, "_called_from_test") or not sys._called_from_test:
                     self._connection_failure += 1
                     # After 5 failure we close the project using the compute to avoid sync issues
                     if self._connection_failure == 5:
-                        log.warning("Can't connect to compute %s", self._id)
+                        log.warning("Cannot connect to compute '{}': {}".format(self._id, e))
                         yield from self._controller.close_compute_projects(self)
-
                     asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self._try_reconnect()))
-
                 return
             except aiohttp.web.HTTPNotFound:
                 raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server or it's a 1.X server".format(self._id))
@@ -522,10 +522,16 @@ class Compute:
                 else:
                     data = json.dumps(data).encode("utf-8")
         try:
+            log.debug("Attempting request to compute: {method} {url} {headers}".format(
+                method=method,
+                url=url,
+                headers=headers
+            ))
             response = yield from self._session().request(method, url, headers=headers, data=data, auth=self._auth, chunked=chunked, timeout=timeout)
         except asyncio.TimeoutError as e:
             raise ComputeError("Timeout error when connecting to {}".format(url))
-        except (aiohttp.ClientError, aiohttp.ServerDisconnectedError, ValueError, KeyError) as e:
+        except (aiohttp.ClientError, aiohttp.ServerDisconnectedError, ValueError, KeyError, socket.gaierror) as e:
+            #  aiohttp 2.3.1 raises socket.gaierror when cannot find host
             raise ComputeError(str(e))
         body = yield from response.read()
         if body and not raw:

gns3server/controller/gns3vm/__init__.py

@@ -19,6 +19,7 @@ import sys
 import copy
 import asyncio
 import aiohttp
+import ipaddress
 
 from ...utils.asyncio import locked_coroutine
 from .vmware_gns3_vm import VMwareGNS3VM
@@ -26,6 +27,7 @@ from .virtualbox_gns3_vm import VirtualBoxGNS3VM
 from .remote_gns3_vm import RemoteGNS3VM
 from .gns3_vm_error import GNS3VMError
 from ...version import __version__
+from ..compute import ComputeError
 
 import logging
 log = logging.getLogger(__name__)
@@ -280,7 +282,8 @@ class GNS3VM:
             compute = yield from self._controller.add_compute(compute_id="vm",
                                                               name="GNS3 VM is starting ({})".format(engine.vmname),
                                                               host=None,
-                                                              force=True)
+                                                              force=True,
+                                                              connect=False)
 
             try:
                 yield from engine.start()
@@ -289,6 +292,7 @@ class GNS3VM:
                 log.error("Can't start the GNS3 VM: {}".format(str(e)))
                 yield from compute.update(name="GNS3 VM ({})".format(engine.vmname))
                 raise e
+            yield from compute.connect()  # we can connect now that the VM has started
             yield from compute.update(name="GNS3 VM ({})".format(engine.vmname),
                                       protocol=self.protocol,
                                       host=self.ip_address,
@@ -296,6 +300,43 @@ class GNS3VM:
                                       user=self.user,
                                       password=self.password)
 
+            # check if the VM is in the same subnet as the local server, start 10 seconds later to give
+            # some time for the compute in the VM to be ready for requests
+            asyncio.get_event_loop().call_later(10, lambda: asyncio.async(self._check_network(compute)))
+
+    @asyncio.coroutine
+    def _check_network(self, compute):
+        """
+        Check that the VM is in the same subnet as the local server
+        """
+
+        try:
+            vm_interfaces = yield from compute.interfaces()
+            vm_interface_netmask = None
+            for interface in vm_interfaces:
+                if interface["ip_address"] == self.ip_address:
+                    vm_interface_netmask = interface["netmask"]
+                    break
+            if vm_interface_netmask:
+                vm_network = ipaddress.ip_interface("{}/{}".format(compute.host_ip, vm_interface_netmask)).network
+                for compute_id in self._controller.computes:
+                    if compute_id == "local":
+                        compute = self._controller.get_compute(compute_id)
+                        interfaces = yield from compute.interfaces()
+                        netmask = None
+                        for interface in interfaces:
+                            if interface["ip_address"] == compute.host_ip:
+                                netmask = interface["netmask"]
+                                break
+                        if netmask:
+                            compute_network = ipaddress.ip_interface("{}/{}".format(compute.host_ip, netmask)).network
+                            if vm_network.compare_networks(compute_network) != 0:
+                                msg = "The GNS3 VM ({}) is not on the same network as the {} server ({}), please make sure the local server binding is in the same network as the GNS3 VM".format(
+                                    vm_network, compute_id, compute_network)
+                                self._controller.notification.emit("log.warning", {"message": msg})
+        except ComputeError as e:
+            log.warning("Could not check the VM is in the same subnet as the local server: {}".format(e))
+
     @locked_coroutine
     def _suspend(self):
         """

gns3server/controller/gns3vm/vmware_gns3_vm.py

@@ -171,7 +171,7 @@ class VMwareGNS3VM(BaseGNS3VM):
             trial -= 1
             # If ip not found fallback on old method
             if trial == 0:
-                log.warn("No IP found for the VM via readVariable fallback to getGuestIPAddress")
+                log.warning("No IP found for the VM via readVariable fallback to getGuestIPAddress")
                 guest_ip_address = yield from self._execute("getGuestIPAddress", [self._vmx_path, "-wait"], timeout=120)
                 break
             yield from asyncio.sleep(1)

gns3server/controller/link.py

@@ -300,9 +300,12 @@ class Link:
         try:
             stream_content = yield from self.read_pcap_from_source()
         except aiohttp.web.HTTPException as e:
-            log.error("Could not stream pcap file: error {}: {}".format(e.status, e.text))
+            error_msg = "Could not stream PCAP file: error {}: {}".format(e.status, e.text)
+            log.error(error_msg)
             self._capturing = False
+            self._project.notification.emit("log.error", {"message": error_msg})
             self._project.controller.notification.emit("link.updated", self.__json__())
+
         with stream_content as stream:
             with open(self.capture_file_path, "wb+") as f:
                 while self._capturing:

gns3server/controller/node.py

@@ -86,8 +86,7 @@ class Node:
         self._first_port_name = None
 
         # This properties will be recompute
-        ignore_properties = ("width", "height")
-
+        ignore_properties = ("width", "height", "hover_symbol")
         self.properties = kwargs.pop('properties', {})
 
         # Update node properties with additional elements
@@ -104,7 +103,15 @@ class Node:
                         self.properties[prop] = kwargs[prop]
 
         if self._symbol is None:
-            self.symbol = ":/symbols/computer.svg"
+            # compatibility with old node templates
+            if "default_symbol" in self.properties:
+                default_symbol = self.properties.pop("default_symbol")
+                if default_symbol.endswith("normal.svg"):
+                    self.symbol = default_symbol[:-11] + ".svg"
+                else:
+                    self.symbol = default_symbol
+            else:
+                self.symbol = ":/symbols/computer.svg"
 
     def is_always_running(self):
         """
@@ -325,7 +332,7 @@ class Node:
         if self._node_type == "docker":
             timeout = None
         else:
-            timeout = 120
+            timeout = 1200
         trial = 0
         while trial != 6:
             try:

gns3server/controller/project.py

@@ -86,6 +86,7 @@ class Project:
         self._show_grid = show_grid
         self._show_interface_labels = show_interface_labels
         self._loading = False
+        self._add_node_lock = asyncio.Lock()
 
         # Disallow overwrite of existing project
         if project_id is None and path is not None:
@@ -435,34 +436,38 @@ class Project:
         :param dump: Dump topology to disk
         :param kwargs: See the documentation of node
         """
+
         if node_id in self._nodes:
             return self._nodes[node_id]
 
-        if node_type == "iou" and 'application_id' not in kwargs.keys():
+        with (yield from self._add_node_lock):
+            # wait for a node to be completely created before adding a new one
+            # this is important otherwise we allocate the same application ID
+            # when creating multiple IOU node at the same time
+            if node_type == "iou" and 'application_id' not in kwargs.keys():
+                kwargs['application_id'] = get_next_application_id(self._nodes.values())
 
-            kwargs['application_id'] = get_next_application_id(self._nodes.values())
+            node = Node(self, compute, name, node_id=node_id, node_type=node_type, **kwargs)
+            if compute not in self._project_created_on_compute:
+                # For a local server we send the project path
+                if compute.id == "local":
+                    yield from compute.post("/projects", data={
+                        "name": self._name,
+                        "project_id": self._id,
+                        "path": self._path
+                    })
+                else:
+                    yield from compute.post("/projects", data={
+                        "name": self._name,
+                        "project_id": self._id,
+                    })
 
-        node = Node(self, compute, name, node_id=node_id, node_type=node_type, **kwargs)
-        if compute not in self._project_created_on_compute:
-            # For a local server we send the project path
-            if compute.id == "local":
-                yield from compute.post("/projects", data={
-                    "name": self._name,
-                    "project_id": self._id,
-                    "path": self._path
-                })
-            else:
-                yield from compute.post("/projects", data={
-                    "name": self._name,
-                    "project_id": self._id,
-                })
-
-            self._project_created_on_compute.add(compute)
-        yield from node.create()
-        self._nodes[node.id] = node
-        self.controller.notification.emit("node.created", node.__json__())
-        if dump:
-            self.dump()
+                self._project_created_on_compute.add(compute)
+            yield from node.create()
+            self._nodes[node.id] = node
+            self.controller.notification.emit("node.created", node.__json__())
+            if dump:
+                self.dump()
         return node
 
     @locked_coroutine
@@ -976,10 +981,12 @@ class Project:
             raise aiohttp.web.HTTPConflict(text="Cannot duplicate node data while the node is running")
 
         data = copy.deepcopy(node.__json__(topology_dump=True))
-        # Some properties like internal ID should not be duplicate
+        # Some properties like internal ID should not be duplicated
         for unique_property in (
                 'node_id',
                 'name',
+                'mac_addr',
+                'mac_address',
                 'compute_id',
                 'application_id',
                 'dynamips_id'):