Fix installation with Python 3.7. Fixes #1414.

Fix deprecated use of aiohttp.Timeout. Fixes #1296.
Use "async with" with aiohttp.ClientSession().
Make sure websocket connections are properly closed, see https://docs.aiohttp.org/en/stable/web_advanced.html#graceful-shutdown
Finish to drop Python 3.4.

gns3server/controller/__init__.py

@@ -67,50 +67,46 @@ class Controller:
     @locking
     async def download_appliance_templates(self):
 
-        session = aiohttp.ClientSession()
         try:
             headers = {}
             if self._appliance_templates_etag:
                 log.info("Checking if appliance templates are up-to-date (ETag {})".format(self._appliance_templates_etag))
                 headers["If-None-Match"] = self._appliance_templates_etag
-            response = await session.get('https://api.github.com/repos/GNS3/gns3-registry/contents/appliances', headers=headers)
-            if response.status == 304:
-                log.info("Appliance templates are already up-to-date (ETag {})".format(self._appliance_templates_etag))
-                return
-            elif response.status != 200:
-                raise aiohttp.web.HTTPConflict(text="Could not retrieve appliance templates on GitHub due to HTTP error code {}".format(response.status))
-            etag = response.headers.get("ETag")
-            if etag:
-                self._appliance_templates_etag = etag
-                self.save()
-            json_data = await response.json()
-            response.close()
-            appliances_dir = get_resource('appliances')
-            for appliance in json_data:
-                if appliance["type"] == "file":
-                    appliance_name = appliance["name"]
-                    log.info("Download appliance template file from '{}'".format(appliance["download_url"]))
-                    response = await session.get(appliance["download_url"])
-                    if response.status != 200:
-                        log.warning("Could not download '{}' due to HTTP error code {}".format(appliance["download_url"], response.status))
-                        continue
-                    try:
-                        appliance_data = await response.read()
-                    except asyncio.TimeoutError:
-                        log.warning("Timeout while downloading '{}'".format(appliance["download_url"]))
-                        continue
-                    path = os.path.join(appliances_dir, appliance_name)
-
-                    try:
-                        log.info("Saving {} file to {}".format(appliance_name, path))
-                        with open(path, 'wb') as f:
-                            f.write(appliance_data)
-                    except OSError as e:
-                        raise aiohttp.web.HTTPConflict(text="Could not write appliance template file '{}': {}".format(path, e))
+            async with aiohttp.ClientSession() as session:
+                async with session.get('https://api.github.com/repos/GNS3/gns3-registry/contents/appliances', headers=headers) as response:
+                    if response.status == 304:
+                        log.info("Appliance templates are already up-to-date (ETag {})".format(self._appliance_templates_etag))
+                        return
+                    elif response.status != 200:
+                        raise aiohttp.web.HTTPConflict(text="Could not retrieve appliance templates on GitHub due to HTTP error code {}".format(response.status))
+                    etag = response.headers.get("ETag")
+                    if etag:
+                        self._appliance_templates_etag = etag
+                        self.save()
+                    json_data = await response.json()
+                appliances_dir = get_resource('appliances')
+                for appliance in json_data:
+                    if appliance["type"] == "file":
+                        appliance_name = appliance["name"]
+                        log.info("Download appliance template file from '{}'".format(appliance["download_url"]))
+                        async with session.get(appliance["download_url"]) as response:
+                            if response.status != 200:
+                                log.warning("Could not download '{}' due to HTTP error code {}".format(appliance["download_url"], response.status))
+                                continue
+                            try:
+                                appliance_data = await response.read()
+                            except asyncio.TimeoutError:
+                                log.warning("Timeout while downloading '{}'".format(appliance["download_url"]))
+                                continue
+                            path = os.path.join(appliances_dir, appliance_name)
+                            try:
+                                log.info("Saving {} file to {}".format(appliance_name, path))
+                                with open(path, 'wb') as f:
+                                    f.write(appliance_data)
+                            except OSError as e:
+                                raise aiohttp.web.HTTPConflict(text="Could not write appliance template file '{}': {}".format(path, e))
         except ValueError as e:
             raise aiohttp.web.HTTPConflict(text="Could not read appliance templates information from GitHub: {}".format(e))
-        finally:
-            session.close()
 
     def load_appliance_templates(self):
 

gns3server/controller/compute.py

@@ -18,6 +18,7 @@
 import ipaddress
 import aiohttp
 import asyncio
+import async_timeout
 import socket
 import json
 import uuid
@@ -52,22 +53,6 @@ class ComputeConflict(aiohttp.web.HTTPConflict):
         self.response = response
 
 
-class Timeout(aiohttp.Timeout):
-    """
-    Could be removed with aiohttp 0.22 that support None timeout
-    """
-
-    def __enter__(self):
-        if self._timeout:
-            return super().__enter__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._timeout:
-            return super().__exit__(exc_type, exc_val, exc_tb)
-        return self
-
-
 class Compute:
     """
     A GNS3 compute.
@@ -101,12 +86,8 @@ class Compute:
             "node_types": []
         }
         self.name = name
-        # Websocket for notifications
-        self._ws = None
-
         # Cache of interfaces on remote host
         self._interfaces_cache = None
-
         self._connection_failure = 0
 
     def _session(self):
@@ -114,9 +95,10 @@ class Compute:
             self._http_session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=None, force_close=True))
         return self._http_session
 
-    def __del__(self):
-        if self._http_session:
-            self._http_session.close()
+    #def __del__(self):
+    #    pass
+    #    if self._http_session:
+    #        self._http_session.close()
 
     def _set_auth(self, user, password):
         """
@@ -162,19 +144,16 @@ class Compute:
         # It's important to set user and password at the same time
         if "user" in kwargs or "password" in kwargs:
             self._set_auth(kwargs.get("user", self._user), kwargs.get("password", self._password))
-        if self._http_session:
-            self._http_session.close()
+        if self._http_session and not self._http_session.closed:
+            await self._http_session.close()
         self._connected = False
         self._controller.notification.controller_emit("compute.updated", self.__json__())
         self._controller.save()
 
     async def close(self):
         self._connected = False
-        if self._http_session:
-            self._http_session.close()
-        if self._ws:
-            await self._ws.close()
-            self._ws = None
+        if self._http_session and not self._http_session.closed:
+            await self._http_session.close()
         self._closed = True
 
     @property
@@ -474,35 +453,27 @@ class Compute:
         """
         Connect to the notification stream
         """
-        try:
-            self._ws = await self._session().ws_connect(self._getUrl("/notifications/ws"), auth=self._auth)
-        except (aiohttp.WSServerHandshakeError, aiohttp.ClientResponseError):
-            self._ws = None
-        while self._ws is not None:
-            try:
-                response = await self._ws.receive()
-            except aiohttp.WSServerHandshakeError:
-                self._ws = None
-                break
-            if response.tp == aiohttp.WSMsgType.closed or response.tp == aiohttp.WSMsgType.error or response.data is None:
-                self._connected = False
-                break
-            msg = json.loads(response.data)
-            action = msg.pop("action")
-            event = msg.pop("event")
-            if action == "ping":
-                self._cpu_usage_percent = event["cpu_usage_percent"]
-                self._memory_usage_percent = event["memory_usage_percent"]
-                self._controller.notification.controller_emit("compute.updated", self.__json__())
-            else:
-                await self._controller.notification.dispatch(action, event, compute_id=self.id)
-        if self._ws:
-            await self._ws.close()
+
+        async with self._session().ws_connect(self._getUrl("/notifications/ws"), auth=self._auth) as ws:
+            async for response in ws:
+                if response.type == aiohttp.WSMsgType.TEXT and response.data:
+                    msg = json.loads(response.data)
+                    action = msg.pop("action")
+                    event = msg.pop("event")
+                    if action == "ping":
+                        self._cpu_usage_percent = event["cpu_usage_percent"]
+                        self._memory_usage_percent = event["memory_usage_percent"]
+                        self._controller.notification.controller_emit("compute.updated", self.__json__())
+                    else:
+                        await self._controller.notification.dispatch(action, event, compute_id=self.id)
+                elif response.type == aiohttp.WSMsgType.CLOSED or response.type == aiohttp.WSMsgType.ERROR or response.data is None:
+                    self._connected = False
+                    break
 
         # Try to reconnect after 1 seconds if server unavailable only if not during tests (otherwise we create a ressources usage bomb)
         if not hasattr(sys, "_called_from_test") or not sys._called_from_test:
             asyncio.get_event_loop().call_later(1, lambda: asyncio.ensure_future(self.connect()))
-        self._ws = None
+
         self._cpu_usage_percent = None
         self._memory_usage_percent = None
         self._controller.notification.controller_emit("compute.updated", self.__json__())
@@ -527,7 +498,7 @@ class Compute:
         return self._getUrl(path)
 
     async def _run_http_query(self, method, path, data=None, timeout=20, raw=False):
-        with Timeout(timeout):
+        with async_timeout.timeout(timeout):
             url = self._getUrl(path)
             headers = {}
             headers['content-type'] = 'application/json'

gns3server/controller/gns3vm/virtualbox_gns3_vm.py

@@ -261,29 +261,22 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
         """
         remaining_try = 300
         while remaining_try > 0:
-            json_data = None
-            session = aiohttp.ClientSession()
-            try:
-                resp = None
-                resp = await session.get('http://127.0.0.1:{}/v2/compute/network/interfaces'.format(api_port))
-            except (OSError, aiohttp.ClientError, TimeoutError, asyncio.TimeoutError):
-                pass
-
-            if resp:
-                if resp.status < 300:
-                    try:
-                        json_data = await resp.json()
-                    except ValueError:
-                        pass
-                resp.close()
-
-            session.close()
-
-            if json_data:
-                for interface in json_data:
-                    if "name" in interface and interface["name"] == "eth{}".format(hostonly_interface_number - 1):
-                        if "ip_address" in interface and len(interface["ip_address"]) > 0:
-                            return interface["ip_address"]
+            async with aiohttp.ClientSession() as session:
+                try:
+                    async with session.get('http://127.0.0.1:{}/v2/compute/network/interfaces'.format(api_port)) as resp:
+                        if resp.status < 300:
+                            try:
+                                json_data = await resp.json()
+                                if json_data:
+                                    for interface in json_data:
+                                        if "name" in interface and interface["name"] == "eth{}".format(
+                                                hostonly_interface_number - 1):
+                                            if "ip_address" in interface and len(interface["ip_address"]) > 0:
+                                                return interface["ip_address"]
+                            except ValueError:
+                                pass
+                except (OSError, aiohttp.ClientError, TimeoutError, asyncio.TimeoutError):
+                    pass
             remaining_try -= 1
             await asyncio.sleep(1)
         raise GNS3VMError("Could not get the GNS3 VM ip make sure the VM receive an IP from VirtualBox")