Drop Python 3.4 and switch to async / await syntax for asyncio. Fixes #1425

tests/compute/docker/test_docker.py

@@ -39,8 +39,7 @@ def test_query_success(loop, vm):
     response.status = 200
     response.headers = {'CONTENT-TYPE': 'application/json'}
 
-    @asyncio.coroutine
-    def read():
+    async def read():
         return b'{"c": false}'
 
     response.read.side_effect = read
@@ -61,8 +60,7 @@ def test_query_error(loop, vm):
     response = MagicMock()
     response.status = 404
 
-    @asyncio.coroutine
-    def read():
+    async def read():
         return b"NOT FOUND"
 
     response.read.side_effect = read
@@ -82,8 +80,7 @@ def test_query_error_json(loop, vm):
     response = MagicMock()
     response.status = 404
 
-    @asyncio.coroutine
-    def read():
+    async def read():
         return b'{"message": "Error"}'
 
     response.read.side_effect = read
@@ -145,8 +142,7 @@ def test_pull_image(loop):
         def __init__(self):
             self._read = -1
 
-        @asyncio.coroutine
-        def read(self, size):
+        async def read(self, size):
             self._read += 1
             if self._read == 0:
                 return b'{"progress": "0/100",'

tests/compute/docker/test_docker_vm.py

@@ -366,8 +366,7 @@ def test_create_image_not_available(loop, project, manager):
 
     call = 0
 
-    @asyncio.coroutine
-    def information():
+    async def information():
         nonlocal call
         if call == 0:
             call += 1
@@ -927,8 +926,8 @@ def test_stop_capture(vm, tmpdir, manager, free_console_port, loop):
 
 
 def test_get_log(loop, vm):
-    @asyncio.coroutine
-    def read():
+
+    async def read():
         return b'Hello\nWorld'
 
     mock_query = MagicMock()
@@ -1036,8 +1035,7 @@ def test_read_console_output_with_binary_mode(vm, loop):
         def __init__(self):
             self.sent = False
 
-        @asyncio.coroutine
-        def receive(self):
+        async def receive(self):
             if not self.sent:
                 self.sent = True
                 return MagicMock(tp=aiohttp.WSMsgType.BINARY, data=b"test")

tests/compute/virtualbox/test_virtualbox_manager.py

@@ -79,8 +79,7 @@ def test_list_vms(manager, loop):
                '"<inaccessible>" {42b4d095-ff5f-4ac4-bb9d-5f2c7861c1f1}',
                '"Linux Microcore 4.7.1" {ccd8c50b-c172-457d-99fa-dd69371ede0e}']
 
-    @asyncio.coroutine
-    def execute_mock(cmd, args):
+    async def execute_mock(cmd, args):
         if cmd == "list":
             return vm_list
         else:

tests/controller/test_compute.py

@@ -205,8 +205,7 @@ def test_connectNotification(compute, async_run):
 
     call = 0
 
-    @asyncio.coroutine
-    def receive():
+    async def receive():
         nonlocal call
         call += 1
         if call == 1:
@@ -238,8 +237,7 @@ def test_connectNotificationPing(compute, async_run):
 
     call = 0
 
-    @asyncio.coroutine
-    def receive():
+    async def receive():
         nonlocal call
         call += 1
         if call == 1:

tests/controller/test_link.py

@@ -284,10 +284,10 @@ def test_json_serial_link(async_run, project, compute, link):
 
 
 def test_start_streaming_pcap(link, async_run, tmpdir, project):
-    @asyncio.coroutine
-    def fake_reader():
+
+    async def fake_reader():
         output = AsyncioBytesIO()
-        yield from output.write(b"hello")
+        await output.write(b"hello")
         output.seek(0)
         return output
 
@@ -314,8 +314,8 @@ def test_default_capture_file_name(project, compute, async_run):
 
 
 def test_start_capture(link, async_run, tmpdir, project, controller):
-    @asyncio.coroutine
-    def fake_reader():
+
+    async def fake_reader():
         return AsyncioBytesIO()
 
     link.read_pcap_from_source = fake_reader

tests/controller/test_node.py

@@ -211,8 +211,7 @@ def test_create_image_missing(node, compute, project, async_run):
 
     node.__calls = 0
 
-    @asyncio.coroutine
-    def resp(*args, **kwargs):
+    async def resp(*args, **kwargs):
         node.__calls += 1
         response = MagicMock()
         if node.__calls == 1:

tests/controller/test_udp_link.py

@@ -41,8 +41,7 @@ def test_create(async_run, project):
     node2 = Node(project, compute2, "node2", node_type="vpcs")
     node2._ports = [EthernetPort("E0", 0, 3, 1)]
 
-    @asyncio.coroutine
-    def subnet_callback(compute2):
+    async def subnet_callback(compute2):
         """
         Fake subnet callback
         """
@@ -54,8 +53,7 @@ def test_create(async_run, project):
     async_run(link.add_node(node1, 0, 4))
     async_run(link.update_filters({"latency": [10]}))
 
-    @asyncio.coroutine
-    def compute1_callback(path, data={}, **kwargs):
+    async def compute1_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -64,8 +62,7 @@ def test_create(async_run, project):
             response.json = {"udp_port": 1024}
             return response
 
-    @asyncio.coroutine
-    def compute2_callback(path, data={}, **kwargs):
+    async def compute2_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -107,8 +104,7 @@ def test_create_one_side_failure(async_run, project):
     node2 = Node(project, compute2, "node2", node_type="vpcs")
     node2._ports = [EthernetPort("E0", 0, 3, 1)]
 
-    @asyncio.coroutine
-    def subnet_callback(compute2):
+    async def subnet_callback(compute2):
         """
         Fake subnet callback
         """
@@ -119,8 +115,7 @@ def test_create_one_side_failure(async_run, project):
     link = UDPLink(project)
     async_run(link.add_node(node1, 0, 4))
 
-    @asyncio.coroutine
-    def compute1_callback(path, data={}, **kwargs):
+    async def compute1_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -129,8 +124,7 @@ def test_create_one_side_failure(async_run, project):
             response.json = {"udp_port": 1024}
             return response
 
-    @asyncio.coroutine
-    def compute2_callback(path, data={}, **kwargs):
+    async def compute2_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -322,8 +316,7 @@ def test_update(async_run, project):
     node2 = Node(project, compute2, "node2", node_type="vpcs")
     node2._ports = [EthernetPort("E0", 0, 3, 1)]
 
-    @asyncio.coroutine
-    def subnet_callback(compute2):
+    async def subnet_callback(compute2):
         """
         Fake subnet callback
         """
@@ -335,8 +328,7 @@ def test_update(async_run, project):
     async_run(link.add_node(node1, 0, 4))
     async_run(link.update_filters({"latency": [10]}))
 
-    @asyncio.coroutine
-    def compute1_callback(path, data={}, **kwargs):
+    async def compute1_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -345,8 +337,7 @@ def test_update(async_run, project):
             response.json = {"udp_port": 1024}
             return response
 
-    @asyncio.coroutine
-    def compute2_callback(path, data={}, **kwargs):
+    async def compute2_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -402,8 +393,7 @@ def test_update_suspend(async_run, project):
     node2 = Node(project, compute2, "node2", node_type="vpcs")
     node2._ports = [EthernetPort("E0", 0, 3, 1)]
 
-    @asyncio.coroutine
-    def subnet_callback(compute2):
+    async def subnet_callback(compute2):
         """
         Fake subnet callback
         """
@@ -416,8 +406,7 @@ def test_update_suspend(async_run, project):
     async_run(link.update_filters({"latency": [10]}))
     async_run(link.update_suspend(True))
 
-    @asyncio.coroutine
-    def compute1_callback(path, data={}, **kwargs):
+    async def compute1_callback(path, data={}, **kwargs):
         """
         Fake server
         """
@@ -426,8 +415,7 @@ def test_update_suspend(async_run, project):
             response.json = {"udp_port": 1024}
             return response
 
-    @asyncio.coroutine
-    def compute2_callback(path, data={}, **kwargs):
+    async def compute2_callback(path, data={}, **kwargs):
         """
         Fake server
         """

tests/handlers/api/base.py

@@ -42,9 +42,8 @@ class Query:
         self._api_version = api_version
         self._session = None
 
-    @asyncio.coroutine
-    def close(self):
-        yield from self._session.close()
+    async def close(self):
+        await self._session.close()
 
     def post(self, path, body={}, **kwargs):
         return self._fetch("POST", path, body, **kwargs)
@@ -69,9 +68,8 @@ class Query:
         """
         self._session = aiohttp.ClientSession()
 
-        @asyncio.coroutine
-        def go_request(future):
-            response = yield from self._session.ws_connect(self.get_url(path))
+        async def go_request(future):
+            response = await self._session.ws_connect(self.get_url(path))
             future.set_result(response)
         future = asyncio.Future()
         asyncio.ensure_future(go_request(future))
@@ -87,14 +85,13 @@ class Query:
         """
         return self._loop.run_until_complete(asyncio.ensure_future(self._async_fetch(method, path, body=body, **kwargs)))
 
-    @asyncio.coroutine
-    def _async_fetch(self, method, path, body=None, **kwargs):
+    async def _async_fetch(self, method, path, body=None, **kwargs):
         if body is not None and not kwargs.get("raw", False):
             body = json.dumps(body)
 
         connector = aiohttp.TCPConnector()
-        response = yield from aiohttp.request(method, self.get_url(path), data=body, loop=self._loop, connector=connector)
-        response.body = yield from response.read()
+        response = await aiohttp.request(method, self.get_url(path), data=body, loop=self._loop, connector=connector)
+        response.body = await response.read()
         x_route = response.headers.get('X-Route', None)
         if x_route is not None:
             response.route = x_route.replace("/v{}".format(self._api_version), "")

tests/handlers/api/controller/test_link.py

@@ -341,10 +341,10 @@ def test_stop_capture(http_controller, tmpdir, project, compute, async_run):
 
 
 def test_pcap(http_controller, tmpdir, project, compute, loop):
-    @asyncio.coroutine
-    def go(future):
-        response = yield from aiohttp.request("GET", http_controller.get_url("/projects/{}/links/{}/pcap".format(project.id, link.id)))
-        response.body = yield from response.content.read(5)
+
+    async def go(future):
+        response = await aiohttp.request("GET", http_controller.get_url("/projects/{}/links/{}/pcap".format(project.id, link.id)))
+        response.body = await response.content.read(5)
         response.close()
         future.set_result(response)
 

tests/handlers/api/controller/test_project.py

@@ -169,13 +169,13 @@ def test_load_project(http_controller, project, config):
 
 
 def test_notification(http_controller, project, controller, loop, async_run):
-    @asyncio.coroutine
-    def go():
+
+    async def go():
         connector = aiohttp.TCPConnector()
-        response = yield from aiohttp.request("GET", http_controller.get_url("/projects/{project_id}/notifications".format(project_id=project.id)), connector=connector)
-        response.body = yield from response.content.read(200)
+        response = await aiohttp.request("GET", http_controller.get_url("/projects/{project_id}/notifications".format(project_id=project.id)), connector=connector)
+        response.body = await response.content.read(200)
         controller.notification.project_emit("node.created", {"a": "b"})
-        response.body += yield from response.content.readany()
+        response.body += await response.content.readany()
         response.close()
         return response
 

tests/handlers/api/controller/test_server.py

@@ -31,8 +31,8 @@ def web_server():
 
 
 def test_shutdown_local(http_controller, web_server, config):
-    @asyncio.coroutine
-    def hello():
+
+    async def hello():
         return 0
 
     web_server.shutdown_server.return_value = hello()

tests/utils.py

@@ -65,6 +65,13 @@ class _asyncio_patch:
 def asyncio_patch(function, *args, **kwargs):
     return _asyncio_patch(function, *args, **kwargs)
 
+# monkey patch MagicMock,
+# FIXME: would probably be better to use asynctest or pytest.asyncio to test our asyncio code
+async def async_magic():
+    pass
+
+unittest.mock.MagicMock.__await__ = lambda x: async_magic().__await__()
+
 
 class AsyncioMagicMock(unittest.mock.MagicMock):
     """
@@ -100,18 +107,15 @@ class AsyncioMagicMock(unittest.mock.MagicMock):
 
 class AsyncioBytesIO(io.BytesIO):
     """
-    An async wrapper arround io.BytesIO to fake an
+    An async wrapper around io.BytesIO to fake an
     async network connection
     """
 
-    @asyncio.coroutine
-    def read(self, length=-1):
+    async def read(self, length=-1):
         return super().read(length)
 
-    @asyncio.coroutine
-    def write(self, data):
+    async def write(self, data):
         return super().write(data)
 
-    @asyncio.coroutine
-    def close(self):
+    async def close(self):
         return super().close()

tests/utils/asyncio/test_embed_shell.py

@@ -23,27 +23,26 @@ from gns3server.utils.asyncio.embed_shell import EmbedShell
 def test_embed_shell_help(async_run):
     class Application(EmbedShell):
 
-        @asyncio.coroutine
-        def hello(self):
+        async def hello(self):
             """
             The hello world function
 
             The hello usage
             """
-            pass
+            await asyncio.sleep(1)
+
     reader = asyncio.StreamReader()
     writer = asyncio.StreamReader()
     app = Application(reader, writer)
-    assert async_run(app._parse_command('help')) == 'Help:\nhello: The hello world function\nhelp: Show help\n\nhelp command for details about a command\n'
-    assert async_run(app._parse_command('?')) == 'Help:\nhello: The hello world function\nhelp: Show help\n\nhelp command for details about a command\n'
+    assert async_run(app._parse_command('help')) == 'Help:\nhello: The hello world function\n\nhelp command for details about a command\n'
+    assert async_run(app._parse_command('?')) == 'Help:\nhello: The hello world function\n\nhelp command for details about a command\n'
     assert async_run(app._parse_command('? hello')) == 'hello: The hello world function\n\nThe hello usage\n'
 
 
 def test_embed_shell_execute(async_run):
     class Application(EmbedShell):
 
-        @asyncio.coroutine
-        def hello(self):
+        async def hello(self):
             """
             The hello world function
 

tests/utils/test_asyncio.py

@@ -85,10 +85,9 @@ def test_lock_decorator(loop):
             self._test_val = 0
 
         @locking
-        @asyncio.coroutine
-        def method_to_lock(self):
+        async def method_to_lock(self):
             res = self._test_val
-            yield from asyncio.sleep(0.1)
+            await asyncio.sleep(0.1)
             self._test_val += 1
             return res