From 3259ec1220c122a8b340175c13db578c598e2366 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Thu, 15 Dec 2016 21:57:59 +0100 Subject: [PATCH] Close project if one one the compute of the project is down Fix #836 --- gns3server/controller/__init__.py | 15 ++++++++++----- gns3server/controller/compute.py | 19 +++++++++++++++---- gns3server/controller/node.py | 2 +- gns3server/controller/project.py | 2 +- gns3server/utils/asyncio/pool.py | 5 ++++- tests/controller/test_node.py | 2 +- 6 files changed, 32 insertions(+), 13 deletions(-) diff --git a/gns3server/controller/__init__.py b/gns3server/controller/__init__.py index 021315ef..07139a09 100644 --- a/gns3server/controller/__init__.py +++ b/gns3server/controller/__init__.py @@ -270,6 +270,15 @@ class Controller: self.notification.emit("compute.updated", self._computes[compute_id].__json__()) return self._computes[compute_id] + @asyncio.coroutine + def close_compute_projects(self, compute): + """ + Close projects running on a compute + """ + for project in self._projects.values(): + if compute in project.computes: + yield from project.close() + @asyncio.coroutine def delete_compute(self, compute_id): """ @@ -281,11 +290,7 @@ class Controller: compute = self.get_compute(compute_id) except aiohttp.web.HTTPNotFound: return - - for project in self._projects.values(): - if compute in project.computes: - yield from project.close() - + yield from self.close_compute_projects(compute) yield from compute.close() del self._computes[compute_id] self.save() diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index d6b6b7f6..e4032b02 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -107,6 +107,8 @@ class Compute: # Cache of interfaces on remote host self._interfaces_cache = None + self._connection_failure = 0 + def _session(self): if self._http_session is None or self._http_session.closed is True: self._http_session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=None, force_close=True)) @@ -344,14 +346,17 @@ class Compute: return StreamResponse(response) @asyncio.coroutine - def http_query(self, method, path, data=None, **kwargs): - if not self._connected: + def http_query(self, method, path, data=None, dont_connect=False, **kwargs): + """ + :param dont_connect: If true do not reconnect if not connected + """ + if not self._connected and not dont_connect: if self._id == "vm" and not self._controller.gns3vm.running: yield from self._controller.gns3vm.start() yield from self.connect() - if not self._connected: - raise aiohttp.web.HTTPConflict(text="Can't connect to {}".format(self._name)) + if not self._connected and not dont_connect: + raise ComputeError("Can't connect to {}".format(self._name)) response = yield from self._run_http_query(method, path, data=data, **kwargs) return response @@ -366,7 +371,12 @@ class Compute: except ComputeError: # Try to reconnect after 2 seconds if server unavailable only if not during tests (otherwise we create a ressources usage bomb) if not hasattr(sys, "_called_from_test") or not sys._called_from_test: + self._connection_failure += 1 + # After 5 failure we close the project using the compute to avoid sync issues + if self._connection_failure == 5: + yield from self._controller.close_compute_projects(self) asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self.connect())) + return except aiohttp.web.HTTPNotFound: raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server or it's a 1.X server".format(self._id)) @@ -383,6 +393,7 @@ class Compute: self._notifications = asyncio.gather(self._connect_notification()) self._connected = True + self._connection_failure = 0 self._controller.notification.emit("compute.updated", self.__json__()) @asyncio.coroutine diff --git a/gns3server/controller/node.py b/gns3server/controller/node.py index 02017819..bac82e76 100644 --- a/gns3server/controller/node.py +++ b/gns3server/controller/node.py @@ -425,7 +425,7 @@ class Node: Stop a node """ try: - yield from self.post("/stop", timeout=240) + yield from self.post("/stop", timeout=240, dont_connect=True) # We don't care if a node is down at this step except (ComputeError, aiohttp.errors.ClientHttpProcessingError, aiohttp.web.HTTPError): pass diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 9b1a7a68..dd599b3c 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -541,7 +541,7 @@ class Project: yield from self.stop_all() for compute in self._project_created_on_compute: try: - yield from compute.post("/projects/{}/close".format(self._id)) + yield from compute.post("/projects/{}/close".format(self._id), dont_connect=True) # We don't care if a compute is down at this step except (ComputeError, aiohttp.web.HTTPError, aiohttp.ClientResponseError, TimeoutError): pass diff --git a/gns3server/utils/asyncio/pool.py b/gns3server/utils/asyncio/pool.py index e50bfeab..eb03365f 100644 --- a/gns3server/utils/asyncio/pool.py +++ b/gns3server/utils/asyncio/pool.py @@ -36,6 +36,7 @@ class Pool(): Wait for all task to finish """ pending = set() + exceptions = set() while len(self._tasks) > 0 or len(pending) > 0: while len(self._tasks) > 0 and len(pending) < self._concurrency: task, args, kwargs = self._tasks.pop(0) @@ -43,7 +44,9 @@ class Pool(): (done, pending) = yield from asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) for task in done: if task.exception(): - raise task.exception() + exceptions.add(task.exception()) + if len(exceptions) > 0: + raise exceptions.pop() def main(): diff --git a/tests/controller/test_node.py b/tests/controller/test_node.py index 1536cf70..d241bb20 100644 --- a/tests/controller/test_node.py +++ b/tests/controller/test_node.py @@ -366,7 +366,7 @@ def test_stop(node, compute, project, async_run): compute.post = AsyncioMagicMock() async_run(node.stop()) - compute.post.assert_called_with("/projects/{}/vpcs/nodes/{}/stop".format(node.project.id, node.id), timeout=240) + compute.post.assert_called_with("/projects/{}/vpcs/nodes/{}/stop".format(node.project.id, node.id), timeout=240, dont_connect=True) def test_suspend(node, compute, project, async_run):