The server has now 2 notification streams

* A new one for controller related events (compute, appliance templates etc.)
* The existing one for project related events (links, nodes etc.)

gns3server/controller/__init__.py

@@ -429,7 +429,7 @@ class Controller:
         self._settings["modification_uuid"] = str(uuid.uuid4())  # We add a modification id to the settings to help the gui to detect changes
         self.save()
         self.load_appliances()
-        self.notification.emit("settings.updated", val)
+        self.notification.controller_emit("settings.updated", val)
 
     @asyncio.coroutine
     def add_compute(self, compute_id=None, name=None, force=False, connect=True, **kwargs):
@@ -463,12 +463,12 @@ class Controller:
             self.save()
             if connect:
                 yield from compute.connect()
-            self.notification.emit("compute.created", compute.__json__())
+            self.notification.controller_emit("compute.created", compute.__json__())
             return compute
         else:
             if connect:
                 yield from self._computes[compute_id].connect()
-            self.notification.emit("compute.updated", self._computes[compute_id].__json__())
+            self.notification.controller_emit("compute.updated", self._computes[compute_id].__json__())
             return self._computes[compute_id]
 
     @asyncio.coroutine
@@ -509,7 +509,7 @@ class Controller:
         yield from compute.close()
         del self._computes[compute_id]
         self.save()
-        self.notification.emit("compute.deleted", compute.__json__())
+        self.notification.controller_emit("compute.deleted", compute.__json__())
 
     @property
     def notification(self):

gns3server/controller/compute.py

@@ -158,7 +158,7 @@ class Compute:
         if self._http_session:
             self._http_session.close()
         self._connected = False
-        self._controller.notification.emit("compute.updated", self.__json__())
+        self._controller.notification.controller_emit("compute.updated", self.__json__())
         self._controller.save()
 
     @asyncio.coroutine
@@ -442,7 +442,7 @@ class Compute:
             self._notifications = asyncio.gather(self._connect_notification())
             self._connected = True
             self._connection_failure = 0
-            self._controller.notification.emit("compute.updated", self.__json__())
+            self._controller.notification.controller_emit("compute.updated", self.__json__())
 
     @asyncio.coroutine
     def _connect_notification(self):
@@ -468,7 +468,7 @@ class Compute:
             if action == "ping":
                 self._cpu_usage_percent = event["cpu_usage_percent"]
                 self._memory_usage_percent = event["memory_usage_percent"]
-                self._controller.notification.emit("compute.updated", self.__json__())
+                self._controller.notification.controller_emit("compute.updated", self.__json__())
             else:
                 yield from self._controller.notification.dispatch(action, event, compute_id=self.id)
         if self._ws:
@@ -480,7 +480,7 @@ class Compute:
         self._ws = None
         self._cpu_usage_percent = None
         self._memory_usage_percent = None
-        self._controller.notification.emit("compute.updated", self.__json__())
+        self._controller.notification.controller_emit("compute.updated", self.__json__())
 
     def _getUrl(self, path):
         host = self._host

gns3server/controller/drawing.py

@@ -186,7 +186,7 @@ class Drawing:
         data = self.__json__()
         if not svg_changed:
             del data["svg"]
-        self._project.controller.notification.emit("drawing.updated", data)
+        self._project.controller.notification.project_emit("drawing.updated", data)
         self._project.dump()
 
     def __json__(self, topology_dump=False):

gns3server/controller/export_project.py

@@ -72,7 +72,7 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id
             except OSError as e:
                 msg = "Could not export file {}: {}".format(path, e)
                 log.warning(msg)
-                project.controller.notification.emit("log.warning", {"message": msg})
+                project.controller.notification.project_emit("log.warning", {"message": msg})
                 continue
             # ignore the .gns3 file
             if file.endswith(".gns3"):

gns3server/controller/gns3vm/__init__.py

@@ -367,7 +367,7 @@ class GNS3VM:
                             if vm_network.compare_networks(compute_network) != 0:
                                 msg = "The GNS3 VM ({}) is not on the same network as the {} server ({}), please make sure the local server binding is in the same network as the GNS3 VM".format(
                                     vm_network, compute_id, compute_network)
-                                self._controller.notification.emit("log.warning", {"message": msg})
+                                self._controller.notification.controller_emit("log.warning", {"message": msg})
         except ComputeError as e:
             log.warning("Could not check the VM is in the same subnet as the local server: {}".format(e))
 

gns3server/controller/link.py

@@ -174,7 +174,7 @@ class Link:
             self._filters = new_filters
             if self._created:
                 yield from self.update()
-                self._project.controller.notification.emit("link.updated", self.__json__())
+                self._project.controller.notification.project_emit("link.updated", self.__json__())
                 self._project.dump()
 
     @asyncio.coroutine
@@ -182,7 +182,7 @@ class Link:
         if value != self._suspended:
             self._suspended = value
             yield from self.update()
-            self._project.controller.notification.emit("link.updated", self.__json__())
+            self._project.controller.notification.project_emit("link.updated", self.__json__())
             self._project.dump()
 
     @property
@@ -214,14 +214,14 @@ class Link:
 
             if node.node_type in ["nat", "cloud"]:
                 if other_node["node"].node_type in ["nat", "cloud"]:
-                    raise aiohttp.web.HTTPConflict(text="It's not allowed to connect a {} to a {}".format(other_node["node"].node_type, node.node_type))
+                    raise aiohttp.web.HTTPConflict(text="Connecting a {} to a {} is not allowed".format(other_node["node"].node_type, node.node_type))
 
             # Check if user is not connecting serial => ethernet
             other_port = other_node["node"].get_port(other_node["adapter_number"], other_node["port_number"])
             if other_port is None:
                 raise aiohttp.web.HTTPNotFound(text="Port {}/{} for {} not found".format(other_node["adapter_number"], other_node["port_number"], other_node["node"].name))
             if port.link_type != other_port.link_type:
-                raise aiohttp.web.HTTPConflict(text="It's not allowed to connect a {} to a {}".format(other_port.link_type, port.link_type))
+                raise aiohttp.web.HTTPConflict(text="Connecting a {} interface to a {} interface is not allowed".format(other_port.link_type, port.link_type))
 
         if label is None:
             label = {
@@ -246,7 +246,7 @@ class Link:
                 n["node"].add_link(self)
                 n["port"].link = self
             self._created = True
-            self._project.controller.notification.emit("link.created", self.__json__())
+            self._project.controller.notification.project_emit("link.created", self.__json__())
 
         if dump:
             self._project.dump()
@@ -260,7 +260,7 @@ class Link:
                     label = node_data.get("label")
                     if label:
                         port["label"] = label
-        self._project.controller.notification.emit("link.updated", self.__json__())
+        self._project.controller.notification.project_emit("link.updated", self.__json__())
         self._project.dump()
 
     @asyncio.coroutine
@@ -300,7 +300,7 @@ class Link:
         self._capturing = True
         self._capture_file_name = capture_file_name
         self._streaming_pcap = asyncio_ensure_future(self._start_streaming_pcap())
-        self._project.controller.notification.emit("link.updated", self.__json__())
+        self._project.controller.notification.project_emit("link.updated", self.__json__())
 
     @asyncio.coroutine
     def _start_streaming_pcap(self):
@@ -320,8 +320,8 @@ class Link:
             error_msg = "Could not stream PCAP file: error {}: {}".format(e.status, e.text)
             log.error(error_msg)
             self._capturing = False
-            self._project.notification.emit("log.error", {"message": error_msg})
-            self._project.controller.notification.emit("link.updated", self.__json__())
+            self._project.notification.project_emit("log.error", {"message": error_msg})
+            self._project.controller.notification.project_emit("link.updated", self.__json__())
 
         with stream_content as stream:
             try:
@@ -345,7 +345,7 @@ class Link:
         """
 
         self._capturing = False
-        self._project.controller.notification.emit("link.updated", self.__json__())
+        self._project.controller.notification.project_emit("link.updated", self.__json__())
 
     @asyncio.coroutine
     def _read_pcap_from_source(self):

gns3server/controller/node.py

@@ -407,7 +407,7 @@ class Node:
             yield from self.parse_node_response(response.json)
         elif old_json != self.__json__():
             # We send notif only if object has changed
-            self.project.controller.notification.emit("node.updated", self.__json__())
+            self.project.controller.notification.project_emit("node.updated", self.__json__())
         self.project.dump()
 
     @asyncio.coroutine
@@ -575,13 +575,13 @@ class Node:
         for directory in images_directories(type):
             image = os.path.join(directory, img)
             if os.path.exists(image):
-                self.project.controller.notification.emit("log.info", {"message": "Uploading missing image {}".format(img)})
+                self.project.controller.notification.project_emit("log.info", {"message": "Uploading missing image {}".format(img)})
                 try:
                     with open(image, 'rb') as f:
                         yield from self._compute.post("/{}/images/{}".format(self._node_type, os.path.basename(img)), data=f, timeout=None)
                 except OSError as e:
                     raise aiohttp.web.HTTPConflict(text="Can't upload {}: {}".format(image, str(e)))
-                self.project.controller.notification.emit("log.info", {"message": "Upload finished for {}".format(img)})
+                self.project.controller.notification.project_emit("log.info", {"message": "Upload finished for {}".format(img)})
                 return True
         return False
 

gns3server/controller/notification.py

@@ -30,27 +30,63 @@ class Notification:
 
     def __init__(self, controller):
         self._controller = controller
-        self._listeners = {}
+        self._project_listeners = {}
+        self._controller_listeners = []
 
     @contextmanager
-    def queue(self, project):
+    def project_queue(self, project):
         """
         Get a queue of notifications
 
         Use it with Python with
         """
         queue = NotificationQueue()
-        self._listeners.setdefault(project.id, set())
-        self._listeners[project.id].add(queue)
+        self._project_listeners.setdefault(project.id, set())
+        self._project_listeners[project.id].add(queue)
         yield queue
-        self._listeners[project.id].remove(queue)
+        self._project_listeners[project.id].remove(queue)
+
+    @contextmanager
+    def controller_queue(self):
+        """
+        Get a queue of notifications
+
+        Use it with Python with
+        """
+        queue = NotificationQueue()
+        self._controller_listeners.append(queue)
+        yield queue
+        self._controller_listeners.remove(queue)
+
+    def controller_emit(self, action, event):
+        """
+        Send a notification to clients connected to the controller stream
+
+        :param action: Action name
+        :param event: Event to send
+        """
+
+        # If use in tests for documentation we save a sample
+        if os.environ.get("PYTEST_BUILD_DOCUMENTATION") == "1":
+            os.makedirs("docs/api/notifications", exist_ok=True)
+            try:
+                import json
+                data = json.dumps(event, indent=4, sort_keys=True)
+                if "MagicMock" not in data:
+                    with open(os.path.join("docs/api/notifications", action + ".json"), 'w+') as f:
+                        f.write(data)
+            except TypeError:  # If we receive a mock as an event it will raise TypeError when using json dump
+                pass
+
+        for controller_listener in self._controller_listeners:
+            controller_listener.put_nowait((action, event, {}))
 
     def project_has_listeners(self, project):
         """
         :param project_id: Project object
         :returns: True if client listen this project
         """
-        return project.id in self._listeners and len(self._listeners[project.id]) > 0
+        return project.id in self._project_listeners and len(self._project_listeners[project.id]) > 0
 
     @asyncio.coroutine
     def dispatch(self, action, event, compute_id):
@@ -69,16 +105,16 @@ class Notification:
                 node = project.get_node(event["node_id"])
                 yield from node.parse_node_response(event)
 
-                self.emit("node.updated", node.__json__())
+                self.project_emit("node.updated", node.__json__())
             except (aiohttp.web.HTTPNotFound, aiohttp.web.HTTPForbidden):  # Project closing
                 return
         elif action == "ping":
             event["compute_id"] = compute_id
-            self.emit(action, event)
+            self.project_emit(action, event)
         else:
-            self.emit(action, event)
+            self.project_emit(action, event)
 
-    def emit(self, action, event):
+    def project_emit(self, action, event):
         """
         Send a notification to clients scoped by projects
 
@@ -101,7 +137,7 @@ class Notification:
         if "project_id" in event:
             self._send_event_to_project(event["project_id"], action, event)
         else:
-            self._send_event_to_all(action, event)
+            self._send_event_to_all_projects(action, event)
 
     def _send_event_to_project(self, project_id, action, event):
         """
@@ -113,13 +149,13 @@ class Notification:
         :param event: Event to send
         """
         try:
-            project_listeners = self._listeners[project_id]
+            project_listeners = self._project_listeners[project_id]
         except KeyError:
             return
         for listener in project_listeners:
             listener.put_nowait((action, event, {}))
 
-    def _send_event_to_all(self, action, event):
+    def _send_event_to_all_projects(self, action, event):
         """
         Send an event to all the client listening for notifications on all
         projects
@@ -127,6 +163,6 @@ class Notification:
         :param action: Action name
         :param event: Event to send
         """
-        for project_listeners in self._listeners.values():
+        for project_listeners in self._project_listeners.values():
             for listener in project_listeners:
                 listener.put_nowait((action, event, {}))

gns3server/controller/project.py

@@ -135,7 +135,7 @@ class Project:
 
         # We send notif only if object has changed
         if old_json != self.__json__():
-            self.controller.notification.emit("project.updated", self.__json__())
+            self.controller.notification.project_emit("project.updated", self.__json__())
             self.dump()
 
             # update on computes
@@ -519,7 +519,7 @@ class Project:
             self._project_created_on_compute.add(compute)
         yield from node.create()
         self._nodes[node.id] = node
-        self.controller.notification.emit("node.created", node.__json__())
+        self.controller.notification.project_emit("node.created", node.__json__())
         if dump:
             self.dump()
         return node
@@ -545,7 +545,7 @@ class Project:
         del self._nodes[node.id]
         yield from node.destroy()
         self.dump()
-        self.controller.notification.emit("node.deleted", node.__json__())
+        self.controller.notification.project_emit("node.deleted", node.__json__())
 
     @open_required
     def get_node(self, node_id):
@@ -611,7 +611,7 @@ class Project:
         if drawing_id not in self._drawings:
             drawing = Drawing(self, drawing_id=drawing_id, **kwargs)
             self._drawings[drawing.id] = drawing
-            self.controller.notification.emit("drawing.created", drawing.__json__())
+            self.controller.notification.project_emit("drawing.created", drawing.__json__())
             if dump:
                 self.dump()
             return drawing
@@ -633,7 +633,7 @@ class Project:
         drawing = self.get_drawing(drawing_id)
         del self._drawings[drawing.id]
         self.dump()
-        self.controller.notification.emit("drawing.deleted", drawing.__json__())
+        self.controller.notification.project_emit("drawing.deleted", drawing.__json__())
 
     @open_required
     @asyncio.coroutine
@@ -662,7 +662,7 @@ class Project:
             if force_delete is False:
                 raise
         self.dump()
-        self.controller.notification.emit("link.deleted", link.__json__())
+        self.controller.notification.project_emit("link.deleted", link.__json__())
 
     @open_required
     def get_link(self, link_id):
@@ -737,7 +737,7 @@ class Project:
         self._cleanPictures()
         self._status = "closed"
         if not ignore_notification:
-            self.controller.notification.emit("project.closed", self.__json__())
+            self.controller.notification.project_emit("project.closed", self.__json__())
         self.reset()
 
     def _cleanPictures(self):

gns3server/controller/snapshot.py

@@ -124,7 +124,7 @@ class Snapshot:
         except (OSError, PermissionError) as e:
             raise aiohttp.web.HTTPConflict(text=str(e))
         yield from project.open()
-        self._project.controller.notification.emit("snapshot.restored", self.__json__())
+        self._project.controller.notification.project_emit("snapshot.restored", self.__json__())
         return self._project
 
     def __json__(self):

gns3server/handlers/api/compute/server_handler.py

@@ -15,7 +15,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import psutil
 import platform
 
@@ -24,7 +23,6 @@ from gns3server.config import Config
 from gns3server.schemas.version import VERSION_SCHEMA
 from gns3server.compute.port_manager import PortManager
 from gns3server.version import __version__
-from aiohttp.web import HTTPConflict
 
 
 class ServerHandler:
@@ -39,12 +37,11 @@ class ServerHandler:
         local_server = config.get_section_config("Server").getboolean("local", False)
         response.json({"version": __version__, "local": local_server})
 
-
     @Route.get(
         r"/debug",
-        description="Return debug informations about the compute",
+        description="Return debug information about the compute",
         status_codes={
-            201: "Writed"
+            201: "Written"
         })
     def debug(request, response):
         response.content_type = "text/plain"

gns3server/handlers/api/controller/__init__.py

@@ -25,3 +25,4 @@ from .symbol_handler import SymbolHandler
 from .snapshot_handler import SnapshotHandler
 from .appliance_handler import ApplianceHandler
 from .gns3_vm_handler import GNS3VMHandler
+from .notification_handler import NotificationHandler

gns3server/handlers/api/controller/notification_handler.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+import aiohttp
+from aiohttp.web import WebSocketResponse
+from gns3server.web.route import Route
+from gns3server.controller import Controller
+from gns3server.utils.asyncio import asyncio_ensure_future
+
+
+@asyncio.coroutine
+def process_websocket(ws):
+    """
+    Process ping / pong and close message
+    """
+    try:
+        yield from ws.receive()
+    except aiohttp.WSServerHandshakeError:
+        pass
+
+
+class NotificationHandler:
+
+    @Route.get(
+        r"/notifications",
+        description="Receive notifications about the controller",
+        status_codes={
+            200: "End of stream"
+        })
+    def notification(request, response):
+
+        controller = Controller.instance()
+        response.content_type = "application/json"
+        response.set_status(200)
+        response.enable_chunked_encoding()
+
+        yield from response.prepare(request)
+        with controller.notification.controller_queue() as queue:
+            while True:
+                try:
+                    msg = yield from queue.get_json(5)
+                    response.write(("{}\n".format(msg)).encode("utf-8"))
+                except asyncio.futures.CancelledError:
+                    break
+                yield from response.drain()
+
+    @Route.get(
+        r"/notifications/ws",
+        description="Receive notifications about controller from a Websocket",
+        status_codes={
+            200: "End of stream"
+        })
+    def notification_ws(request, response):
+
+        controller = Controller.instance()
+        ws = aiohttp.web.WebSocketResponse()
+        yield from ws.prepare(request)
+
+        asyncio_ensure_future(process_websocket(ws))
+        with controller.notification.controler_queue() as queue:
+            while True:
+                try:
+                    notification = yield from queue.get_json(5)
+                except asyncio.futures.CancelledError:
+                    break
+                if ws.closed:
+                    break
+                ws.send_str(notification)
+        return ws

gns3server/handlers/api/controller/project_handler.py

@@ -227,7 +227,7 @@ class ProjectHandler:
         response.enable_chunked_encoding()
 
         yield from response.prepare(request)
-        with controller.notification.queue(project) as queue:
+        with controller.notification.project_queue(project) as queue:
             while True:
                 try:
                     msg = yield from queue.get_json(5)
@@ -263,7 +263,7 @@ class ProjectHandler:
 
         asyncio_ensure_future(process_websocket(ws))
 
-        with controller.notification.queue(project) as queue:
+        with controller.notification.project_queue(project) as queue:
             while True:
                 try:
                     notification = yield from queue.get_json(5)

gns3server/handlers/api/controller/server_handler.py

@@ -109,6 +109,7 @@ class ServerHandler:
             # The init of the server could take some times
             # we ensure settings are loaded before returning them
             settings = Controller.instance().settings
+
             if settings is not None:
                 break
             yield from asyncio.sleep(0.5)
@@ -116,17 +117,16 @@ class ServerHandler:
 
     @Route.post(
         r"/settings",
-        description="Write gui settings on the server. Temporary will we removed in later releas",
+        description="Write gui settings on the server. Temporary will we removed in later releases",
         status_codes={
-            201: "Writed"
+            201: "Settings saved"
         })
     def write_settings(request, response):
         controller = Controller.instance()
         if controller.settings is None:  # Server is not loaded ignore settings update to prevent buggy client sync issue
             return
-        controller.settings = request.json
         try:
-            controller.save()
+            controller.settings = request.json
         except (OSError, PermissionError) as e:
             raise HTTPConflict(text="Can't save the settings {}".format(str(e)))
         response.json(controller.settings)

gns3server/version.py

@@ -23,7 +23,7 @@
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
 
-__version__ = "2.2.0dev2"
+__version__ = "2.2.0dev3"
 __version_info__ = (2, 2, 0, 99)
 
 # If it's a git checkout try to add the commit