Catch docker namespace error

Fix #424

gns3server/modules/docker/__init__.py

@@ -30,7 +30,7 @@ log = logging.getLogger(__name__)
 from ..base_manager import BaseManager
 from ..project_manager import ProjectManager
 from .docker_vm import DockerVM
-from .docker_error import DockerError
+from .docker_error import *
 
 
 class Docker(BaseManager):
@@ -70,10 +70,14 @@ class Docker(BaseManager):
         :param data: Dictionnary with the body. Will be transformed to a JSON
         :param params: Parameters added as a query arg
         """
+
         response = yield from self.http_query(method, path, data=data, params=params)
         body = yield from response.read()
         if len(body):
-            body = json.loads(body.decode("utf-8"))
+            if response.headers['CONTENT-TYPE'] == 'application/json':
+                body = json.loads(body.decode("utf-8"))
+            else:
+                body = body.decode("utf-8")
         log.debug("Query Docker %s %s params=%s data=%s Response: %s", method, path, params, data, body)
         return body
 
@@ -105,7 +109,12 @@ class Docker(BaseManager):
             except ValueError:
                 pass
             log.debug("Query Docker %s %s params=%s data=%s Response: %s", method, path, params, data, body)
-            raise DockerError("Docker has returned an error: {} {}".format(response.status, body))
+            if response.status == 304:
+                raise DockerHttp304Error("Docker has returned an error: {} {}".format(response.status, body))
+            elif response.status == 404:
+                raise DockerHttp404Error("Docker has returned an error: {} {}".format(response.status, body))
+            else:
+                raise DockerError("Docker has returned an error: {} {}".format(response.status, body))
         return response
 
     @asyncio.coroutine

gns3server/modules/docker/docker_error.py

@@ -24,3 +24,11 @@ from ..vm_error import VMError
 
 class DockerError(VMError):
     pass
+
+
+class DockerHttp304Error(DockerError):
+    pass
+
+
+class DockerHttp404Error(DockerError):
+    pass

gns3server/modules/docker/docker_vm.py

@@ -27,12 +27,15 @@ import aiohttp
 import json
 
 from ...ubridge.hypervisor import Hypervisor
-from .docker_error import DockerError
+from .docker_error import *
 from ..base_vm import BaseVM
 from ..adapters.ethernet_adapter import EthernetAdapter
 from ..nios.nio_udp import NIOUDP
 from ...utils.asyncio.telnet_server import AsyncioTelnetServer
 
+from ...ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError
+
+
 import logging
 log = logging.getLogger(__name__)
 
@@ -165,11 +168,23 @@ class DockerVM(BaseVM):
         else:
             result = yield from self.manager.query("POST", "containers/{}/start".format(self._cid))
 
+            namespace = yield from self._get_namespace()
+
             yield from self._start_ubridge()
+
             for adapter_number in range(0, self.adapters):
                 nio = self._ethernet_adapters[adapter_number].get_nio(0)
                 with (yield from self.manager.ubridge_lock):
-                    yield from self._add_ubridge_connection(nio, adapter_number)
+                    try:
+                        yield from self._add_ubridge_connection(nio, adapter_number, namespace)
+                    except UbridgeNamespaceError:
+                        yield from self.stop()
+
+                        # The container can crash soon after the start this mean we can not move the interface to the container namespace
+                        logdata = yield from self._get_log()
+                        for line in logdata.split('\n'):
+                            log.error(line)
+                        raise DockerError(logdata)
 
             yield from self._start_console()
 
@@ -258,10 +273,15 @@ class DockerVM(BaseVM):
             if self._telnet_server:
                 self._telnet_server.close()
                 self._telnet_server = None
+
             # t=5 number of seconds to wait before killing the container
-            yield from self.manager.query("POST", "containers/{}/stop".format(self._cid), params={"t": 5})
-            log.info("Docker container '{name}' [{image}] stopped".format(
-                name=self._name, image=self._image))
+            try:
+                yield from self.manager.query("POST", "containers/{}/stop".format(self._cid), params={"t": 5})
+                log.info("Docker container '{name}' [{image}] stopped".format(
+                    name=self._name, image=self._image))
+            except DockerHttp304Error:
+                # Container is already stopped
+                pass
         # Ignore runtime error because when closing the server
         except RuntimeError as e:
             log.debug("Docker runtime error when closing: {}".format(str(e)))
@@ -334,12 +354,13 @@ class DockerVM(BaseVM):
         self._closed = True
 
     @asyncio.coroutine
-    def _add_ubridge_connection(self, nio, adapter_number):
+    def _add_ubridge_connection(self, nio, adapter_number, namespace):
         """
         Creates a connection in uBridge.
 
         :param nio: NIO instance or None if it's a dummu interface (if an interface is missing in ubridge you can't see it via ifconfig in the container)
         :param adapter_number: adapter number
+        :param namespace: Container namespace (pid)
         """
         try:
             adapter = self._ethernet_adapters[adapter_number]
@@ -362,11 +383,13 @@ class DockerVM(BaseVM):
             'docker create_veth {hostif} {guestif}'.format(
                 guestif=adapter.guest_ifc, hostif=adapter.host_ifc))
 
-        namespace = yield from self._get_namespace()
         log.debug("Move container %s adapter %s to namespace %s", self.name, adapter.guest_ifc, namespace)
-        yield from self._ubridge_hypervisor.send(
-            'docker move_to_ns {ifc} {ns} eth{adapter}'.format(
-                ifc=adapter.guest_ifc, ns=namespace, adapter=adapter_number))
+        try:
+            yield from self._ubridge_hypervisor.send(
+                'docker move_to_ns {ifc} {ns} eth{adapter}'.format(
+                    ifc=adapter.guest_ifc, ns=namespace, adapter=adapter_number))
+        except UbridgeError as e:
+            raise UbridgeNamespaceError(e)
 
         if isinstance(nio, NIOUDP):
             yield from self._ubridge_hypervisor.send(
@@ -587,3 +610,14 @@ class DockerVM(BaseVM):
         log.info("Docker VM '{name}' [{id}]: stopping packet capture on adapter {adapter_number}".format(name=self.name,
                                                                                                          id=self.id,
                                                                                                          adapter_number=adapter_number))
+
+    @asyncio.coroutine
+    def _get_log(self):
+        """
+        Return the log from the container
+
+        :returns: string
+        """
+
+        result = yield from self.manager.query("GET", "containers/{}/logs".format(self._cid), params={"stderr": 1, "stdout": 1})
+        return result

gns3server/ubridge/ubridge_error.py

@@ -24,3 +24,10 @@ class UbridgeError(Exception):
 
     def __init__(self, message):
         Exception.__init__(self, message)
+
+
+class UbridgeNamespaceError(Exception):
+    """
+    Raised if ubridge can not move a container to a namespace
+    """
+    pass

tests/modules/docker/test_docker.py

@@ -31,6 +31,7 @@ def test_query_success(loop):
     vm._connected = True
     response = MagicMock()
     response.status = 200
+    response.headers = {'CONTENT-TYPE': 'application/json'}
 
     @asyncio.coroutine
     def read():

tests/modules/docker/test_docker_vm.py

@@ -20,10 +20,12 @@ import uuid
 import asyncio
 from tests.utils import asyncio_patch
 
+from gns3server.ubridge.ubridge_error import UbridgeNamespaceError
 from gns3server.modules.docker.docker_vm import DockerVM
 from gns3server.modules.docker.docker_error import DockerError
 from gns3server.modules.docker import Docker
 
+
 from unittest.mock import patch, MagicMock, PropertyMock, call
 from gns3server.config import Config
 
@@ -240,17 +242,42 @@ def test_start(loop, vm, manager, free_console_port):
     with asyncio_patch("gns3server.modules.docker.DockerVM._get_container_state", return_value="stopped"):
         with asyncio_patch("gns3server.modules.docker.Docker.query") as mock_query:
             with asyncio_patch("gns3server.modules.docker.DockerVM._start_ubridge") as mock_start_ubridge:
-                with asyncio_patch("gns3server.modules.docker.DockerVM._add_ubridge_connection") as mock_add_ubridge_connection:
-                    with asyncio_patch("gns3server.modules.docker.DockerVM._start_console") as mock_start_console:
-                        loop.run_until_complete(asyncio.async(vm.start()))
+                with asyncio_patch("gns3server.modules.docker.DockerVM._get_namespace", return_value=42) as mock_namespace:
+                    with asyncio_patch("gns3server.modules.docker.DockerVM._add_ubridge_connection") as mock_add_ubridge_connection:
+                        with asyncio_patch("gns3server.modules.docker.DockerVM._start_console") as mock_start_console:
+                            loop.run_until_complete(asyncio.async(vm.start()))
 
     mock_query.assert_called_with("POST", "containers/e90e34656842/start")
-    mock_add_ubridge_connection.assert_called_once_with(nio, 0)
+    mock_add_ubridge_connection.assert_called_once_with(nio, 0, 42)
     assert mock_start_ubridge.called
     assert mock_start_console.called
     assert vm.status == "started"
 
 
+def test_start_namespace_failed(loop, vm, manager, free_console_port):
+
+    assert vm.status != "started"
+    vm.adapters = 1
+
+    nio = manager.create_nio(0, {"type": "nio_udp", "lport": free_console_port, "rport": free_console_port, "rhost": "127.0.0.1"})
+    loop.run_until_complete(asyncio.async(vm.adapter_add_nio_binding(0, nio)))
+
+    with asyncio_patch("gns3server.modules.docker.DockerVM._get_container_state", return_value="stopped"):
+        with asyncio_patch("gns3server.modules.docker.Docker.query") as mock_query:
+            with asyncio_patch("gns3server.modules.docker.DockerVM._start_ubridge") as mock_start_ubridge:
+                with asyncio_patch("gns3server.modules.docker.DockerVM._get_namespace", return_value=42) as mock_namespace:
+                    with asyncio_patch("gns3server.modules.docker.DockerVM._add_ubridge_connection", side_effect=UbridgeNamespaceError()) as mock_add_ubridge_connection:
+                        with asyncio_patch("gns3server.modules.docker.DockerVM._get_log", return_value='Hello not available') as mock_log:
+
+                            with pytest.raises(DockerError):
+                                loop.run_until_complete(asyncio.async(vm.start()))
+
+    mock_query.assert_any_call("POST", "containers/e90e34656842/start")
+    mock_add_ubridge_connection.assert_called_once_with(nio, 0, 42)
+    assert mock_start_ubridge.called
+    assert vm.status == "stopped"
+
+
 def test_start_without_nio(loop, vm, manager, free_console_port):
     """
     If no nio exists we will create one.
@@ -262,9 +289,10 @@ def test_start_without_nio(loop, vm, manager, free_console_port):
     with asyncio_patch("gns3server.modules.docker.DockerVM._get_container_state", return_value="stopped"):
         with asyncio_patch("gns3server.modules.docker.Docker.query") as mock_query:
             with asyncio_patch("gns3server.modules.docker.DockerVM._start_ubridge") as mock_start_ubridge:
-                with asyncio_patch("gns3server.modules.docker.DockerVM._add_ubridge_connection") as mock_add_ubridge_connection:
-                    with asyncio_patch("gns3server.modules.docker.DockerVM._start_console") as mock_start_console:
-                        loop.run_until_complete(asyncio.async(vm.start()))
+                with asyncio_patch("gns3server.modules.docker.DockerVM._get_namespace", return_value=42) as mock_namespace:
+                    with asyncio_patch("gns3server.modules.docker.DockerVM._add_ubridge_connection") as mock_add_ubridge_connection:
+                        with asyncio_patch("gns3server.modules.docker.DockerVM._start_console") as mock_start_console:
+                            loop.run_until_complete(asyncio.async(vm.start()))
 
     mock_query.assert_called_with("POST", "containers/e90e34656842/start")
     assert mock_add_ubridge_connection.called
@@ -401,8 +429,8 @@ def test_add_ubridge_connection(loop, vm):
     nio = vm.manager.create_nio(0, nio)
     nio.startPacketCapture("/tmp/capture.pcap")
     vm._ubridge_hypervisor = MagicMock()
-    with asyncio_patch("gns3server.modules.docker.DockerVM._get_namespace", return_value=42):
-        loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0)))
+
+    loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0, 42)))
 
     calls = [
         call.send("docker create_veth gns3-veth0ext gns3-veth0int"),
@@ -421,8 +449,8 @@ def test_add_ubridge_connection_none_nio(loop, vm):
 
     nio = None
     vm._ubridge_hypervisor = MagicMock()
-    with asyncio_patch("gns3server.modules.docker.DockerVM._get_namespace", return_value=42):
-        loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0)))
+
+    loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0, 42)))
 
     calls = [
         call.send("docker create_veth gns3-veth0ext gns3-veth0int"),
@@ -440,7 +468,7 @@ def test_add_ubridge_connection_invalid_adapter_number(loop, vm):
            "rhost": "127.0.0.1"}
     nio = vm.manager.create_nio(0, nio)
     with pytest.raises(DockerError):
-        loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 12)))
+        loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 12, 42)))
 
 
 def test_add_ubridge_connection_no_free_interface(loop, vm):
@@ -456,7 +484,7 @@ def test_add_ubridge_connection_no_free_interface(loop, vm):
         interfaces = ["gns3-veth{}ext".format(index) for index in range(128)]
 
         with patch("psutil.net_if_addrs", return_value=interfaces):
-            loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0)))
+            loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0, 42)))
 
 
 def test_delete_ubridge_connection(loop, vm):
@@ -467,8 +495,8 @@ def test_delete_ubridge_connection(loop, vm):
            "rport": 4343,
            "rhost": "127.0.0.1"}
     nio = vm.manager.create_nio(0, nio)
-    with asyncio_patch("gns3server.modules.docker.DockerVM._get_namespace", return_value=42):
-        loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0)))
+
+    loop.run_until_complete(asyncio.async(vm._add_ubridge_connection(nio, 0, 42)))
     loop.run_until_complete(asyncio.async(vm._delete_ubridge_connection(0)))
 
     calls = [
@@ -561,3 +589,16 @@ def test_stop_capture(vm, tmpdir, manager, free_console_port, loop):
     assert vm._ethernet_adapters[0].get_nio(0).capturing
     loop.run_until_complete(asyncio.async(vm.stop_capture(0)))
     assert vm._ethernet_adapters[0].get_nio(0).capturing is False
+
+
+def test_get_log(loop, vm):
+    @asyncio.coroutine
+    def read():
+        return b'Hello\nWorld'
+
+    mock_query = MagicMock()
+    mock_query.read = read
+
+    with asyncio_patch("gns3server.modules.docker.Docker.http_query", return_value=mock_query) as mock:
+        images = loop.run_until_complete(asyncio.async(vm._get_log()))
+        mock.assert_called_with("GET", "containers/e90e34656842/logs", params={"stderr": 1, "stdout": 1}, data={})