diff --git a/.travis.yml b/.travis.yml index 7567504d..b27dd814 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: sudo: false cache: pip install: +- pip install -U setuptools pip - python setup.py install - pip install -rdev-requirements.txt script: diff --git a/CHANGELOG b/CHANGELOG index 477dd91a..8ea402eb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,18 @@ # Change Log +## 2.0.2 30/05/2017 + +* Set correct permission on ubridge when doing a remote installation +* Remote install script should be totally non interactive +* Duplicate project on remote server use UUID +* Fix import of some old topologies from 1.3 +* Fix error in logging of error during starting GNS3 VM +* Fix an error when logging Docker container fail to start +* Use docker version in error message of outdated docker installation +* Support images created by "docker commit". Fixes #1039 +* Do not wait auto start to finish in order to complete project opening +* Improve logging for remote server connection lost + ## 2.0.1 16/05/2017 * Handle HTTP 504 errors from compute node diff --git a/dev-requirements.txt b/dev-requirements.txt index 0d474df5..51e2d6b5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ -rrequirements.txt -sphinx==1.5.6 -pytest==3.0.7 +sphinx==1.6.2 +pytest==3.1.1 pep8==1.7.0 pytest-catchlog==1.2.2 pytest-timeout==1.2.0 diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index 030c5db2..7e3ccd69 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -33,7 +33,9 @@ from gns3server.compute.docker.docker_error import DockerError, DockerHttp304Err log = logging.getLogger(__name__) +# Be carefull to keep it consistent DOCKER_MINIMUM_API_VERSION = "1.25" +DOCKER_MINIMUM_VERSION = "1.13" class Docker(BaseManager): @@ -60,7 +62,7 @@ class Docker(BaseManager): self._connected = False raise DockerError("Can't connect to docker daemon") if parse_version(version["ApiVersion"]) < parse_version(DOCKER_MINIMUM_API_VERSION): - raise DockerError("Docker API version is {}. GNS3 requires a minimum API version of {}".format(version["ApiVersion"], DOCKER_MINIMUM_API_VERSION)) + raise DockerError("Docker version is {}. GNS3 requires a minimum version of {}".format(version["Version"], DOCKER_MINIMUM_VERSION)) def connector(self): if self._connector is None or self._connector.closed: @@ -113,11 +115,13 @@ class Docker(BaseManager): :returns: HTTP response """ data = json.dumps(data) - url = "http://docker/v" + DOCKER_MINIMUM_API_VERSION + "/" + path - if timeout is None: timeout = 60 * 60 * 24 * 31 # One month timeout + if path == 'version': + url = "http://docker/v1.12/" + path # API of docker v1.0 + else: + url = "http://docker/v" + DOCKER_MINIMUM_API_VERSION + "/" + path try: if path != "version": # version is use by check connection yield from self._check_connection() @@ -162,10 +166,9 @@ class Docker(BaseManager): """ url = "http://docker/v" + DOCKER_MINIMUM_API_VERSION + "/" + path - connection = yield from aiohttp.ws_connect(url, - connector=self.connector(), - origin="http://docker", - autoping=True) + connection = yield from self._session.ws_connect(url, + origin="http://docker", + autoping=True) return connection @locked_coroutine diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index ee11a507..00b80b0a 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -213,7 +213,7 @@ class DockerVM(BaseNode): self._volumes = ["/etc/network"] - volumes = image_infos.get("ContainerConfig", {}).get("Volumes") + volumes = image_infos.get("Config", {}).get("Volumes") if volumes is None: return binds for volume in volumes.keys(): @@ -361,7 +361,7 @@ class DockerVM(BaseNode): try: yield from self._add_ubridge_connection(nio, adapter_number) except UbridgeNamespaceError: - log.error("Container {} failed to start", self.name) + log.error("Container %s failed to start", self.name) yield from self.stop() # The container can crash soon after the start, this means we can not move the interface to the container namespace diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index f3916266..02112234 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -580,8 +580,10 @@ class Compute: Forward a call to the emulator on compute """ try: - res = yield from self.http_query(method, "/{}/{}".format(type, path), data=data, timeout=None) + action = "/{}/{}".format(type, path) + res = yield from self.http_query(method, action, data=data, timeout=None) except aiohttp.ServerDisconnectedError: + log.error("Connection lost to %s during %s %s", self._id, method, action) raise aiohttp.web.HTTPGatewayTimeout() return res.json diff --git a/gns3server/controller/gns3vm/__init__.py b/gns3server/controller/gns3vm/__init__.py index bace3022..a24ae702 100644 --- a/gns3server/controller/gns3vm/__init__.py +++ b/gns3server/controller/gns3vm/__init__.py @@ -250,7 +250,7 @@ class GNS3VM: force=True) except aiohttp.web.HTTPConflict: pass - log.error("Can't start the GNS3 VM: {}", str(e)) + log.error("Can't start the GNS3 VM: %s", str(e)) @asyncio.coroutine def exit_vm(self): @@ -287,7 +287,7 @@ class GNS3VM: yield from engine.start() except Exception as e: yield from self._controller.delete_compute("vm") - log.error("Can't start the GNS3 VM: {}", str(e)) + log.error("Can't start the GNS3 VM: {}".format(str(e))) yield from compute.update(name="GNS3 VM ({})".format(engine.vmname)) raise e yield from compute.update(name="GNS3 VM ({})".format(engine.vmname), diff --git a/gns3server/controller/import_project.py b/gns3server/controller/import_project.py index 0bdccc1d..d3cb6f00 100644 --- a/gns3server/controller/import_project.py +++ b/gns3server/controller/import_project.py @@ -24,7 +24,6 @@ import asyncio import zipfile import aiohttp -from ..config import Config from .topology import load_topology @@ -74,7 +73,7 @@ def import_project(controller, project_id, stream, location=None, name=None, kee path = location else: projects_path = controller.projects_directory() - path = os.path.join(projects_path, project_name) + path = os.path.join(projects_path, project_id) try: os.makedirs(path, exist_ok=True) except UnicodeEncodeError as e: diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index fc5161e8..d28bee97 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -699,7 +699,10 @@ class Project: self._loading = False # Should we start the nodes when project is open if self._auto_start: - yield from self.start_all() + # Start all in the background without waiting for completion + # we ignore errors because we want to let the user open + # their project and fix it + asyncio.async(self.start_all()) @asyncio.coroutine def wait_loaded(self): diff --git a/gns3server/controller/topology.py b/gns3server/controller/topology.py index 01a40b90..fc75795a 100644 --- a/gns3server/controller/topology.py +++ b/gns3server/controller/topology.py @@ -321,6 +321,10 @@ def _convert_1_3_later(topo, topo_path): node["properties"] = {} + # Some old dynamips node don't have type + if "type" not in old_node: + old_node["type"] = old_node["properties"]["platform"].upper() + if old_node["type"] == "VPCSDevice": node["node_type"] = "vpcs" elif old_node["type"] == "QemuVM": @@ -348,7 +352,7 @@ def _convert_1_3_later(topo, topo_path): node["symbol"] = ":/symbols/ethernet_switch.svg" node["console_type"] = None node["properties"]["ports_mapping"] = [] - for port in old_node["ports"]: + for port in old_node.get("ports", []): node["properties"]["ports_mapping"].append({ "name": "Ethernet{}".format(port["port_number"] - 1), "port_number": port["port_number"] - 1, @@ -359,12 +363,12 @@ def _convert_1_3_later(topo, topo_path): node["node_type"] = "frame_relay_switch" node["symbol"] = ":/symbols/frame_relay_switch.svg" node["console_type"] = None - elif old_node["type"] in ["C1700", "C2600", "C2691", "C3600", "C3620", "C3640", "C3660", "C3725", "C3745", "C7200", "EtherSwitchRouter"]: + elif old_node["type"].upper() in ["C1700", "C2600", "C2691", "C3600", "C3620", "C3640", "C3660", "C3725", "C3745", "C7200", "EtherSwitchRouter"]: if node["symbol"] is None: node["symbol"] = ":/symbols/router.svg" node["node_type"] = "dynamips" node["properties"]["dynamips_id"] = old_node.get("dynamips_id") - if "platform" not in node["properties"] and old_node["type"].startswith("C"): + if "platform" not in node["properties"] and old_node["type"].upper().startswith("C"): node["properties"]["platform"] = old_node["type"].lower() if node["properties"]["platform"].startswith("c36"): node["properties"]["platform"] = "c3600" diff --git a/gns3server/crash_report.py b/gns3server/crash_report.py index abae1235..af8f9339 100644 --- a/gns3server/crash_report.py +++ b/gns3server/crash_report.py @@ -57,7 +57,7 @@ class CrashReport: Report crash to a third party service """ - DSN = "sync+https://9b1156a90ee943eba20e032cf007297d:024b75c7b11844a58df147a4fb059774@sentry.io/38482" + DSN = "sync+https://67b93949a78d4ef5978388cc4b8906f9:271ee1dd01db4a39b919097f452cb6c5@sentry.io/38482" if hasattr(sys, "frozen"): cacert = get_resource("cacert.pem") if cacert is not None and os.path.isfile(cacert): diff --git a/gns3server/handlers/api/controller/project_handler.py b/gns3server/handlers/api/controller/project_handler.py index 9f703928..cc9a8300 100644 --- a/gns3server/handlers/api/controller/project_handler.py +++ b/gns3server/handlers/api/controller/project_handler.py @@ -22,7 +22,6 @@ import tempfile from gns3server.web.route import Route from gns3server.controller import Controller -from gns3server.controller.project import Project from gns3server.controller.import_project import import_project from gns3server.controller.export_project import export_project from gns3server.config import Config diff --git a/gns3server/schemas/dynamips_vm.py b/gns3server/schemas/dynamips_vm.py index f1a66ba5..43b7da4f 100644 --- a/gns3server/schemas/dynamips_vm.py +++ b/gns3server/schemas/dynamips_vm.py @@ -33,7 +33,7 @@ VM_CREATE_SCHEMA = { }, "dynamips_id": { "description": "Dynamips ID", - "type": "integer" + "type": ["integer", "null"] }, "name": { "description": "Dynamips VM instance name", diff --git a/gns3server/web/response.py b/gns3server/web/response.py index c394d305..13c97742 100644 --- a/gns3server/web/response.py +++ b/gns3server/web/response.py @@ -46,7 +46,8 @@ class Response(aiohttp.web.Response): def enable_chunked_encoding(self): # Very important: do not send a content length otherwise QT closes the connection (curl can consume the feed) - self.content_length = None + if self.content_length: + self.content_length = None super().enable_chunked_encoding() @asyncio.coroutine diff --git a/gns3server/web/web_server.py b/gns3server/web/web_server.py index 0bcc71a5..be24f2ab 100644 --- a/gns3server/web/web_server.py +++ b/gns3server/web/web_server.py @@ -43,6 +43,9 @@ import gns3server.handlers import logging log = logging.getLogger(__name__) +if not aiohttp.__version__.startswith("2.0"): + raise RuntimeError("You need aiohttp 2.0 for running GNS3") + class WebServer: diff --git a/scripts/remote-install.sh b/scripts/remote-install.sh index 260c8d8d..8e9c741e 100644 --- a/scripts/remote-install.sh +++ b/scripts/remote-install.sh @@ -152,7 +152,7 @@ log "Update system packages" apt-get update log "Upgrade packages" -apt-get upgrade -y +apt-get upgrade --yes --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" log " Install GNS3 packages" apt-get install -y gns3-server @@ -163,6 +163,10 @@ then useradd -d /opt/gns3/ -m gns3 fi + +log "Add GNS3 to the ubridge group" +usermod -aG ubridge gns3 + log "Install docker" if [ ! -f "/usr/bin/docker" ] then diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 3d28f11a..6ca6fd14 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -840,7 +840,7 @@ def test_get_image_informations(project, manager, loop): @pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supported on Windows") def test_mount_binds(vm, tmpdir): image_infos = { - "ContainerConfig": { + "Config": { "Volumes": { "/test/experimental": {} } diff --git a/tests/conftest.py b/tests/conftest.py index 2ecd7764..3308e5b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - +import gc import pytest import socket import asyncio diff --git a/tests/controller/test_export_project.py b/tests/controller/test_export_project.py index f48df7c8..81586dc2 100644 --- a/tests/controller/test_export_project.py +++ b/tests/controller/test_export_project.py @@ -27,13 +27,12 @@ from unittest.mock import MagicMock from tests.utils import AsyncioMagicMock, AsyncioBytesIO from gns3server.controller.project import Project -from gns3server.controller.compute import Compute from gns3server.controller.export_project import export_project, _filter_files @pytest.fixture def project(controller): - p = Project(controller=controller, name="Test") + p = Project(controller=controller, name="test") p.dump = MagicMock() return p @@ -179,7 +178,7 @@ def test_export_disallow_running(tmpdir, project, node, async_run): node._status = "started" with pytest.raises(aiohttp.web.HTTPConflict): - z = async_run(export_project(project, str(tmpdir))) + async_run(export_project(project, str(tmpdir))) def test_export_disallow_some_type(tmpdir, project, async_run): diff --git a/tests/controller/test_import_project.py b/tests/controller/test_import_project.py index e61e1ed8..7e03ea28 100644 --- a/tests/controller/test_import_project.py +++ b/tests/controller/test_import_project.py @@ -35,6 +35,8 @@ def test_import_project(async_run, tmpdir, controller): topology = { "project_id": str(uuid.uuid4()), "name": "test", + "auto_open": True, + "auto_start": True, "topology": { }, "version": "2.0.0" @@ -67,6 +69,8 @@ def test_import_project(async_run, tmpdir, controller): # A new project name is generated when you import twice the same name with open(zip_path, "rb") as f: project = async_run(import_project(controller, str(uuid.uuid4()), f)) + assert project.auto_open is False + assert project.auto_start is False assert project.name != "test" diff --git a/tests/topologies/1_3_dynamips_missing_type/after/1_3_dynamips.gns3 b/tests/topologies/1_3_dynamips_missing_type/after/1_3_dynamips.gns3 new file mode 100644 index 00000000..30326e4e --- /dev/null +++ b/tests/topologies/1_3_dynamips_missing_type/after/1_3_dynamips.gns3 @@ -0,0 +1,76 @@ +{ + "auto_start": false, + "name": "1_3_dynamips", + "project_id": "ba5790e1-2f51-443e-a3cc-1a2eee132888", + "revision": 6, + "topology": { + "computes": [ + { + "compute_id": "local", + "host": "127.0.0.1", + "name": "Local", + "port": 8000, + "protocol": "http" + } + ], + "drawings": [], + "links": [], + "nodes": [ + { + "symbol": ":/symbols/iosv_virl.svg", + "compute_id": "local", + "console": 2001, + "console_type": "telnet", + "label": { + "rotation": 0, + "style": "font-family: TypeWriter;font-size: 10;font-weight: bold;fill: #000000;fill-opacity: 1.0;", + "text": "R1", + "x": 22, + "y": -25 + }, + "name": "R1", + "node_id": "0bce6ad5-c688-4d4d-a425-f21aaf3927e2", + "node_type": "dynamips", + "port_name_format": "Ethernet{0}", + "port_segment_size": 0, + "first_port_name": null, + "properties": { + "dynamips_id": 1, + "auto_delete_disks": true, + "clock_divisor": 4, + "disk0": 0, + "disk1": 0, + "exec_area": 64, + "idlemax": 500, + "idlesleep": 30, + "image": "c7200-adventerprisek9-mz.124-24.T8.image", + "mac_addr": "ca01.2f39.0000", + "midplane": "vxr", + "mmap": true, + "npe": "npe-400", + "nvram": 512, + "platform": "c7200", + "power_supplies": [ + 1, + 1 + ], + "ram": 512, + "sensors": [ + 22, + 22, + 22, + 22 + ], + "slot0": "C7200-IO-FE", + "sparsemem": true, + "system_id": "FTX0945W0MY" + }, + "x": -112, + "y": -100, + "z": 1 + } + ] + }, + "type": "topology", + "version": "ANYSTR" +} diff --git a/tests/topologies/1_3_dynamips_missing_type/before/1_3_dynamips.gns3 b/tests/topologies/1_3_dynamips_missing_type/before/1_3_dynamips.gns3 new file mode 100644 index 00000000..14f2f95d --- /dev/null +++ b/tests/topologies/1_3_dynamips_missing_type/before/1_3_dynamips.gns3 @@ -0,0 +1,80 @@ +{ + "auto_start": false, + "name": "1_3_dynamips", + "project_id": "ba5790e1-2f51-443e-a3cc-1a2eee132888", + "revision": 3, + "topology": { + "nodes": [ + { + "default_symbol": ":/symbols/iosv_virl.normal.svg", + "description": "Router c7200", + "dynamips_id": 1, + "hover_symbol": ":/symbols/iosv_virl.selected.svg", + "id": 1, + "label": { + "color": "#000000", + "font": "TypeWriter,10,-1,5,75,0,0,0,0,0", + "text": "R1", + "x": 22.6171875, + "y": -25.0 + }, + "ports": [ + { + "adapter_number": 0, + "id": 1, + "name": "FastEthernet0/0", + "port_number": 0 + } + ], + "properties": { + "auto_delete_disks": true, + "clock_divisor": 4, + "console": 2001, + "disk0": 0, + "disk1": 0, + "exec_area": 64, + "idlemax": 500, + "idlesleep": 30, + "image": "c7200-adventerprisek9-mz.124-24.T8.image", + "mac_addr": "ca01.2f39.0000", + "midplane": "vxr", + "mmap": true, + "name": "R1", + "npe": "npe-400", + "nvram": 512, + "platform": "c7200", + "power_supplies": [ + 1, + 1 + ], + "ram": 512, + "sensors": [ + 22, + 22, + 22, + 22 + ], + "slot0": "C7200-IO-FE", + "sparsemem": true, + "startup_config": "configs/i1_startup-config.cfg", + "system_id": "FTX0945W0MY" + }, + "server_id": 1, + "vm_id": "0bce6ad5-c688-4d4d-a425-f21aaf3927e2", + "x": -112.0, + "y": -100.0 + } + ], + "servers": [ + { + "cloud": false, + "host": "127.0.0.1", + "id": 1, + "local": true, + "port": 8000 + } + ] + }, + "type": "topology", + "version": "1.3.13" +} diff --git a/tests/utils.py b/tests/utils.py index 3c638d52..f1cfd191 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,6 +16,7 @@ # along with this program. If not, see . import io +import types import asyncio import unittest.mock @@ -69,6 +70,10 @@ class AsyncioMagicMock(unittest.mock.MagicMock): """ Magic mock returning coroutine """ + try: + __class__ = types.CoroutineType + except AttributeError: # Not supported with Python 3.4 + __class__ = types.GeneratorType def __init__(self, return_value=None, return_values=None, **kwargs): """