From 8b879c061443a69217fe341fd1a916c084ea63d8 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Mon, 20 Apr 2015 16:57:03 +0200 Subject: [PATCH 1/3] Support wireshark remote capture --- gns3server/handlers/__init__.py | 1 + gns3server/handlers/api/file_handler.py | 54 +++++++++++++++++++++ gns3server/schemas/file.py | 32 +++++++++++++ tests/handlers/api/base.py | 4 +- tests/handlers/api/test_file.py | 62 +++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 gns3server/handlers/api/file_handler.py create mode 100644 gns3server/schemas/file.py create mode 100644 tests/handlers/api/test_file.py diff --git a/gns3server/handlers/__init__.py b/gns3server/handlers/__init__.py index 307eece3..5db20453 100644 --- a/gns3server/handlers/__init__.py +++ b/gns3server/handlers/__init__.py @@ -27,6 +27,7 @@ from gns3server.handlers.api.virtualbox_handler import VirtualBoxHandler from gns3server.handlers.api.vpcs_handler import VPCSHandler from gns3server.handlers.api.config_handler import ConfigHandler from gns3server.handlers.api.server_handler import ServerHandler +from gns3server.handlers.api.file_handler import FileHandler from gns3server.handlers.upload_handler import UploadHandler from gns3server.handlers.index_handler import IndexHandler diff --git a/gns3server/handlers/api/file_handler.py b/gns3server/handlers/api/file_handler.py new file mode 100644 index 00000000..1dc2e31b --- /dev/null +++ b/gns3server/handlers/api/file_handler.py @@ -0,0 +1,54 @@ +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import asyncio + +from ...web.route import Route +from ...schemas.file import FILE_STREAM_SCHEMA + + +class FileHandler: + + @classmethod + @Route.get( + r"/files/stream", + description="Stream a file from the server", + status_codes={ + 200: "File retrieved", + 404: "File doesn't exist" + }, + input=FILE_STREAM_SCHEMA + ) + def read(request, response): + try: + with open(request.json.get("location"), "rb") as f: + loop = asyncio.get_event_loop() + response.content_type = "application/octet-stream" + response.set_status(200) + response.enable_chunked_encoding() + # Very important: do not send a content lenght otherwise QT close the connection but curl can consume the Feed + response.content_length = None + + response.start(request) + + while True: + data = yield from loop.run_in_executor(None, f.read, 16) + if len(data) == 0: + yield from asyncio.sleep(0.1) + else: + response.write(data) + except FileNotFoundError: + response.set_status(404) diff --git a/gns3server/schemas/file.py b/gns3server/schemas/file.py new file mode 100644 index 00000000..38ce7a10 --- /dev/null +++ b/gns3server/schemas/file.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +FILE_STREAM_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Request validation retrieval of a file stream", + "type": "object", + "properties": { + "location": { + "description": "File path", + "type": ["string"], + "minLength": 1 + } + }, + "additionalProperties": False, + "required": ["location"] +} diff --git a/tests/handlers/api/base.py b/tests/handlers/api/base.py index 6a4fd02e..4b677d17 100644 --- a/tests/handlers/api/base.py +++ b/tests/handlers/api/base.py @@ -44,7 +44,7 @@ class Query: def delete(self, path, **kwargs): return self._fetch("DELETE", path, **kwargs) - def _get_url(self, path, version): + def get_url(self, path, version): if version is None: return "http://{}:{}{}".format(self._host, self._port, path) return "http://{}:{}/v{}{}".format(self._host, self._port, version, path) @@ -62,7 +62,7 @@ class Query: @asyncio.coroutine def go(future): - response = yield from aiohttp.request(method, self._get_url(path, api_version), data=body) + response = yield from aiohttp.request(method, self.get_url(path, api_version), data=body) future.set_result(response) future = asyncio.Future() asyncio.async(go(future)) diff --git a/tests/handlers/api/test_file.py b/tests/handlers/api/test_file.py new file mode 100644 index 00000000..0b110b2e --- /dev/null +++ b/tests/handlers/api/test_file.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +This test suite check /files endpoint +""" + +import json +import asyncio +import aiohttp + +from gns3server.version import __version__ + + +def test_stream(server, tmpdir, loop): + with open(str(tmpdir / "test"), 'w+') as f: + f.write("hello") + + def go(future): + query = json.dumps({"location": str(tmpdir / "test")}) + headers = {'content-type': 'application/json'} + response = yield from aiohttp.request("GET", server.get_url("/files/stream", 1), data=query, headers=headers) + response.body = yield from response.content.read(5) + with open(str(tmpdir / "test"), 'a') as f: + f.write("world") + response.body += yield from response.content.read(5) + response.close() + future.set_result(response) + + future = asyncio.Future() + asyncio.async(go(future)) + response = loop.run_until_complete(future) + assert response.status == 200 + assert response.body == b'helloworld' + + +def test_stream_file_not_found(server, tmpdir, loop): + def go(future): + query = json.dumps({"location": str(tmpdir / "test")}) + headers = {'content-type': 'application/json'} + response = yield from aiohttp.request("GET", server.get_url("/files/stream", 1), data=query, headers=headers) + response.close() + future.set_result(response) + + future = asyncio.Future() + asyncio.async(go(future)) + response = loop.run_until_complete(future) + assert response.status == 404 From bf618d321c5570974641ab50866dea86c6b4d10a Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 22 Apr 2015 10:39:43 +0200 Subject: [PATCH 2/3] Max 100 thread executor --- gns3server/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gns3server/main.py b/gns3server/main.py index ac6c6b19..2ced1294 100644 --- a/gns3server/main.py +++ b/gns3server/main.py @@ -21,6 +21,8 @@ import datetime import sys import locale import argparse +import asyncio +import concurrent from gns3server.server import Server from gns3server.web.logger import init_logger @@ -173,6 +175,9 @@ def main(): Project.clean_project_directory() + executor = concurrent.futures.ThreadPoolExecutor(max_workers=100) # We allow 100 parallel executors + loop = asyncio.get_event_loop().set_default_executor(executor) + CrashReport.instance() host = server_config["host"] port = int(server_config["port"]) From 83f2509cfe778b69a1aed2475d58047f81317e17 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 22 Apr 2015 10:47:49 +0200 Subject: [PATCH 3/3] Cleanup exceptions --- gns3server/handlers/api/file_handler.py | 12 +++++++++--- gns3server/main.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/gns3server/handlers/api/file_handler.py b/gns3server/handlers/api/file_handler.py index 1dc2e31b..e705ffc2 100644 --- a/gns3server/handlers/api/file_handler.py +++ b/gns3server/handlers/api/file_handler.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import asyncio +import aiohttp from ...web.route import Route from ...schemas.file import FILE_STREAM_SCHEMA @@ -28,17 +29,19 @@ class FileHandler: description="Stream a file from the server", status_codes={ 200: "File retrieved", - 404: "File doesn't exist" + 404: "File doesn't exist", + 409: "Can't access to file" }, input=FILE_STREAM_SCHEMA ) def read(request, response): + response.enable_chunked_encoding() + try: with open(request.json.get("location"), "rb") as f: loop = asyncio.get_event_loop() response.content_type = "application/octet-stream" response.set_status(200) - response.enable_chunked_encoding() # Very important: do not send a content lenght otherwise QT close the connection but curl can consume the Feed response.content_length = None @@ -51,4 +54,7 @@ class FileHandler: else: response.write(data) except FileNotFoundError: - response.set_status(404) + raise aiohttp.web.HTTPNotFound() + except OSError as e: + raise aiohttp.web.HTTPConflict(text=str(e)) + diff --git a/gns3server/main.py b/gns3server/main.py index 2ced1294..7404365c 100644 --- a/gns3server/main.py +++ b/gns3server/main.py @@ -175,7 +175,7 @@ def main(): Project.clean_project_directory() - executor = concurrent.futures.ThreadPoolExecutor(max_workers=100) # We allow 100 parallel executors + executor = concurrent.futures.ThreadPoolExecutor(max_workers=100) # We allow 100 parallel executors loop = asyncio.get_event_loop().set_default_executor(executor) CrashReport.instance()