From 270017d9456833c3f6fa28cf94fd96c1e41142f0 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Tue, 21 Jul 2015 14:20:58 +0200 Subject: [PATCH 1/3] Allow user to backup projects and images --- gns3server/handlers/upload_handler.py | 48 +++++++++++++++++++ gns3server/templates/layout.html | 7 +++ tests/handlers/api/base.py | 5 +- tests/handlers/test_upload.py | 68 +++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/gns3server/handlers/upload_handler.py b/gns3server/handlers/upload_handler.py index 9a431276..0e8036b8 100644 --- a/gns3server/handlers/upload_handler.py +++ b/gns3server/handlers/upload_handler.py @@ -18,6 +18,9 @@ import os import aiohttp import stat +import io +import tarfile +import asyncio from ..config import Config from ..web.route import Route @@ -81,7 +84,52 @@ class UploadHandler: return response.redirect("/upload") + @classmethod + @Route.get( + r"/upload/backup/images.tar", + description="Backup GNS3 images", + api_version=None + ) + def backup_images(request, response): + yield from UploadHandler._backup_directory(request, response, UploadHandler.image_directory()) + + @classmethod + @Route.get( + r"/upload/backup/projects.tar", + description="Backup GNS3 projects", + api_version=None + ) + def backup_images(request, response): + yield from UploadHandler._backup_directory(request, response, UploadHandler.project_directory()) + + @staticmethod + @asyncio.coroutine + def _backup_directory(request, response, directory): + response.content_type = 'application/x-gtar' + response.set_status(200) + response.enable_chunked_encoding() + # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed + response.content_length = None + response.start(request) + + buffer = io.BytesIO() + with tarfile.open('arch.tar', 'w', fileobj=buffer) as tar: + for root, dirs, files in os.walk(directory): + for file in files: + path = os.path.join(root, file) + tar.add(os.path.join(root, file), arcname=os.path.relpath(path, directory)) + response.write(buffer.getvalue()) + yield from response.drain() + buffer.truncate(0) + buffer.seek(0) + yield from response.write_eof() + @staticmethod def image_directory(): server_config = Config.instance().get_section_config("Server") return os.path.expanduser(server_config.get("images_path", "~/GNS3/images")) + + @staticmethod + def project_directory(): + server_config = Config.instance().get_section_config("Server") + return os.path.expanduser(server_config.get("projects_path", "~/GNS3/images")) diff --git a/gns3server/templates/layout.html b/gns3server/templates/layout.html index 9ecbc82c..bcb38145 100644 --- a/gns3server/templates/layout.html +++ b/gns3server/templates/layout.html @@ -4,6 +4,13 @@ GNS3 Server +
+ Home + | + Backup images + | + Backup projects +
{% block body %}{% endblock %} diff --git a/tests/handlers/api/base.py b/tests/handlers/api/base.py index 6a4fd02e..a4a3a034 100644 --- a/tests/handlers/api/base.py +++ b/tests/handlers/api/base.py @@ -88,7 +88,10 @@ class Query: except ValueError: response.json = None else: - response.html = response.body.decode("utf-8") + try: + response.html = response.body.decode("utf-8") + except UnicodeDecodeError: + response.html = None else: response.json = {} response.html = "" diff --git a/tests/handlers/test_upload.py b/tests/handlers/test_upload.py index 64fdab73..8d3a8b66 100644 --- a/tests/handlers/test_upload.py +++ b/tests/handlers/test_upload.py @@ -17,10 +17,15 @@ import aiohttp +import asyncio import os +import tarfile from unittest.mock import patch + + from gns3server.config import Config + def test_index_upload(server): response = server.get('/upload', api_version=None) assert response.status == 200 @@ -44,3 +49,66 @@ def test_upload(server, tmpdir): assert f.read() == "TEST" assert "test2" in response.body.decode("utf-8") + + +def test_backup_images(server, tmpdir, loop): + Config.instance().set('Server', 'images_path', str(tmpdir)) + + os.makedirs(str(tmpdir / 'QEMU')) + with open(str(tmpdir / 'QEMU' / 'a.img'), 'w+') as f: + f.write('hello') + with open(str(tmpdir / 'QEMU' / 'b.img'), 'w+') as f: + f.write('world') + + response = server.get('/upload/backup/images.tar', api_version=None, raw=True) + assert response.status == 200 + assert response.headers['CONTENT-TYPE'] == 'application/x-gtar' + + with open(str(tmpdir / 'images.tar'), 'wb+') as f: + print(len(response.body)) + f.write(response.body) + + tar = tarfile.open(str(tmpdir / 'images.tar'), 'r') + os.makedirs(str(tmpdir / 'extract')) + os.chdir(str(tmpdir / 'extract')) + # Extract to current working directory + tar.extractall() + tar.close() + + assert os.path.exists(os.path.join('QEMU', 'a.img')) + open(os.path.join('QEMU', 'a.img')).read() == 'hello' + + assert os.path.exists(os.path.join('QEMU', 'b.img')) + open(os.path.join('QEMU', 'b.img')).read() == 'world' + + +def test_backup_projects(server, tmpdir, loop): + Config.instance().set('Server', 'projects_path', str(tmpdir)) + + os.makedirs(str(tmpdir / 'a')) + with open(str(tmpdir / 'a' / 'a.gns3'), 'w+') as f: + f.write('hello') + os.makedirs(str(tmpdir / 'b')) + with open(str(tmpdir / 'b' / 'b.gns3'), 'w+') as f: + f.write('world') + + response = server.get('/upload/backup/projects.tar', api_version=None, raw=True) + assert response.status == 200 + assert response.headers['CONTENT-TYPE'] == 'application/x-gtar' + + with open(str(tmpdir / 'projects.tar'), 'wb+') as f: + print(len(response.body)) + f.write(response.body) + + tar = tarfile.open(str(tmpdir / 'projects.tar'), 'r') + os.makedirs(str(tmpdir / 'extract')) + os.chdir(str(tmpdir / 'extract')) + # Extract to current working directory + tar.extractall() + tar.close() + + assert os.path.exists(os.path.join('a', 'a.gns3')) + open(os.path.join('a', 'a.gns3')).read() == 'hello' + + assert os.path.exists(os.path.join('b', 'b.gns3')) + open(os.path.join('b', 'b.gns3')).read() == 'world' From fc14deee1bdf45e862ba83ac44fcb59899ecb0dc Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Tue, 21 Jul 2015 16:14:03 +0200 Subject: [PATCH 2/3] Restore images & projects tarballs --- gns3server/handlers/upload_handler.py | 56 +++++++++++++++------ gns3server/templates/index.html | 2 +- gns3server/templates/layout.html | 8 +-- gns3server/templates/upload.html | 2 + tests/handlers/test_upload.py | 72 ++++++++++++++++++++++++++- 5 files changed, 118 insertions(+), 22 deletions(-) diff --git a/gns3server/handlers/upload_handler.py b/gns3server/handlers/upload_handler.py index 0e8036b8..716b52a1 100644 --- a/gns3server/handlers/upload_handler.py +++ b/gns3server/handlers/upload_handler.py @@ -62,23 +62,29 @@ class UploadHandler: response.redirect("/upload") return - if data["type"] not in ["IOU", "IOURC", "QEMU", "IOS"]: - raise aiohttp.web.HTTPForbidden("You are not authorized to upload this kind of image {}".format(data["type"])) + if data["type"] not in ["IOU", "IOURC", "QEMU", "IOS", "IMAGES", "PROJECTS"]: + raise aiohttp.web.HTTPForbidden(text="You are not authorized to upload this kind of image {}".format(data["type"])) - if data["type"] == "IOURC": - destination_dir = os.path.expanduser("~/") - destination_path = os.path.join(destination_dir, ".iourc") - else: - destination_dir = os.path.join(UploadHandler.image_directory(), data["type"]) - destination_path = os.path.join(destination_dir, data["file"].filename) try: - os.makedirs(destination_dir, exist_ok=True) - with open(destination_path, "wb+") as f: - chunk = data["file"].file.read() - f.write(chunk) - st = os.stat(destination_path) - os.chmod(destination_path, st.st_mode | stat.S_IXUSR) + if data["type"] == "IMAGES": + UploadHandler._restore_directory(data["file"], UploadHandler.image_directory()) + elif data["type"] == "PROJECTS": + UploadHandler._restore_directory(data["file"], UploadHandler.project_directory()) + else: + if data["type"] == "IOURC": + destination_dir = os.path.expanduser("~/") + destination_path = os.path.join(destination_dir, ".iourc") + else: + destination_dir = os.path.join(UploadHandler.image_directory(), data["type"]) + destination_path = os.path.join(destination_dir, data["file"].filename) + os.makedirs(destination_dir, exist_ok=True) + with open(destination_path, "wb+") as f: + chunk = data["file"].file.read() + f.write(chunk) + st = os.stat(destination_path) + os.chmod(destination_path, st.st_mode | stat.S_IXUSR) except OSError as e: + print(e) response.html("Could not upload file: {}".format(e)) response.set_status(200) return @@ -86,7 +92,7 @@ class UploadHandler: @classmethod @Route.get( - r"/upload/backup/images.tar", + r"/backup/images.tar", description="Backup GNS3 images", api_version=None ) @@ -95,16 +101,34 @@ class UploadHandler: @classmethod @Route.get( - r"/upload/backup/projects.tar", + r"/backup/projects.tar", description="Backup GNS3 projects", api_version=None ) def backup_images(request, response): yield from UploadHandler._backup_directory(request, response, UploadHandler.project_directory()) + @staticmethod + def _restore_directory(file, directory): + """ + Extract from HTTP stream the content of a tar + """ + destination_path = os.path.join(directory, "archive.tar") + os.makedirs(directory, exist_ok=True) + with open(destination_path, "wb+") as f: + chunk = file.file.read() + f.write(chunk) + t = tarfile.open(destination_path) + t.extractall(directory) + t.close() + os.remove(destination_path) + @staticmethod @asyncio.coroutine def _backup_directory(request, response, directory): + """ + Return a tar archive from a directory + """ response.content_type = 'application/x-gtar' response.set_status(200) response.enable_chunked_encoding() diff --git a/gns3server/templates/index.html b/gns3server/templates/index.html index f0fa4304..aa0e14f5 100644 --- a/gns3server/templates/index.html +++ b/gns3server/templates/index.html @@ -6,6 +6,6 @@ {% endblock %} diff --git a/gns3server/templates/layout.html b/gns3server/templates/layout.html index bcb38145..cc451233 100644 --- a/gns3server/templates/layout.html +++ b/gns3server/templates/layout.html @@ -5,11 +5,13 @@
- Home + Home | - Backup images + Upload + | + Backup images | - Backup projects + Backup projects
{% block body %}{% endblock %} diff --git a/gns3server/templates/upload.html b/gns3server/templates/upload.html index a894e467..91db249e 100644 --- a/gns3server/templates/upload.html +++ b/gns3server/templates/upload.html @@ -8,6 +8,8 @@ + +

diff --git a/tests/handlers/test_upload.py b/tests/handlers/test_upload.py index 8d3a8b66..2ad8d49c 100644 --- a/tests/handlers/test_upload.py +++ b/tests/handlers/test_upload.py @@ -51,6 +51,74 @@ def test_upload(server, tmpdir): assert "test2" in response.body.decode("utf-8") +def test_upload_images_backup(server, tmpdir): + Config.instance().set("Server", "images_path", str(tmpdir / 'images')) + os.makedirs(str(tmpdir / 'images' / 'IOU')) + # An old IOU image that we need to replace + with open(str(tmpdir / 'images' / 'IOU' / 'b.img'), 'w+') as f: + f.write('bad') + + os.makedirs(str(tmpdir / 'old' / 'QEMU')) + with open(str(tmpdir / 'old' / 'QEMU' / 'a.img'), 'w+') as f: + f.write('hello') + os.makedirs(str(tmpdir / 'old' / 'IOU')) + with open(str(tmpdir / 'old' / 'IOU' / 'b.img'), 'w+') as f: + f.write('world') + + os.chdir(str(tmpdir / 'old')) + with tarfile.open(str(tmpdir / 'test.tar'), 'w') as tar: + tar.add('.', recursive=True) + + body = aiohttp.FormData() + body.add_field('type', 'IMAGES') + body.add_field('file', open(str(tmpdir / 'test.tar'), 'rb'), content_type='application/x-gtar', filename='test.tar') + response = server.post('/upload', api_version=None, body=body, raw=True) + assert response.status == 200 + + with open(str(tmpdir / 'images' / 'QEMU' / 'a.img')) as f: + assert f.read() == 'hello' + with open(str(tmpdir / 'images' / 'IOU' / 'b.img')) as f: + assert f.read() == 'world' + + assert 'a.img' in response.body.decode('utf-8') + assert 'b.img' in response.body.decode('utf-8') + assert not os.path.exists(str(tmpdir / 'images' / 'archive.tar')) + + +def test_upload_projects_backup(server, tmpdir): + Config.instance().set("Server", "projects_path", str(tmpdir / 'projects')) + os.makedirs(str(tmpdir / 'projects' / 'b')) + # An old b image that we need to replace + with open(str(tmpdir / 'projects' / 'b' / 'b.img'), 'w+') as f: + f.write('bad') + + os.makedirs(str(tmpdir / 'old' / 'a')) + with open(str(tmpdir / 'old' / 'a' / 'a.img'), 'w+') as f: + f.write('hello') + os.makedirs(str(tmpdir / 'old' / 'b')) + with open(str(tmpdir / 'old' / 'b' / 'b.img'), 'w+') as f: + f.write('world') + + os.chdir(str(tmpdir / 'old')) + with tarfile.open(str(tmpdir / 'test.tar'), 'w') as tar: + tar.add('.', recursive=True) + + body = aiohttp.FormData() + body.add_field('type', 'PROJECTS') + body.add_field('file', open(str(tmpdir / 'test.tar'), 'rb'), content_type='application/x-gtar', filename='test.tar') + response = server.post('/upload', api_version=None, body=body, raw=True) + assert response.status == 200 + + with open(str(tmpdir / 'projects' / 'a' / 'a.img')) as f: + assert f.read() == 'hello' + with open(str(tmpdir / 'projects' / 'b' / 'b.img')) as f: + assert f.read() == 'world' + + assert 'a.img' not in response.body.decode('utf-8') + assert 'b.img' not in response.body.decode('utf-8') + assert not os.path.exists(str(tmpdir / 'projects' / 'archive.tar')) + + def test_backup_images(server, tmpdir, loop): Config.instance().set('Server', 'images_path', str(tmpdir)) @@ -60,7 +128,7 @@ def test_backup_images(server, tmpdir, loop): with open(str(tmpdir / 'QEMU' / 'b.img'), 'w+') as f: f.write('world') - response = server.get('/upload/backup/images.tar', api_version=None, raw=True) + response = server.get('/backup/images.tar', api_version=None, raw=True) assert response.status == 200 assert response.headers['CONTENT-TYPE'] == 'application/x-gtar' @@ -92,7 +160,7 @@ def test_backup_projects(server, tmpdir, loop): with open(str(tmpdir / 'b' / 'b.gns3'), 'w+') as f: f.write('world') - response = server.get('/upload/backup/projects.tar', api_version=None, raw=True) + response = server.get('/backup/projects.tar', api_version=None, raw=True) assert response.status == 200 assert response.headers['CONTENT-TYPE'] == 'application/x-gtar' From 6bf7a6aa3884932b36e9133f97477a8d8ec7641e Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Tue, 21 Jul 2015 20:19:29 +0200 Subject: [PATCH 3/3] Fix after jeremy feedback --- gns3server/handlers/upload_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gns3server/handlers/upload_handler.py b/gns3server/handlers/upload_handler.py index 716b52a1..45e0cf4b 100644 --- a/gns3server/handlers/upload_handler.py +++ b/gns3server/handlers/upload_handler.py @@ -84,7 +84,6 @@ class UploadHandler: st = os.stat(destination_path) os.chmod(destination_path, st.st_mode | stat.S_IXUSR) except OSError as e: - print(e) response.html("Could not upload file: {}".format(e)) response.set_status(200) return @@ -156,4 +155,4 @@ class UploadHandler: @staticmethod def project_directory(): server_config = Config.instance().get_section_config("Server") - return os.path.expanduser(server_config.get("projects_path", "~/GNS3/images")) + return os.path.expanduser(server_config.get("projects_path", "~/GNS3/projects"))