diff --git a/gns3server/handlers/upload_handler.py b/gns3server/handlers/upload_handler.py index 2ca39658..789760e9 100644 --- a/gns3server/handlers/upload_handler.py +++ b/gns3server/handlers/upload_handler.py @@ -21,6 +21,7 @@ import stat from ..config import Config from ..web.route import Route +from ..utils.images import remove_checksum, md5sum class UploadHandler: @@ -36,7 +37,7 @@ class UploadHandler: try: for root, _, files in os.walk(UploadHandler.image_directory()): for filename in files: - if not filename.startswith("."): + if not filename.startswith(".") and not filename.endswith(".md5sum"): image_file = os.path.join(root, filename) uploaded_files.append(image_file) except OSError: @@ -70,12 +71,14 @@ class UploadHandler: destination_path = os.path.join(destination_dir, data["file"].filename) try: os.makedirs(destination_dir, exist_ok=True) + remove_checksum(destination_path) with open(destination_path, "wb+") as f: while True: chunk = data["file"].file.read(512) if not chunk: break f.write(chunk) + md5sum(destination_path) st = os.stat(destination_path) os.chmod(destination_path, st.st_mode | stat.S_IXUSR) except OSError as e: diff --git a/gns3server/modules/base_manager.py b/gns3server/modules/base_manager.py index 3ee0b347..ebf6d0d3 100644 --- a/gns3server/modules/base_manager.py +++ b/gns3server/modules/base_manager.py @@ -37,6 +37,7 @@ from .nios.nio_udp import NIOUDP from .nios.nio_tap import NIOTAP from .nios.nio_nat import NIONAT from .nios.nio_generic_ethernet import NIOGenericEthernet +from ..utils.images import md5sum, remove_checksum class BaseManager: @@ -444,7 +445,7 @@ class BaseManager: files.sort() images = [] for filename in files: - if filename[0] != ".": + if filename[0] != "." and not filename.endswith(".md5sum"): images.append({"filename": filename}) return images @@ -461,6 +462,7 @@ class BaseManager: path = os.path.join(directory, os.path.basename(filename)) log.info("Writting image file %s", path) try: + remove_checksum(path) os.makedirs(directory, exist_ok=True) with open(path, 'wb+') as f: while True: @@ -469,5 +471,6 @@ class BaseManager: break f.write(packet) os.chmod(path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC) + md5sum(path) except OSError as e: raise aiohttp.web.HTTPConflict(text="Could not write image: {} to {}".format(filename, e)) diff --git a/gns3server/modules/dynamips/nodes/router.py b/gns3server/modules/dynamips/nodes/router.py index 5342cf72..9af59229 100644 --- a/gns3server/modules/dynamips/nodes/router.py +++ b/gns3server/modules/dynamips/nodes/router.py @@ -37,6 +37,7 @@ from ..nios.nio_udp import NIOUDP from gns3server.config import Config from gns3server.utils.asyncio import wait_run_in_executor, monitor_process +from gns3server.utils.images import md5sum class Router(BaseVM): @@ -134,6 +135,7 @@ class Router(BaseVM): "dynamips_id": self._dynamips_id, "platform": self._platform, "image": self._image, + "image_md5sum": md5sum(self._image), "startup_config": self._startup_config, "private_config": self._private_config, "ram": self._ram, diff --git a/gns3server/modules/iou/iou_vm.py b/gns3server/modules/iou/iou_vm.py index 3c8bdaf4..7dbd34da 100644 --- a/gns3server/modules/iou/iou_vm.py +++ b/gns3server/modules/iou/iou_vm.py @@ -46,6 +46,7 @@ from .utils.iou_import import nvram_import from .utils.iou_export import nvram_export from .ioucon import start_ioucon import gns3server.utils.asyncio +import gns3server.utils.images import logging @@ -208,6 +209,7 @@ class IOUVM(BaseVM): "console": self._console, "project_id": self.project.id, "path": self.path, + "md5sum": gns3server.utils.images.md5sum(self.path), "ethernet_adapters": len(self._ethernet_adapters), "serial_adapters": len(self._serial_adapters), "ram": self._ram, @@ -789,7 +791,7 @@ class IOUVM(BaseVM): # do not let IOU create the NVRAM anymore #startup_config_file = self.startup_config_file - #if startup_config_file: + # if startup_config_file: # command.extend(["-c", os.path.basename(startup_config_file)]) if self._l1_keepalives: diff --git a/gns3server/modules/qemu/qemu_vm.py b/gns3server/modules/qemu/qemu_vm.py index 38eb02a0..84be4d8b 100644 --- a/gns3server/modules/qemu/qemu_vm.py +++ b/gns3server/modules/qemu/qemu_vm.py @@ -38,6 +38,7 @@ from ..nios.nio_nat import NIONAT from ..base_vm import BaseVM from ...schemas.qemu import QEMU_OBJECT_SCHEMA, QEMU_PLATFORMS from ...utils.asyncio import monitor_process +from ...utils.images import md5sum import logging log = logging.getLogger(__name__) @@ -1217,13 +1218,23 @@ class QemuVM(BaseVM): # Qemu has a long list of options. The JSON schema is the single source of information for field in QEMU_OBJECT_SCHEMA["required"]: if field not in answer: - answer[field] = getattr(self, field) + try: + answer[field] = getattr(self, field) + except AttributeError: + pass answer["hda_disk_image"] = self.manager.get_relative_image_path(self._hda_disk_image) + answer["hda_disk_image_md5sum"] = md5sum(self._hda_disk_image) answer["hdb_disk_image"] = self.manager.get_relative_image_path(self._hdb_disk_image) + answer["hdb_disk_image_md5sum"] = md5sum(self._hdb_disk_image) answer["hdc_disk_image"] = self.manager.get_relative_image_path(self._hdc_disk_image) + answer["hdc_disk_image_md5sum"] = md5sum(self._hdc_disk_image) answer["hdd_disk_image"] = self.manager.get_relative_image_path(self._hdd_disk_image) + answer["hdd_disk_image_md5sum"] = md5sum(self._hdd_disk_image) answer["initrd"] = self.manager.get_relative_image_path(self._initrd) + answer["initrd_md5sum"] = md5sum(self._initrd) + answer["kernel_image"] = self.manager.get_relative_image_path(self._kernel_image) + answer["kernel_image_md5sum"] = md5sum(self._kernel_image) return answer diff --git a/gns3server/schemas/dynamips_vm.py b/gns3server/schemas/dynamips_vm.py index 4e6e9ee4..1bfdcc50 100644 --- a/gns3server/schemas/dynamips_vm.py +++ b/gns3server/schemas/dynamips_vm.py @@ -546,6 +546,11 @@ VM_OBJECT_SCHEMA = { "type": "string", "minLength": 1, }, + "image_md5sum": { + "description": "checksum of the IOS image", + "type": "string", + "minLength": 1, + }, "startup_config": { "description": "path to the IOS startup configuration file", "type": "string", diff --git a/gns3server/schemas/iou.py b/gns3server/schemas/iou.py index 00ce5bfc..c5db5700 100644 --- a/gns3server/schemas/iou.py +++ b/gns3server/schemas/iou.py @@ -189,6 +189,10 @@ IOU_OBJECT_SCHEMA = { "description": "Path of iou binary", "type": "string" }, + "md5sum": { + "description": "Checksum of iou binary", + "type": "string" + }, "serial_adapters": { "description": "How many serial adapters are connected to the IOU", "type": "integer" @@ -227,7 +231,7 @@ IOU_OBJECT_SCHEMA = { } }, "additionalProperties": False, - "required": ["name", "vm_id", "console", "project_id", "path", "serial_adapters", "ethernet_adapters", + "required": ["name", "vm_id", "console", "project_id", "path", "md5sum", "serial_adapters", "ethernet_adapters", "ram", "nvram", "l1_keepalives", "startup_config", "private_config", "use_default_iou_values"] } diff --git a/gns3server/schemas/qemu.py b/gns3server/schemas/qemu.py index 40cd4f3b..3df7242c 100644 --- a/gns3server/schemas/qemu.py +++ b/gns3server/schemas/qemu.py @@ -282,18 +282,34 @@ QEMU_OBJECT_SCHEMA = { "description": "QEMU hda disk image path", "type": "string", }, + "hda_disk_image_md5sum": { + "description": "QEMU hda disk image checksum", + "type": ["string", "null"] + }, "hdb_disk_image": { "description": "QEMU hdb disk image path", "type": "string", }, + "hdb_disk_image_md5sum": { + "description": "QEMU hdb disk image checksum", + "type": ["string", "null"], + }, "hdc_disk_image": { "description": "QEMU hdc disk image path", "type": "string", }, + "hdc_disk_image_md5sum": { + "description": "QEMU hdc disk image checksum", + "type": ["string", "null"], + }, "hdd_disk_image": { "description": "QEMU hdd disk image path", "type": "string", }, + "hdd_disk_image_md5sum": { + "description": "QEMU hdd disk image checksum", + "type": ["string", "null"], + }, "ram": { "description": "amount of RAM in MB", "type": "integer" @@ -325,10 +341,18 @@ QEMU_OBJECT_SCHEMA = { "description": "QEMU initrd path", "type": "string", }, + "initrd_md5sum": { + "description": "QEMU initrd path", + "type": ["string", "null"], + }, "kernel_image": { "description": "QEMU kernel image path", "type": "string", }, + "kernel_image_md5sum": { + "description": "QEMU kernel image checksum", + "type": ["string", "null"], + }, "kernel_command_line": { "description": "QEMU kernel command line", "type": "string", @@ -367,9 +391,10 @@ QEMU_OBJECT_SCHEMA = { }, "additionalProperties": False, "required": ["vm_id", "project_id", "name", "qemu_path", "platform", "hda_disk_image", "hdb_disk_image", - "hdc_disk_image", "hdd_disk_image", "ram", "adapters", "adapter_type", "mac_address", "console", - "initrd", "kernel_image", "kernel_command_line", "legacy_networking", "acpi_shutdown", "kvm", - "cpu_throttling", "process_priority", "options"] + "hdc_disk_image", "hdd_disk_image", "hda_disk_image_md5sum", "hdb_disk_image_md5sum", + "hdc_disk_image_md5sum", "hdd_disk_image_md5sum", "ram", "adapters", "adapter_type", "mac_address", + "console", "initrd", "kernel_image", "initrd_md5sum", "kernel_image_md5sum", "kernel_command_line", + "legacy_networking", "acpi_shutdown", "kvm", "cpu_throttling", "process_priority", "options"] } QEMU_BINARY_LIST_SCHEMA = { diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py new file mode 100644 index 00000000..3d45092c --- /dev/null +++ b/gns3server/utils/images.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2014 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import hashlib + + +def md5sum(path): + """ + Return the md5sum of an image and cache it on disk + + :param path: Path to the image + :returns: Digest of the image + """ + + if path is None or len(path) == 0: + return None + + try: + with open(path + '.md5sum') as f: + return f.read() + except OSError: + pass + + m = hashlib.md5() + with open(path, 'rb') as f: + while True: + buf = f.read(128) + if not buf: + break + m.update(buf) + digest = m.hexdigest() + + try: + with open('{}.md5sum'.format(path), 'w+') as f: + f.write(digest) + except OSError as e: + log.error("Can't write digest of %s: %s", path, str(e)) + + return digest + + +def remove_checksum(path): + """ + Remove the checksum of an image from cache if exists + """ + + path = '{}.md5sum'.format(path) + if os.path.exists(path): + os.remove(path) diff --git a/tests/handlers/api/test_dynamips.py b/tests/handlers/api/test_dynamips.py index 9deb65af..4fb5b704 100644 --- a/tests/handlers/api/test_dynamips.py +++ b/tests/handlers/api/test_dynamips.py @@ -156,6 +156,10 @@ def test_upload_vm(server, tmpdir): with open(str(tmpdir / "test2")) as f: assert f.read() == "TEST" + with open(str(tmpdir / "test2.md5sum")) as f: + checksum = f.read() + assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf" + def test_upload_vm_permission_denied(server, tmpdir): with open(str(tmpdir / "test2"), "w+") as f: diff --git a/tests/handlers/api/test_iou.py b/tests/handlers/api/test_iou.py index de6e0a22..169ff4ee 100644 --- a/tests/handlers/api/test_iou.py +++ b/tests/handlers/api/test_iou.py @@ -301,6 +301,7 @@ def test_get_configs_without_configs_file(server, vm): assert "startup_config" not in response.json assert "private_config" not in response.json + def test_get_configs_with_startup_config_file(server, project, vm): path = startup_config_file(project, vm) @@ -328,6 +329,10 @@ def test_upload_vm(server, tmpdir): with open(str(tmpdir / "test2")) as f: assert f.read() == "TEST" + with open(str(tmpdir / "test2.md5sum")) as f: + checksum = f.read() + assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf" + def test_upload_vm_permission_denied(server, tmpdir): with open(str(tmpdir / "test2"), "w+") as f: diff --git a/tests/handlers/api/test_qemu.py b/tests/handlers/api/test_qemu.py index c91fa792..a5e8ab81 100644 --- a/tests/handlers/api/test_qemu.py +++ b/tests/handlers/api/test_qemu.py @@ -88,10 +88,10 @@ def test_qemu_create_platform(server, project, base_params, fake_qemu_bin): assert response.json["platform"] == "x86_64" -def test_qemu_create_with_params(server, project, base_params): +def test_qemu_create_with_params(server, project, base_params, fake_qemu_vm): params = base_params params["ram"] = 1024 - params["hda_disk_image"] = "/tmp/hda" + params["hda_disk_image"] = fake_qemu_vm response = server.post("/projects/{project_id}/qemu/vms".format(project_id=project.id), params, example=True) assert response.status == 201 @@ -99,7 +99,7 @@ def test_qemu_create_with_params(server, project, base_params): assert response.json["name"] == "PC TEST 1" assert response.json["project_id"] == project.id assert response.json["ram"] == 1024 - assert response.json["hda_disk_image"] == "/tmp/hda" + assert response.json["hda_disk_image"] == fake_qemu_vm def test_qemu_get(server, project, vm): @@ -152,18 +152,18 @@ def test_qemu_delete(server, vm): assert response.status == 204 -def test_qemu_update(server, vm, tmpdir, free_console_port, project): +def test_qemu_update(server, vm, tmpdir, free_console_port, project, fake_qemu_vm): params = { "name": "test", "console": free_console_port, "ram": 1024, - "hdb_disk_image": "/tmp/hdb" + "hdb_disk_image": fake_qemu_vm } response = server.put("/projects/{project_id}/qemu/vms/{vm_id}".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), params, example=True) assert response.status == 200 assert response.json["name"] == "test" assert response.json["console"] == free_console_port - assert response.json["hdb_disk_image"] == "/tmp/hdb" + assert response.json["hdb_disk_image"] == fake_qemu_vm assert response.json["ram"] == 1024 @@ -225,6 +225,10 @@ def test_upload_vm(server, tmpdir): with open(str(tmpdir / "test2")) as f: assert f.read() == "TEST" + with open(str(tmpdir / "test2.md5sum")) as f: + checksum = f.read() + assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf" + def test_upload_vm_permission_denied(server, tmpdir): with open(str(tmpdir / "test2"), "w+") as f: diff --git a/tests/handlers/test_upload.py b/tests/handlers/test_upload.py index b108668c..ffeb009d 100644 --- a/tests/handlers/test_upload.py +++ b/tests/handlers/test_upload.py @@ -21,12 +21,23 @@ import os from unittest.mock import patch from gns3server.config import Config -def test_index_upload(server): + +def test_index_upload(server, tmpdir): + + Config.instance().set("Server", "images_path", str(tmpdir)) + + open(str(tmpdir / "alpha"), "w+").close() + open(str(tmpdir / "alpha.md5sum"), "w+").close() + open(str(tmpdir / ".beta"), "w+").close() + response = server.get('/upload', api_version=None) assert response.status == 200 html = response.html assert "GNS3 Server" in html assert "Select & Upload" in html + assert "alpha" in html + assert ".beta" not in html + assert "alpha.md5sum" not in html def test_upload(server, tmpdir): @@ -40,9 +51,43 @@ def test_upload(server, tmpdir): body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2") Config.instance().set("Server", "images_path", str(tmpdir)) + response = server.post('/upload', api_version=None, body=body, raw=True) + assert "test2" in response.body.decode("utf-8") + with open(str(tmpdir / "QEMU" / "test2")) as f: assert f.read() == content + with open(str(tmpdir / "QEMU" / "test2.md5sum")) as f: + checksum = f.read() + assert checksum == "ae187e1febee2a150b64849c32d566ca" + + +def test_upload_previous_checksum(server, tmpdir): + + content = ''.join(['a' for _ in range(0, 1025)]) + + with open(str(tmpdir / "test"), "w+") as f: + f.write(content) + body = aiohttp.FormData() + body.add_field("type", "QEMU") + body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2") + + Config.instance().set("Server", "images_path", str(tmpdir)) + + os.makedirs(str(tmpdir / "QEMU")) + + with open(str(tmpdir / "QEMU" / "test2.md5sum"), 'w+') as f: + f.write("FAKE checksum") + + response = server.post('/upload', api_version=None, body=body, raw=True) + assert "test2" in response.body.decode("utf-8") + + with open(str(tmpdir / "QEMU" / "test2")) as f: + assert f.read() == content + + with open(str(tmpdir / "QEMU" / "test2.md5sum")) as f: + checksum = f.read() + assert checksum == "ae187e1febee2a150b64849c32d566ca" diff --git a/tests/modules/test_manager.py b/tests/modules/test_manager.py index 2a0e94d2..4ad3b31a 100644 --- a/tests/modules/test_manager.py +++ b/tests/modules/test_manager.py @@ -125,7 +125,7 @@ def test_get_relative_image_path(qemu, tmpdir): def test_list_images(loop, qemu, tmpdir): - fake_images = ["a.bin", "b.bin", ".blu.bin"] + fake_images = ["a.bin", "b.bin", ".blu.bin", "a.bin.md5sum"] for image in fake_images: with open(str(tmpdir / image), "w+") as f: f.write("1") diff --git a/tests/utils/test_images.py b/tests/utils/test_images.py new file mode 100644 index 00000000..d13bda69 --- /dev/null +++ b/tests/utils/test_images.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2014 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from gns3server.utils.images import md5sum, remove_checksum + + +def test_md5sum(tmpdir): + fake_img = str(tmpdir / 'hello') + + with open(fake_img, 'w+') as f: + f.write('hello') + + assert md5sum(fake_img) == '5d41402abc4b2a76b9719d911017c592' + with open(str(tmpdir / 'hello.md5sum')) as f: + assert f.read() == '5d41402abc4b2a76b9719d911017c592' + + +def test_md5sum_existing_digest(tmpdir): + fake_img = str(tmpdir / 'hello') + + with open(str(tmpdir / 'hello.md5sum'), 'w+') as f: + f.write('aaaaa02abc4b2a76b9719d911017c592') + + assert md5sum(fake_img) == 'aaaaa02abc4b2a76b9719d911017c592' + + +def test_md5sum_none(tmpdir): + assert md5sum(None) is None + + +def test_remove_checksum(tmpdir): + + with open(str(tmpdir / 'hello.md5sum'), 'w+') as f: + f.write('aaaaa02abc4b2a76b9719d911017c592') + remove_checksum(str(tmpdir / 'hello')) + + assert not os.path.exists(str(tmpdir / 'hello.md5sum')) + + remove_checksum(str(tmpdir / 'not_exists'))