From 24bfc205db85c5d0f40401f34dd861d9de4eed4c Mon Sep 17 00:00:00 2001 From: grossmj Date: Sat, 19 Oct 2024 15:49:23 +1000 Subject: [PATCH] Symbolic links support for project export/import --- gns3server/controller/export_project.py | 23 +++++------ gns3server/controller/import_project.py | 21 ++++++++++ gns3server/controller/project.py | 2 +- gns3server/utils/asyncio/aiozipstream.py | 24 ++++++----- tests/controller/test_export_project.py | 8 +++- tests/controller/test_import_project.py | 52 ++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 25 deletions(-) diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index 7f62be18..a2d6b5a8 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -70,14 +70,15 @@ async def export_project(zstream, project, temporary_dir, include_images=False, files = [f for f in files if _is_exportable(os.path.join(root, f), include_snapshots)] for file in files: path = os.path.join(root, file) - # check if we can export the file - try: - open(path).close() - except OSError as e: - msg = "Could not export file {}: {}".format(path, e) - log.warning(msg) - project.emit_notification("log.warning", {"message": msg}) - continue + if not os.path.islink(path): + try: + # check if we can export the file + open(path).close() + except OSError as e: + msg = "Could not export file {}: {}".format(path, e) + log.warning(msg) + project.emit_notification("log.warning", {"message": msg}) + continue # ignore the .gns3 file if file.endswith(".gns3"): continue @@ -128,7 +129,7 @@ def _patch_mtime(path): if sys.platform.startswith("win"): # only UNIX type platforms return - st = os.stat(path) + st = os.stat(path, follow_symlinks=False) file_date = datetime.fromtimestamp(st.st_mtime) if file_date.year < 1980: new_mtime = file_date.replace(year=1980).timestamp() @@ -144,10 +145,6 @@ def _is_exportable(path, include_snapshots=False): if include_snapshots is False and path.endswith("snapshots"): return False - # do not export symlinks - if os.path.islink(path): - return False - # do not export directories of snapshots if include_snapshots is False and "{sep}snapshots{sep}".format(sep=os.path.sep) in path: return False diff --git a/gns3server/controller/import_project.py b/gns3server/controller/import_project.py index 9d0e0dea..d7e47980 100644 --- a/gns3server/controller/import_project.py +++ b/gns3server/controller/import_project.py @@ -17,6 +17,7 @@ import os import sys +import stat import json import uuid import shutil @@ -93,6 +94,7 @@ async def import_project(controller, project_id, stream, location=None, name=Non try: with zipfile.ZipFile(stream) as zip_file: await wait_run_in_executor(zip_file.extractall, path) + _create_symbolic_links(zip_file, path) except zipfile.BadZipFile: raise aiohttp.web.HTTPConflict(text="Cannot extract files from GNS3 project (invalid zip)") @@ -174,6 +176,24 @@ async def import_project(controller, project_id, stream, location=None, name=Non project = await controller.load_project(dot_gns3_path, load=False) return project +def _create_symbolic_links(zip_file, path): + """ + Manually create symbolic links (if any) because ZipFile does not support it. + + :param zip_file: ZipFile instance + :param path: project location + """ + + for zip_info in zip_file.infolist(): + if stat.S_ISLNK(zip_info.external_attr >> 16): + symlink_target = zip_file.read(zip_info.filename).decode() + symlink_path = os.path.join(path, zip_info.filename) + try: + # remove the regular file and replace it by a symbolic link + os.remove(symlink_path) + os.symlink(symlink_target, symlink_path) + except OSError as e: + raise aiohttp.web.HTTPConflict(text=f"Cannot create symbolic link: {e}") def _move_node_file(path, old_id, new_id): """ @@ -257,6 +277,7 @@ async def _import_snapshots(snapshots_path, project_name, project_id): with open(snapshot_path, "rb") as f: with zipfile.ZipFile(f) as zip_file: await wait_run_in_executor(zip_file.extractall, tmpdir) + _create_symbolic_links(zip_file, tmpdir) except OSError as e: raise aiohttp.web.HTTPConflict(text="Cannot open snapshot '{}': {}".format(os.path.basename(snapshot), e)) except zipfile.BadZipFile: diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 3c5c5229..b1651c1a 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -1277,7 +1277,7 @@ class Project: new_project_id = str(uuid.uuid4()) new_project_path = p_work.joinpath(new_project_id) # copy dir - await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix()) + await wait_run_in_executor(shutil.copytree, self.path, new_project_path.as_posix(), symlinks=True, ignore_dangling_symlinks=True) log.info("Project content copied from '{}' to '{}' in {}s".format(self.path, new_project_path, time.time() - t0)) topology = json.loads(new_project_path.joinpath('{}.gns3'.format(self.name)).read_bytes()) project_name = name or topology["name"] diff --git a/gns3server/utils/asyncio/aiozipstream.py b/gns3server/utils/asyncio/aiozipstream.py index cbff3aff..633a08d0 100644 --- a/gns3server/utils/asyncio/aiozipstream.py +++ b/gns3server/utils/asyncio/aiozipstream.py @@ -161,14 +161,17 @@ class ZipFile(zipfile.ZipFile): self._comment = comment self._didModify = True - async def data_generator(self, path): + async def data_generator(self, path, islink=False): - async with aiofiles.open(path, "rb") as f: - while True: - part = await f.read(self._chunksize) - if not part: - break - yield part + if islink: + yield os.readlink(path).encode() + else: + async with aiofiles.open(path, "rb") as f: + while True: + part = await f.read(self._chunksize) + if not part: + break + yield part return async def _run_in_executor(self, task, *args, **kwargs): @@ -224,12 +227,13 @@ class ZipFile(zipfile.ZipFile): raise ValueError("either (exclusively) filename or iterable shall be not None") if filename: - st = os.stat(filename) + st = os.stat(filename, follow_symlinks=False) isdir = stat.S_ISDIR(st.st_mode) + islink = stat.S_ISLNK(st.st_mode) mtime = time.localtime(st.st_mtime) date_time = mtime[0:6] else: - st, isdir, date_time = None, False, time.localtime()[0:6] + st, isdir, islink, date_time = None, False, False, time.localtime()[0:6] # Create ZipInfo instance to store file information if arcname is None: arcname = filename @@ -282,7 +286,7 @@ class ZipFile(zipfile.ZipFile): file_size = 0 if filename: - async for buf in self.data_generator(filename): + async for buf in self.data_generator(filename, islink): file_size = file_size + len(buf) CRC = zipfile.crc32(buf, CRC) & 0xffffffff if cmpr: diff --git a/tests/controller/test_export_project.py b/tests/controller/test_export_project.py index 80cf4cc3..de6335a4 100644 --- a/tests/controller/test_export_project.py +++ b/tests/controller/test_export_project.py @@ -21,6 +21,7 @@ import json import pytest import aiohttp import zipfile +import stat from pathlib import Path from unittest.mock import patch @@ -116,6 +117,8 @@ async def test_export(tmpdir, project): with open(os.path.join(path, "project-files", "snapshots", "test"), 'w+') as f: f.write("WORLD") + os.symlink("/tmp/anywhere", os.path.join(path, "vm-1", "dynamips", "symlink")) + with aiozipstream.ZipFile() as z: with patch("gns3server.compute.Dynamips.get_images_directory", return_value=str(tmpdir / "IOS"),): await export_project(z, project, str(tmpdir), include_images=False) @@ -131,9 +134,12 @@ async def test_export(tmpdir, project): assert 'vm-1/dynamips/empty-dir/' in myzip.namelist() assert 'project-files/snapshots/test' not in myzip.namelist() assert 'vm-1/dynamips/test_log.txt' not in myzip.namelist() - assert 'images/IOS/test.image' not in myzip.namelist() + assert 'vm-1/dynamips/symlink' in myzip.namelist() + zip_info = myzip.getinfo('vm-1/dynamips/symlink') + assert stat.S_ISLNK(zip_info.external_attr >> 16) + with myzip.open("project.gns3") as myfile: topo = json.loads(myfile.read().decode())["topology"] assert topo["nodes"][0]["compute_id"] == "local" # All node should have compute_id local after export diff --git a/tests/controller/test_import_project.py b/tests/controller/test_import_project.py index 9917046e..b70a360a 100644 --- a/tests/controller/test_import_project.py +++ b/tests/controller/test_import_project.py @@ -21,7 +21,11 @@ import json import zipfile from tests.utils import asyncio_patch, AsyncioMagicMock +from unittest.mock import patch, MagicMock +from gns3server.utils.asyncio import aiozipstream +from gns3server.controller.project import Project +from gns3server.controller.export_project import export_project from gns3server.controller.import_project import import_project, _move_files_to_compute from gns3server.version import __version__ @@ -106,6 +110,54 @@ async def test_import_project_override(tmpdir, controller): assert project.name == "test" +async def write_file(path, z): + + with open(path, 'wb') as f: + async for chunk in z: + f.write(chunk) + + +async def test_import_project_containing_symlink(tmpdir, controller): + + project = Project(controller=controller, name="test") + project.dump = MagicMock() + path = project.path + + project_id = str(uuid.uuid4()) + topology = { + "project_id": str(uuid.uuid4()), + "name": "test", + "auto_open": True, + "auto_start": True, + "topology": { + }, + "version": "2.0.0" + } + + with open(os.path.join(path, "project.gns3"), 'w+') as f: + json.dump(topology, f) + + os.makedirs(os.path.join(path, "vm1", "dynamips")) + symlink_path = os.path.join(project.path, "vm1", "dynamips", "symlink") + symlink_target = "/tmp/anywhere" + os.symlink(symlink_target, symlink_path) + + zip_path = str(tmpdir / "project.zip") + with aiozipstream.ZipFile() as z: + with patch("gns3server.compute.Dynamips.get_images_directory", return_value=str(tmpdir / "IOS"),): + await export_project(z, project, str(tmpdir), include_images=False) + await write_file(zip_path, z) + + with open(zip_path, "rb") as f: + project = await import_project(controller, project_id, f) + + assert project.name == "test" + assert project.id == project_id + symlink_path = os.path.join(project.path, "vm1", "dynamips", "symlink") + assert os.path.islink(symlink_path) + assert os.readlink(symlink_path) == symlink_target + + async def test_import_upgrade(tmpdir, controller): """ Topology made for previous GNS3 version are upgraded during the process