From 08e482004fd58956981f4300524001c2848c7afe Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Tue, 7 Jun 2016 19:38:01 +0200 Subject: [PATCH] Auto upload missing images --- gns3server/compute/base_manager.py | 34 +++++---------------- gns3server/compute/dynamips/__init__.py | 1 + gns3server/compute/error.py | 1 + gns3server/compute/iou/__init__.py | 1 + gns3server/compute/qemu/__init__.py | 1 + gns3server/controller/compute.py | 31 +++++++++++++++---- gns3server/controller/node.py | 37 +++++++++++++++++++++-- gns3server/utils/images.py | 34 +++++++++++++++++++++ gns3server/web/route.py | 7 ++++- tests/compute/test_manager.py | 22 -------------- tests/controller/test_compute.py | 16 ++++++++-- tests/controller/test_node.py | 40 +++++++++++++++++++++++-- tests/utils.py | 5 +++- tests/utils/test_images.py | 27 ++++++++++++++++- tests/utils/test_path.py | 2 ++ 15 files changed, 194 insertions(+), 65 deletions(-) diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index 6574b8ca..ac1d6e92 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -38,7 +38,7 @@ from .project_manager import ProjectManager from .nios.nio_udp import NIOUDP from .nios.nio_tap import NIOTAP from .nios.nio_ethernet import NIOEthernet -from ..utils.images import md5sum, remove_checksum +from ..utils.images import md5sum, remove_checksum, images_directories from .error import NodeError, ImageMissingError @@ -389,24 +389,6 @@ class BaseManager: assert nio is not None return nio - def images_directories(self): - """ - Return all directory where we will look for images - by priority - """ - server_config = self.config.get_section_config("Server") - - paths = [] - img_directory = self.get_images_directory() - os.makedirs(img_directory, exist_ok=True) - paths.append(img_directory) - for directory in server_config.get("additional_images_path", "").split(":"): - paths.append(directory) - # Compatibility with old topologies we look in parent directory - paths.append(os.path.normpath(os.path.join(self.get_images_directory(), '..'))) - # Return only the existings paths - return [force_unix_path(p) for p in paths if os.path.exists(p)] - def get_abs_image_path(self, path): """ Get the absolute path of an image @@ -417,6 +399,7 @@ class BaseManager: if not path: return "" + orig_path = path server_config = self.config.get_section_config("Server") img_directory = self.get_images_directory() @@ -427,8 +410,7 @@ class BaseManager: raise NodeError("{} is not allowed on this remote server. Please use only a filename in {}.".format(path, img_directory)) if not os.path.isabs(path): - orig_path = path - for directory in self.images_directories(): + for directory in images_directories(self._NODE_TYPE): path = self._recursive_search_file_in_directory(directory, orig_path) if path: return force_unix_path(path) @@ -438,21 +420,21 @@ class BaseManager: path = force_unix_path(os.path.join(self.get_images_directory(), *s)) if os.path.exists(path): return path - raise ImageMissingError(path) + raise ImageMissingError(orig_path) # For non local server we disallow using absolute path outside image directory if server_config.get("local", False) is True: path = force_unix_path(path) if os.path.exists(path): return path - raise ImageMissingError(path) + raise ImageMissingError(orig_path) path = force_unix_path(path) - for directory in self.images_directories(): + for directory in images_directories(self._NODE_TYPE): if os.path.commonprefix([directory, path]) == directory: if os.path.exists(path): return path - raise ImageMissingError(path) + raise ImageMissingError(orig_path) raise NodeError("{} is not allowed on this remote server. Please use only a filename in {}.".format(path, self.get_images_directory())) def _recursive_search_file_in_directory(self, directory, searched_file): @@ -485,7 +467,7 @@ class BaseManager: if not path: return "" path = force_unix_path(self.get_abs_image_path(path)) - for directory in self.images_directories(): + for directory in images_directories(self._NODE_TYPE): if os.path.commonprefix([directory, path]) == directory: return os.path.relpath(path, directory) return path diff --git a/gns3server/compute/dynamips/__init__.py b/gns3server/compute/dynamips/__init__.py index 548619f6..56676e5d 100644 --- a/gns3server/compute/dynamips/__init__.py +++ b/gns3server/compute/dynamips/__init__.py @@ -103,6 +103,7 @@ WIC_MATRIX = {"WIC-1ENET": WIC_1ENET, class Dynamips(BaseManager): _NODE_CLASS = DynamipsVMFactory + _NODE_TYPE = "dynamips" _DEVICE_CLASS = DynamipsDeviceFactory _ghost_ios_lock = None diff --git a/gns3server/compute/error.py b/gns3server/compute/error.py index 3666205c..f7d8b52e 100644 --- a/gns3server/compute/error.py +++ b/gns3server/compute/error.py @@ -39,3 +39,4 @@ class ImageMissingError(Exception): def __init__(self, image): super().__init__("The image {} is missing".format(image)) + self.image = image diff --git a/gns3server/compute/iou/__init__.py b/gns3server/compute/iou/__init__.py index 59f1e10d..7d7ff0f9 100644 --- a/gns3server/compute/iou/__init__.py +++ b/gns3server/compute/iou/__init__.py @@ -33,6 +33,7 @@ log = logging.getLogger(__name__) class IOU(BaseManager): _NODE_CLASS = IOUVM + _NODE_TYPE = "iou" def __init__(self): diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 58d3fe82..68dd0020 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -38,6 +38,7 @@ log = logging.getLogger(__name__) class Qemu(BaseManager): _NODE_CLASS = QemuVM + _NODE_TYPE = "qemu" @staticmethod @asyncio.coroutine diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 69f44591..c9956e62 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -19,6 +19,7 @@ import aiohttp import asyncio import json import uuid +import io from ..utils import parse_version from ..controller.controller_error import ControllerError @@ -34,6 +35,17 @@ class ComputeError(ControllerError): pass +class ComputeConflict(aiohttp.web.HTTPConflict): + """ + Raise when the compute send a 409 that we can handle + + :param response: The response of the compute + """ + def __init__(self, response): + super().__init__(text=response["message"]) + self.response = response + + class Timeout(aiohttp.Timeout): """ Could be removed with aiohttp 0.22 that support None timeout @@ -293,7 +305,7 @@ class Compute: if hasattr(data, '__json__'): data = json.dumps(data.__json__()) # Stream the request - elif isinstance(data, aiohttp.streams.StreamReader): + elif isinstance(data, aiohttp.streams.StreamReader) or isinstance(data, io.BufferedIOBase): chunked = True headers['content-type'] = 'application/octet-stream' else: @@ -306,10 +318,13 @@ class Compute: if response.status >= 300: # Try to decode the GNS3 error - try: - msg = json.loads(body)["message"] - except (KeyError, json.decoder.JSONDecodeError): - msg = body + if body: + try: + msg = json.loads(body)["message"] + except (KeyError, json.decoder.JSONDecodeError): + msg = body + else: + msg = "" if response.status == 400: raise aiohttp.web.HTTPBadRequest(text="Bad request {} {}".format(url, body)) @@ -320,7 +335,11 @@ class Compute: elif response.status == 404: raise aiohttp.web.HTTPNotFound(text=msg) elif response.status == 409: - raise aiohttp.web.HTTPConflict(text=msg) + try: + raise ComputeConflict(json.loads(body)) + # If the 409 doesn't come from a GNS3 server + except json.decoder.JSONDecodeError: + raise aiohttp.web.HTTPConflict(text=msg) elif response.status == 500: raise aiohttp.web.HTTPInternalServerError(text="Internal server error {}".format(url)) elif response.status == 503: diff --git a/gns3server/controller/node.py b/gns3server/controller/node.py index 3d37354b..b8ca9ee4 100644 --- a/gns3server/controller/node.py +++ b/gns3server/controller/node.py @@ -15,12 +15,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - +import aiohttp import asyncio import copy import uuid +import os +from .compute import ComputeConflict +from ..utils.images import images_directories + class Node: def __init__(self, project, compute, node_id=None, node_type=None, name=None, console=None, console_type=None, properties={}): @@ -101,8 +105,19 @@ class Node: """ data = self._node_data() data["node_id"] = self._id - response = yield from self._compute.post("/projects/{}/{}/nodes".format(self._project.id, self._node_type), data=data) - self.parse_node_response(response.json) + trial = 0 + while trial != 6: + try: + response = yield from self._compute.post("/projects/{}/{}/nodes".format(self._project.id, self._node_type), data=data) + except ComputeConflict as e: + if e.response.get("exception") == "ImageMissingError": + res = yield from self._upload_missing_image(self._node_type, e.response["image"]) + if not res: + raise e + else: + self.parse_node_response(response.json) + return True + trial += 1 @asyncio.coroutine def update(self, name=None, console=None, console_type=None, properties={}): @@ -236,6 +251,22 @@ class Node: else: return (yield from self._compute.delete("/projects/{}/{}/nodes/{}{}".format(self._project.id, self._node_type, self._id, path))) + @asyncio.coroutine + def _upload_missing_image(self, type, img): + """ + Search an image on local computer and upload it to remote compute + if the image exists + """ + for directory in images_directories(type): + image = os.path.join(directory, img) + if os.path.exists(image): + self.project.controller.notification.emit("log.info", {"message": "Uploading missing image {}".format(img)}) + with open(image, 'rb') as f: + yield from self._compute.post("/{}/images/{}".format(self._node_type, os.path.basename(img)), data=f, timeout=None) + self.project.controller.notification.emit("log.info", {"message": "Upload finished for {}".format(img)}) + return True + return False + @asyncio.coroutine def dynamips_auto_idlepc(self): """ diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py index 02055a65..c9de9a96 100644 --- a/gns3server/utils/images.py +++ b/gns3server/utils/images.py @@ -18,10 +18,44 @@ import os import hashlib + +from ..config import Config +from . import force_unix_path + + import logging log = logging.getLogger(__name__) +def images_directories(type): + """ + Return all directory where we will look for images + by priority + + :param type: Type of emulator + """ + server_config = Config.instance().get_section_config("Server") + + paths = [] + img_dir = os.path.expanduser(server_config.get("images_path", "~/GNS3/images")) + if type == "qemu": + type_img_directory = os.path.join(img_dir, "QEMU") + elif type == "iou": + type_img_directory = os.path.join(img_dir, "IOU") + elif type == "dynamips": + type_img_directory = os.path.join(img_dir, "IOS") + else: + raise NotImplementedError("%s is not supported", type) + os.makedirs(type_img_directory, exist_ok=True) + paths.append(type_img_directory) + for directory in server_config.get("additional_images_path", "").split(":"): + paths.append(directory) + # Compatibility with old topologies we look in parent directory + paths.append(img_dir) + # Return only the existings paths + return [force_unix_path(p) for p in paths if os.path.exists(p)] + + def md5sum(path): """ Return the md5sum of an image and cache it on disk diff --git a/gns3server/web/route.py b/gns3server/web/route.py index 8cba4e7e..6e594749 100644 --- a/gns3server/web/route.py +++ b/gns3server/web/route.py @@ -197,11 +197,16 @@ class Route(object): response = Response(request=request, route=route) response.set_status(409) response.json({"message": str(e), "status": 409}) - except (NodeError, UbridgeError, ImageMissingError) as e: + except (NodeError, UbridgeError) as e: log.error("Node error detected: {type}".format(type=e.__class__.__name__), exc_info=1) response = Response(request=request, route=route) response.set_status(409) response.json({"message": str(e), "status": 409, "exception": e.__class__.__name__}) + except (ImageMissingError) as e: + log.error("Image missing error detected: {}".format(e.image)) + response = Response(request=request, route=route) + response.set_status(409) + response.json({"message": str(e), "status": 409, "image": e.image, "exception": e.__class__.__name__}) except asyncio.futures.CancelledError as e: log.error("Request canceled") response = Response(request=request, route=route) diff --git a/tests/compute/test_manager.py b/tests/compute/test_manager.py index 44267900..5121ba10 100644 --- a/tests/compute/test_manager.py +++ b/tests/compute/test_manager.py @@ -88,28 +88,6 @@ def test_create_node_old_topology(loop, project, tmpdir, vpcs): assert f.read() == "1" -def test_images_directories(qemu, tmpdir): - path1 = tmpdir / "images1" / "QEMU" / "test1.bin" - path1.write("1", ensure=True) - path1 = force_unix_path(str(path1)) - - path2 = tmpdir / "images2" / "test2.bin" - path2.write("1", ensure=True) - path2 = force_unix_path(str(path2)) - - with patch("gns3server.config.Config.get_section_config", return_value={ - "images_path": str(tmpdir / "images1"), - "additional_images_path": "/tmp/null24564:{}".format(tmpdir / "images2"), - "local": False}): - - # /tmp/null24564 is ignored because doesn't exists - res = qemu.images_directories() - assert res[0] == str(tmpdir / "images1" / "QEMU") - assert res[1] == str(tmpdir / "images2") - assert res[2] == str(tmpdir / "images1") - assert len(res) == 3 - - def test_get_abs_image_path(qemu, tmpdir): os.makedirs(str(tmpdir / "QEMU")) path1 = force_unix_path(str(tmpdir / "test1.bin")) diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py index 5755d9c7..ce02ed29 100644 --- a/tests/controller/test_compute.py +++ b/tests/controller/test_compute.py @@ -23,7 +23,7 @@ import asyncio from unittest.mock import patch, MagicMock from gns3server.controller.project import Project -from gns3server.controller.compute import Compute, ComputeError +from gns3server.controller.compute import Compute, ComputeError, ComputeConflict from gns3server.version import __version__ from tests.utils import asyncio_patch, AsyncioMagicMock @@ -139,9 +139,19 @@ def test_compute_httpQueryNotConnectedNonGNS3Server2(compute, async_run): def test_compute_httpQueryError(compute, async_run): response = MagicMock() with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock: - response.status = 409 + response.status = 404 - with pytest.raises(aiohttp.web.HTTPConflict): + with pytest.raises(aiohttp.web.HTTPNotFound): + async_run(compute.post("/projects", {"a": "b"})) + + +def test_compute_httpQueryConflictError(compute, async_run): + response = MagicMock() + with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock: + response.status = 409 + response.read = AsyncioMagicMock(return_value=b'{"message": "Test"}') + + with pytest.raises(ComputeConflict): async_run(compute.post("/projects", {"a": "b"})) diff --git a/tests/controller/test_node.py b/tests/controller/test_node.py index 58a706a1..a001ad2a 100644 --- a/tests/controller/test_node.py +++ b/tests/controller/test_node.py @@ -18,7 +18,8 @@ import pytest import uuid import asyncio -from unittest.mock import MagicMock +import os +from unittest.mock import MagicMock, ANY from tests.utils import AsyncioMagicMock @@ -77,7 +78,7 @@ def test_create(node, compute, project, async_run): response.json = {"console": 2048} compute.post = AsyncioMagicMock(return_value=response) - async_run(node.create()) + assert async_run(node.create()) is True data = { "console": 2048, "console_type": "vnc", @@ -90,6 +91,28 @@ def test_create(node, compute, project, async_run): assert node._properties == {"startup_script": "echo test"} +def test_create_image_missing(node, compute, project, async_run): + node._console = 2048 + + node.__calls = 0 + @asyncio.coroutine + def resp(*args, **kwargs): + node.__calls += 1 + response = MagicMock() + if node.__calls == 1: + response.status = 409 + response.json = {"image": "linux.img", "exception": "ImageMissingError"} + else: + response.status = 200 + return response + + compute.post = AsyncioMagicMock(side_effect=resp) + node._upload_missing_image = AsyncioMagicMock(return_value=True) + + assert async_run(node.create()) is True + node._upload_missing_image.called is True + + def test_update(node, compute, project, async_run): response = MagicMock() response.json = {"console": 2048} @@ -193,3 +216,16 @@ def test_dynamips_idlepc_proposals(node, async_run, compute): async_run(node.dynamips_idlepc_proposals()) compute.get.assert_called_with("/projects/{}/dynamips/nodes/{}/idlepc_proposals".format(node.project.id, node.id), timeout=240) + + +def test_upload_missing_image(compute, controller, async_run, images_dir): + project = Project(str(uuid.uuid4()), controller=controller) + node = Node(project, compute, + name="demo", + node_id=str(uuid.uuid4()), + node_type="qemu", + properties={"hda_disk_image": "linux.img"}) + open(os.path.join(images_dir, "linux.img"), 'w+').close() + assert async_run(node._upload_missing_image("qemu", "linux.img")) is True + compute.post.assert_called_with("/qemu/images/linux.img", data=ANY, timeout=None) + diff --git a/tests/utils.py b/tests/utils.py index 70ce6e01..72c0152a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -70,7 +70,10 @@ class AsyncioMagicMock(unittest.mock.MagicMock): Magic mock returning coroutine """ - def __init__(self, return_value=None, **kwargs): + def __init__(self, return_value=None, return_values=None, **kwargs): + """ + :return_values: Array of return value at each call will return the next + """ if return_value: future = asyncio.Future() future.set_result(return_value) diff --git a/tests/utils/test_images.py b/tests/utils/test_images.py index e3f80fc0..725da532 100644 --- a/tests/utils/test_images.py +++ b/tests/utils/test_images.py @@ -16,8 +16,33 @@ # along with this program. If not, see . import os +from unittest.mock import patch -from gns3server.utils.images import md5sum, remove_checksum + +from gns3server.utils import force_unix_path +from gns3server.utils.images import md5sum, remove_checksum, images_directories + + +def test_images_directories(tmpdir): + path1 = tmpdir / "images1" / "QEMU" / "test1.bin" + path1.write("1", ensure=True) + path1 = force_unix_path(str(path1)) + + path2 = tmpdir / "images2" / "test2.bin" + path2.write("1", ensure=True) + path2 = force_unix_path(str(path2)) + + with patch("gns3server.config.Config.get_section_config", return_value={ + "images_path": str(tmpdir / "images1"), + "additional_images_path": "/tmp/null24564:{}".format(tmpdir / "images2"), + "local": False}): + + # /tmp/null24564 is ignored because doesn't exists + res = images_directories("qemu") + assert res[0] == str(tmpdir / "images1" / "QEMU") + assert res[1] == str(tmpdir / "images2") + assert res[2] == str(tmpdir / "images1") + assert len(res) == 3 def test_md5sum(tmpdir): diff --git a/tests/utils/test_path.py b/tests/utils/test_path.py index 08c7042f..c0b1c3c3 100644 --- a/tests/utils/test_path.py +++ b/tests/utils/test_path.py @@ -19,7 +19,9 @@ import os import pytest import aiohttp + from gns3server.utils.path import check_path_allowed, get_default_project_directory +from gns3server.utils import force_unix_path def test_check_path_allowed(config, tmpdir):