Merge branch '2.1' into embed_appliances

This commit is contained in:
Julien Duponchelle committed 2017-02-28 14:55:33 +01:00
commit 5a399b90fe
68 files changed
+1527 -484

No files matched your search

+38 -7
View File
@@ -18,6 +18,7 @@
import os
import json
import socket
import shutil
import asyncio
import aiohttp
@@ -68,16 +69,22 @@ class Controller:
@asyncio.coroutine
def start(self):
log.info("Start controller")
yield from self.load()
self.load_base_files()
server_config = Config.instance().get_section_config("Server")
host = server_config.get("host", "localhost")
# If console_host is 0.0.0.0 client will use the ip they use
# to connect to the controller
console_host = host
if host == "0.0.0.0":
host = "127.0.0.1"
name = socket.gethostname()
if name == "gns3vm":
name = "Main server"
yield from self.add_compute(compute_id="local",
name=socket.gethostname(),
name=name,
protocol=server_config.get("protocol", "http"),
host=host,
console_host=console_host,
@@ -85,6 +92,7 @@ class Controller:
user=server_config.get("user", ""),
password=server_config.get("password", ""),
force=True)
yield from self._load_controller_settings()
yield from self.load_projects()
yield from self.gns3vm.auto_start_vm()
yield from self._project_auto_open()
@@ -131,7 +139,7 @@ class Controller:
json.dump(data, f, indent=4)
@asyncio.coroutine
def load(self):
def _load_controller_settings(self):
"""
Reload the controller configuration from disk
"""
@@ -177,6 +185,20 @@ class Controller:
except OSError as e:
log.error(str(e))
def load_base_files(self):
"""
At startup we copy base file to the user location to allow
them to customize it
"""
dst_path = self.configs_path()
src_path = get_resource('configs')
try:
for file in os.listdir(src_path):
if not os.path.exists(os.path.join(dst_path, file)):
shutil.copy(os.path.join(src_path, file), os.path.join(dst_path, file))
except OSError:
pass
def images_path(self):
"""
Get the image storage directory
@@ -186,6 +208,15 @@ class Controller:
os.makedirs(images_path, exist_ok=True)
return images_path
def configs_path(self):
"""
Get the configs storage directory
"""
server_config = Config.instance().get_section_config("Server")
images_path = os.path.expanduser(server_config.get("configs_path", "~/GNS3/projects"))
os.makedirs(images_path, exist_ok=True)
return images_path
@asyncio.coroutine
def _import_gns3_gui_conf(self):
"""
@@ -269,7 +300,7 @@ class Controller:
return None
for compute in self._computes.values():
if name and compute.name == name:
if name and compute.name == name and not force:
raise aiohttp.web.HTTPConflict(text='Compute name "{}" already exists'.format(name))
compute = Compute(compute_id=compute_id, controller=self, name=name, **kwargs)
@@ -332,7 +363,6 @@ class Controller:
try:
return self._computes[compute_id]
except KeyError:
server_config = Config.instance().get_section_config("Server")
if compute_id == "vm":
raise aiohttp.web.HTTPNotFound(text="You try to use a node on the GNS3 VM server but the GNS3 VM is not configured")
raise aiohttp.web.HTTPNotFound(text="Compute ID {} doesn't exist".format(compute_id))
@@ -383,7 +413,8 @@ class Controller:
return project
def remove_project(self, project):
del self._projects[project.id]
if project.id in self._projects:
del self._projects[project.id]
@asyncio.coroutine
def load_project(self, path, load=True):
@@ -394,7 +425,7 @@ class Controller:
:param load: Load the topology
"""
topo_data = load_topology(path)
topology = topo_data.pop("topology")
topo_data.pop("topology")
topo_data.pop("version")
topo_data.pop("revision")
topo_data.pop("type")
+22 -7
View File
@@ -22,14 +22,12 @@ import socket
import json
import uuid
import sys
import os
import io
from ..utils import parse_version
from ..utils.images import list_images, md5sum
from ..utils.images import list_images
from ..utils.asyncio import locked_coroutine
from ..controller.controller_error import ControllerError
from ..config import Config
from ..version import __version__
@@ -216,7 +214,10 @@ class Compute:
"""
Return the IP associated to the host
"""
return socket.gethostbyname(self._host)
try:
return socket.gethostbyname(self._host)
except socket.gaierror:
return '0.0.0.0'
@host.setter
def host(self, host):
@@ -360,6 +361,16 @@ class Compute:
response = yield from self._run_http_query(method, path, data=data, **kwargs)
return response
@asyncio.coroutine
def _try_reconnect(self):
"""
We catch error during reconnect
"""
try:
yield from self.connect()
except aiohttp.web.HTTPConflict:
pass
@locked_coroutine
def connect(self):
"""
@@ -374,14 +385,18 @@ class Compute:
self._connection_failure += 1
# After 5 failure we close the project using the compute to avoid sync issues
if self._connection_failure == 5:
log.warning("Can't connect to compute %s", self._id)
yield from self._controller.close_compute_projects(self)
asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self.connect()))
asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self._try_reconnect()))
return
except aiohttp.web.HTTPNotFound:
raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server or it's a 1.X server".format(self._id))
except aiohttp.web.HTTPUnauthorized:
raise aiohttp.web.HTTPConflict(text="Invalid auth for server {} ".format(self._id))
raise aiohttp.web.HTTPConflict(text="Invalid auth for server {}".format(self._id))
except aiohttp.web.HTTPServiceUnavailable:
raise aiohttp.web.HTTPConflict(text="The server {} is unavailable".format(self._id))
if "version" not in response.json:
self._http_session.close()
@@ -411,7 +426,7 @@ class Compute:
except aiohttp.errors.WSServerHandshakeError:
self._ws = None
break
if response.tp == aiohttp.MsgType.closed or response.tp == aiohttp.MsgType.error:
if response.tp == aiohttp.MsgType.closed or response.tp == aiohttp.MsgType.error or response.data is None:
self._connected = False
break
msg = json.loads(response.data)
+1
View File
@@ -43,6 +43,7 @@ class Drawing:
self._id = str(uuid.uuid4())
else:
self._id = drawing_id
self._svg = "<svg></svg>"
self.svg = svg
self._x = x
self._y = y
+5
View File
@@ -47,6 +47,9 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id
if project.is_running():
raise aiohttp.web.HTTPConflict(text="Running topology could not be exported")
# Make sure we save the project
project.dump()
z = zipstream.ZipFile(allowZip64=True)
if not os.path.exists(project._path):
@@ -136,6 +139,8 @@ def _export_project_file(project, path, z, include_images, keep_compute_id, allo
if "topology" in topology:
if "nodes" in topology["topology"]:
for node in topology["topology"]["nodes"]:
if node["node_type"] == "virtualbox" and node.get("properties", {}).get("linked_clone"):
raise aiohttp.web.HTTPConflict(text="Topology with a linked {} clone could not be exported. Use qemu instead.".format(node["node_type"]))
if not allow_all_nodes and node["node_type"] in ["virtualbox", "vmware", "cloud"]:
raise aiohttp.web.HTTPConflict(text="Topology with a {} could not be exported".format(node["node_type"]))
+10 -2
View File
@@ -222,8 +222,14 @@ class GNS3VM:
"""
engine = self._get_engine(engine)
vms = []
for vm in (yield from engine.list()):
vms.append({"vmname": vm["vmname"]})
try:
for vm in (yield from engine.list()):
vms.append({"vmname": vm["vmname"]})
except GNS3VMError as e:
# We raise error only if user activated the GNS3 VM
# otherwise you have noise when VMware is not installed
if self.enable:
raise e
return vms
@asyncio.coroutine
@@ -267,6 +273,7 @@ class GNS3VM:
engine.vmname = self._settings["vmname"]
engine.ram = self._settings["ram"]
engine.vpcus = self._settings["vcpus"]
engine.headless = self._settings["headless"]
compute = yield from self._controller.add_compute(compute_id="vm",
name="GNS3 VM is starting ({})".format(engine.vmname),
host=None,
@@ -277,6 +284,7 @@ class GNS3VM:
except Exception as e:
yield from self._controller.delete_compute("vm")
log.error("Can't start the GNS3 VM: {}", str(e))
yield from compute.update(name="GNS3 VM ({})".format(engine.vmname))
raise e
yield from compute.update(name="GNS3 VM ({})".format(engine.vmname),
protocol=self.protocol,
@@ -221,7 +221,7 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
second to a GNS3 endpoint in order to get the list of the interfaces and
their IP and after that match it with VirtualBox host only.
"""
remaining_try = 240
remaining_try = 300
while remaining_try > 0:
json_data = None
session = aiohttp.ClientSession()
+5 -2
View File
@@ -52,9 +52,11 @@ class Link:
return self._created
@asyncio.coroutine
def add_node(self, node, adapter_number, port_number, label=None):
def add_node(self, node, adapter_number, port_number, label=None, dump=True):
"""
Add a node to the link
:param dump: Dump project on disk
"""
port = node.get_port(adapter_number, port_number)
@@ -101,7 +103,8 @@ class Link:
self._created = True
self._project.controller.notification.emit("link.created", self.__json__())
self._project.dump()
if dump:
self._project.dump()
@asyncio.coroutine
def update_nodes(self, nodes):
+28 -15
View File
@@ -146,6 +146,15 @@ class Node:
def properties(self, val):
self._properties = val
def _base_config_file_content(self, path):
if not os.path.isabs(path):
path = os.path.join(self.project.controller.configs_path(), path)
try:
with open(path) as f:
return f.read()
except (PermissionError, OSError):
return None
@property
def project(self):
return self._project
@@ -366,8 +375,12 @@ class Node:
self._console_type = value
elif key == "name":
self.name = value
elif key in ["node_id", "project_id", "console_host"]:
pass
elif key in ["node_id", "project_id", "console_host",
"startup_config_content",
"private_config_content",
"startup_script"]:
if key in self._properties:
del self._properties[key]
else:
self._properties[key] = value
self._list_ports()
@@ -384,6 +397,17 @@ class Node:
data = copy.copy(properties)
else:
data = copy.copy(self._properties)
# We replace the startup script name by the content of the file
mapping = {
"base_script_file": "startup_script",
"startup_config": "startup_config_content",
"private_config": "private_config_content",
}
for k, v in mapping.items():
if k in list(self._properties.keys()):
data[v] = self._base_config_file_content(self._properties[k])
del data[k]
del self._properties[k] # We send the file only one time
data["name"] = self._name
if self._console:
# console is optional for builtin nodes
@@ -585,17 +609,6 @@ class Node:
return False
return self.id == other.id and other.project.id == self.project.id
def _filter_properties(self):
"""
Some properties are private and should not be exposed
"""
PRIVATE_PROPERTIES = ("iourc_content", )
prop = copy.copy(self._properties)
for k in list(prop.keys()):
if k in PRIVATE_PROPERTIES:
del prop[k]
return prop
def __json__(self, topology_dump=False):
"""
:param topology_dump: Filter to keep only properties require for saving on disk
@@ -608,7 +621,7 @@ class Node:
"name": self._name,
"console": self._console,
"console_type": self._console_type,
"properties": self._filter_properties(),
"properties": self._properties,
"label": self._label,
"x": self._x,
"y": self._y,
@@ -631,7 +644,7 @@ class Node:
"console_host": str(self._compute.console_host),
"console_type": self._console_type,
"command_line": self._command_line,
"properties": self._filter_properties(),
"properties": self._properties,
"status": self._status,
"label": self._label,
"x": self._x,
+10 -5
View File
@@ -15,6 +15,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import aiohttp
from .atm_port import ATMPort
from .frame_relay_port import FrameRelayPort
from .gigabitethernet_port import GigabitEthernetPort
@@ -64,11 +66,14 @@ class StandardPortFactory:
port_name = first_port_name
port = PortFactory(port_name, segment_number, adapter_number, port_number, "ethernet")
else:
port_name = port_name_format.format(
interface_number,
segment_number,
adapter=adapter_number,
**cls._generate_replacement(interface_number, segment_number))
try:
port_name = port_name_format.format(
interface_number,
segment_number,
adapter=adapter_number,
**cls._generate_replacement(interface_number, segment_number))
except (ValueError, KeyError) as e:
raise aiohttp.web.HTTPConflict(text="Invalid port name format {}: {}".format(port_name_format, str(e)))
port = PortFactory(port_name, segment_number, adapter_number, port_number, "ethernet")
interface_number += 1
if port_segment_size:
+25 -12
View File
@@ -289,7 +289,12 @@ class Project:
if '{0}' in base_name or '{id}' in base_name:
# base name is a template, replace {0} or {id} by an unique identifier
for number in range(1, 1000000):
name = base_name.format(number, id=number)
try:
name = base_name.format(number, id=number, name="Node")
except KeyError as e:
raise aiohttp.web.HTTPConflict(text="{" + e.args[0] + "} is not a valid replacement string in the node name")
except ValueError as e:
raise aiohttp.web.HTTPConflict(text="{} is not a valid replacement string in the node name".format(base_name))
if name not in self._allocated_node_names:
self._allocated_node_names.add(name)
return name
@@ -314,10 +319,11 @@ class Project:
@open_required
@asyncio.coroutine
def add_node(self, compute, name, node_id, node_type=None, **kwargs):
def add_node(self, compute, name, node_id, dump=True, node_type=None, **kwargs):
"""
Create a node or return an existing node
:param dump: Dump topology to disk
:param kwargs: See the documentation of node
"""
if node_id in self._nodes:
@@ -349,7 +355,8 @@ class Project:
yield from node.create()
self._nodes[node.id] = node
self.controller.notification.emit("node.created", node.__json__())
self.dump()
if dump:
self.dump()
return node
@locked_coroutine
@@ -401,17 +408,19 @@ class Project:
@open_required
@asyncio.coroutine
def add_drawing(self, drawing_id=None, **kwargs):
def add_drawing(self, drawing_id=None, dump=True, **kwargs):
"""
Create an drawing or return an existing drawing
:param dump: Dump the topology to disk
:param kwargs: See the documentation of drawing
"""
if drawing_id not in self._drawings:
drawing = Drawing(self, drawing_id=drawing_id, **kwargs)
self._drawings[drawing.id] = drawing
self.controller.notification.emit("drawing.created", drawing.__json__())
self.dump()
if dump:
self.dump()
return drawing
return self._drawings[drawing_id]
@@ -435,15 +444,18 @@ class Project:
@open_required
@asyncio.coroutine
def add_link(self, link_id=None):
def add_link(self, link_id=None, dump=True):
"""
Create a link. By default the link is empty
:param dump: Dump topology to disk
"""
if link_id and link_id in self._links:
return self._links[link.id]
return self._links[link_id]
link = UDPLink(self, link_id=link_id)
self._links[link.id] = link
self.dump()
if dump:
self.dump()
return link
@open_required
@@ -526,7 +538,7 @@ class Project:
@asyncio.coroutine
def close(self, ignore_notification=False):
yield from self.stop_all()
for compute in self._project_created_on_compute:
for compute in list(self._project_created_on_compute):
try:
yield from compute.post("/projects/{}/close".format(self._id), dont_connect=True)
# We don't care if a compute is down at this step
@@ -626,15 +638,16 @@ class Project:
compute = self.controller.get_compute(node.pop("compute_id"))
name = node.pop("name")
node_id = node.pop("node_id")
yield from self.add_node(compute, name, node_id, **node)
yield from self.add_node(compute, name, node_id, dump=False, **node)
for link_data in topology.get("links", []):
link = yield from self.add_link(link_id=link_data["link_id"])
for node_link in link_data["nodes"]:
node = self.get_node(node_link["node_id"])
yield from link.add_node(node, node_link["adapter_number"], node_link["port_number"], label=node_link.get("label"))
yield from link.add_node(node, node_link["adapter_number"], node_link["port_number"], label=node_link.get("label"), dump=False)
for drawing_data in topology.get("drawings", []):
drawing = yield from self.add_drawing(**drawing_data)
yield from self.add_drawing(dump=False, **drawing_data)
self.dump()
# We catch all error to be able to rollback the .gns3 to the previous state
except Exception as e:
for compute in self._project_created_on_compute:
+8 -4
View File
@@ -20,6 +20,7 @@ import os
import uuid
import shutil
import asyncio
import aiohttp.web
from datetime import datetime, timezone
@@ -80,10 +81,13 @@ class Snapshot:
# We don't send close notif to clients because the close / open dance is purely internal
yield from self._project.close(ignore_notification=True)
self._project.controller.notification.emit("snapshot.restored", self.__json__())
if os.path.exists(os.path.join(self._project.path, "project-files")):
shutil.rmtree(os.path.join(self._project.path, "project-files"))
with open(self._path, "rb") as f:
project = yield from import_project(self._project.controller, self._project.id, f, location=self._project.path)
try:
if os.path.exists(os.path.join(self._project.path, "project-files")):
shutil.rmtree(os.path.join(self._project.path, "project-files"))
with open(self._path, "rb") as f:
project = yield from import_project(self._project.controller, self._project.id, f, location=self._project.path)
except (OSError, PermissionError) as e:
raise aiohttp.web.HTTPConflict(text=str(e))
yield from project.open()
return project
+11 -10
View File
@@ -39,16 +39,17 @@ class Symbols:
def list(self):
self._symbols_path = {}
symbols = []
for file in os.listdir(get_resource("symbols")):
if file.startswith('.'):
continue
symbol_id = ':/symbols/' + file
symbols.append({
'symbol_id': symbol_id,
'filename': file,
'builtin': True,
})
self._symbols_path[symbol_id] = os.path.join(get_resource("symbols"), file)
if get_resource("symbols"):
for file in os.listdir(get_resource("symbols")):
if file.startswith('.'):
continue
symbol_id = ':/symbols/' + file
symbols.append({
'symbol_id': symbol_id,
'filename': file,
'builtin': True,
})
self._symbols_path[symbol_id] = os.path.join(get_resource("symbols"), file)
directory = self.symbols_path()
if directory:
for file in os.listdir(directory):
+63 -20
View File
@@ -23,7 +23,6 @@ import glob
import shutil
import zipfile
import aiohttp
import platform
import jsonschema
@@ -37,7 +36,7 @@ import logging
log = logging.getLogger(__name__)
GNS3_FILE_FORMAT_REVISION = 7
GNS3_FILE_FORMAT_REVISION = 8
def _check_topology_schema(topo):
@@ -117,34 +116,65 @@ def load_topology(path):
topo = json.load(f)
except (OSError, UnicodeDecodeError, ValueError) as e:
raise aiohttp.web.HTTPConflict(text="Could not load topology {}: {}".format(path, str(e)))
if "revision" not in topo or topo["revision"] < 5:
if topo.get("revision", 0) > GNS3_FILE_FORMAT_REVISION:
raise aiohttp.web.HTTPConflict(text="This project is designed for a more recent version of GNS3 please update GNS3 to version {} or later".format(topo["version"]))
changed = False
if "revision" not in topo or topo["revision"] < GNS3_FILE_FORMAT_REVISION:
# If it's an old GNS3 file we need to convert it
# first we backup the file
shutil.copy(path, path + ".backup{}".format(topo.get("revision", 0)))
changed = True
if "revision" not in topo or topo["revision"] < 5:
topo = _convert_1_3_later(topo, path)
_check_topology_schema(topo)
with open(path, "w+", encoding="utf-8") as f:
json.dump(topo, f, indent=4, sort_keys=True)
# Version before GNS3 2.0 alpha 4
if topo["revision"] < 6:
shutil.copy(path, path + ".backup{}".format(topo.get("revision", 0)))
topo = _convert_2_0_0_alpha(topo, path)
_check_topology_schema(topo)
with open(path, "w+", encoding="utf-8") as f:
json.dump(topo, f, indent=4, sort_keys=True)
# Version before GNS3 2.0 beta 3
if topo["revision"] < 7:
shutil.copy(path, path + ".backup{}".format(topo.get("revision", 0)))
topo = _convert_2_0_0_beta_2(topo, path)
_check_topology_schema(topo)
# Version before GNS3 2.1
if topo["revision"] < 8:
topo = _convert_2_0_0(topo, path)
_check_topology_schema(topo)
if changed:
with open(path, "w+", encoding="utf-8") as f:
json.dump(topo, f, indent=4, sort_keys=True)
return topo
if topo["revision"] > GNS3_FILE_FORMAT_REVISION:
raise aiohttp.web.HTTPConflict(text="This project is designed for a more recent version of GNS3 please update GNS3 to version {} or later".format(topo["version"]))
_check_topology_schema(topo)
def _convert_2_0_0(topo, topo_path):
"""
Convert topologies from GNS3 2.0.0 to 2.1
Changes:
* Remove startup_script_path from VPCS and base config file for IOU and Dynamips
"""
topo["revision"] = 8
for node in topo.get("topology", {}).get("nodes", []):
if "properties" in node:
if node["node_type"] == "vpcs":
if "startup_script_path" in node["properties"]:
del node["properties"]["startup_script_path"]
if "startup_script" in node["properties"]:
del node["properties"]["startup_script"]
elif node["node_type"] == "dynamips" or node["node_type"] == "iou":
if "startup_config" in node["properties"]:
del node["properties"]["startup_config"]
if "private_config" in node["properties"]:
del node["properties"]["private_config"]
if "startup_config_content" in node["properties"]:
del node["properties"]["startup_config_content"]
if "private_config_content" in node["properties"]:
del node["properties"]["private_config_content"]
return topo
@@ -165,11 +195,14 @@ def _convert_2_0_0_beta_2(topo, topo_path):
dynamips_dir = os.path.join(topo_dir, "project-files", "dynamips")
node_dir = os.path.join(dynamips_dir, node_id)
os.makedirs(os.path.join(node_dir, "configs"), exist_ok=True)
for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "*_i{}_*".format(dynamips_id))):
shutil.move(path, os.path.join(node_dir, os.path.basename(path)))
for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "configs", "i{}_*".format(dynamips_id))):
shutil.move(path, os.path.join(node_dir, "configs", os.path.basename(path)))
try:
os.makedirs(os.path.join(node_dir, "configs"), exist_ok=True)
for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "*_i{}_*".format(dynamips_id))):
shutil.move(path, os.path.join(node_dir, os.path.basename(path)))
for path in glob.glob(os.path.join(glob.escape(dynamips_dir), "configs", "i{}_*".format(dynamips_id))):
shutil.move(path, os.path.join(node_dir, "configs", os.path.basename(path)))
except OSError as e:
raise aiohttp.web.HTTPConflict(text="Can't convert project {}: {}".format(topo_path, str(e)))
return topo
@@ -320,14 +353,24 @@ def _convert_1_3_later(topo, topo_path):
node["properties"]["ram"] = PLATFORMS_DEFAULT_RAM[old_node["type"].lower()]
elif old_node["type"] == "VMwareVM":
node["node_type"] = "vmware"
node["properties"]["linked_clone"] = old_node.get("linked_clone", False)
if node["symbol"] is None:
node["symbol"] = ":/symbols/vmware_guest.svg"
elif old_node["type"] == "VirtualBoxVM":
node["node_type"] = "virtualbox"
node["properties"]["linked_clone"] = old_node.get("linked_clone", False)
if node["symbol"] is None:
node["symbol"] = ":/symbols/vbox_guest.svg"
elif old_node["type"] == "IOUDevice":
node["node_type"] = "iou"
node["port_name_format"] = old_node.get("port_name_format", "Ethernet{segment0}/{port0}")
node["port_segment_size"] = int(old_node.get("port_segment_size", "4"))
if node["symbol"] is None:
if "l2" in node["properties"].get("path", ""):
node["symbol"] = ":/symbols/multilayer_switch.svg"
else:
node["symbol"] = ":/symbols/router.svg"
elif old_node["type"] == "Cloud":
old_node["ports"] = _create_cloud(node, old_node, ":/symbols/cloud.svg")
elif old_node["type"] == "Host":