Refactor template management to use database.

This commit is contained in:
grossmj committed 2021-03-28 11:15:08 +10:30
1 parent b417bc4dec
commit d730c591b3
20 files changed
+1704 -1535

No files matched your search

+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python
#
# Copyright (C) 2021 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from .base import Base
from .users import User
from .templates import (
Template,
CloudTemplate,
DockerTemplate,
DynamipsTemplate,
EthernetHubTemplate,
EthernetSwitchTemplate,
IOUTemplate,
QemuTemplate,
VirtualBoxTemplate,
VMwareTemplate,
VPCSTemplate
)
@@ -17,21 +17,33 @@
import uuid
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, func
from sqlalchemy.orm import relationship
from fastapi.encoders import jsonable_encoder
from sqlalchemy import Column, DateTime, func, inspect
from sqlalchemy.types import TypeDecorator, CHAR
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.declarative import as_declarative
from sqlalchemy.orm import declarative_base
Base = declarative_base()
@as_declarative()
class Base:
def _asdict(self):
return {c.key: getattr(self, c.key)
for c in inspect(self).mapper.column_attrs}
def _asjson(self):
return jsonable_encoder(self._asdict())
class GUID(TypeDecorator):
"""Platform-independent GUID type.
"""
Platform-independent GUID type.
Uses PostgreSQL's UUID type, otherwise uses
CHAR(32), storing as stringified hex values.
"""
impl = CHAR
def load_dialect_impl(self, dialect):
@@ -73,30 +85,3 @@ class BaseTable(Base):
def generate_uuid():
return str(uuid.uuid4())
class User(BaseTable):
__tablename__ = "users"
user_id = Column(GUID, primary_key=True, default=generate_uuid)
username = Column(String, unique=True, index=True)
email = Column(String, unique=True, index=True)
full_name = Column(String)
hashed_password = Column(String)
is_active = Column(Boolean, default=True)
is_superuser = Column(Boolean, default=False)
# items = relationship("Item", back_populates="owner")
#
#
# class Item(Base):
# __tablename__ = "items"
#
# id = Column(Integer, primary_key=True, index=True)
# title = Column(String, index=True)
# description = Column(String, index=True)
# owner_id = Column(Integer, ForeignKey("users.id"))
#
# owner = relationship("User", back_populates="items")
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python
#
# Copyright (C) 2021 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sqlalchemy import Boolean, Column, String, Integer, ForeignKey, PickleType
from .base import BaseTable, generate_uuid, GUID
class Template(BaseTable):
__tablename__ = "templates"
template_id = Column(GUID, primary_key=True, default=generate_uuid)
name = Column(String, index=True)
category = Column(String)
default_name_format = Column(String)
symbol = Column(String)
builtin = Column(Boolean, default=False)
compute_id = Column(String)
usage = Column(String)
template_type = Column(String)
__mapper_args__ = {
"polymorphic_identity": "templates",
"polymorphic_on": template_type
}
class CloudTemplate(Template):
__tablename__ = "cloud_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
ports_mapping = Column(PickleType)
remote_console_host = Column(String)
remote_console_port = Column(Integer)
remote_console_type = Column(String)
remote_console_http_path = Column(String)
__mapper_args__ = {
"polymorphic_identity": "cloud"
}
class DockerTemplate(Template):
__tablename__ = "docker_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
image = Column(String)
adapters = Column(Integer)
start_command = Column(String)
environment = Column(String)
console_type = Column(String)
aux_type = Column(String)
console_auto_start = Column(Boolean)
console_http_port = Column(Integer)
console_http_path = Column(String)
console_resolution = Column(String)
extra_hosts = Column(String)
extra_volumes = Column(PickleType)
memory = Column(Integer)
cpus = Column(Integer)
custom_adapters = Column(PickleType)
__mapper_args__ = {
"polymorphic_identity": "docker"
}
class DynamipsTemplate(Template):
__tablename__ = "dynamips_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
platform = Column(String)
chassis = Column(String)
image = Column(String)
exec_area = Column(Integer)
mmap = Column(Boolean)
mac_addr = Column(String)
system_id = Column(String)
startup_config = Column(String)
private_config = Column(String)
idlepc = Column(String)
idlemax = Column(Integer)
idlesleep = Column(Integer)
disk0 = Column(Integer)
disk1 = Column(Integer)
auto_delete_disks = Column(Boolean)
console_type = Column(String)
console_auto_start = Column(Boolean)
aux_type = Column(String)
ram = Column(Integer)
nvram = Column(Integer)
npe = Column(String)
midplane = Column(String)
sparsemem = Column(Boolean)
iomem = Column(Integer)
slot0 = Column(String)
slot1 = Column(String)
slot2 = Column(String)
slot3 = Column(String)
slot4 = Column(String)
slot5 = Column(String)
slot6 = Column(String)
wic0 = Column(String)
wic1 = Column(String)
wic2 = Column(String)
__mapper_args__ = {
"polymorphic_identity": "dynamips"
}
class EthernetHubTemplate(Template):
__tablename__ = "ethernet_hub_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
ports_mapping = Column(PickleType)
__mapper_args__ = {
"polymorphic_identity": "ethernet_hub"
}
class EthernetSwitchTemplate(Template):
__tablename__ = "ethernet_switch_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
ports_mapping = Column(PickleType)
console_type = Column(String)
__mapper_args__ = {
"polymorphic_identity": "ethernet_switch"
}
class IOUTemplate(Template):
__tablename__ = "iou_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
path = Column(String)
ethernet_adapters = Column(Integer)
serial_adapters = Column(Integer)
ram = Column(Integer)
nvram = Column(Integer)
use_default_iou_values = Column(Boolean)
startup_config = Column(String)
private_config = Column(String)
l1_keepalives = Column(Boolean)
console_type = Column(String)
console_auto_start = Column(Boolean)
__mapper_args__ = {
"polymorphic_identity": "iou"
}
class QemuTemplate(Template):
__tablename__ = "qemu_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
qemu_path = Column(String)
platform = Column(String)
linked_clone = Column(Boolean)
ram = Column(Integer)
cpus = Column(Integer)
maxcpus = Column(Integer)
adapters = Column(Integer)
adapter_type = Column(String)
mac_address = Column(String)
first_port_name = Column(String)
port_name_format = Column(String)
port_segment_size = Column(Integer)
console_type = Column(String)
console_auto_start = Column(Boolean)
aux_type = Column(String)
boot_priority = Column(String)
hda_disk_image = Column(String)
hda_disk_interface = Column(String)
hdb_disk_image = Column(String)
hdb_disk_interface = Column(String)
hdc_disk_image = Column(String)
hdc_disk_interface = Column(String)
hdd_disk_image = Column(String)
hdd_disk_interface = Column(String)
cdrom_image = Column(String)
initrd = Column(String)
kernel_image = Column(String)
bios_image = Column(String)
kernel_command_line = Column(String)
legacy_networking = Column(Boolean)
replicate_network_connection_state = Column(Boolean)
create_config_disk = Column(Boolean)
on_close = Column(String)
cpu_throttling = Column(Integer)
process_priority = Column(String)
options = Column(String)
custom_adapters = Column(PickleType)
__mapper_args__ = {
"polymorphic_identity": "qemu"
}
class VirtualBoxTemplate(Template):
__tablename__ = "virtualbox_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
vmname = Column(String)
ram = Column(Integer)
linked_clone = Column(Boolean)
adapters = Column(Integer)
use_any_adapter = Column(Boolean)
adapter_type = Column(String)
first_port_name = Column(String)
port_name_format = Column(String)
port_segment_size = Column(Integer)
headless = Column(Boolean)
on_close = Column(String)
console_type = Column(String)
console_auto_start = Column(Boolean)
custom_adapters = Column(PickleType)
__mapper_args__ = {
"polymorphic_identity": "virtualbox"
}
class VMwareTemplate(Template):
__tablename__ = "vmware_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
vmx_path = Column(String)
linked_clone = Column(Boolean)
first_port_name = Column(String)
port_name_format = Column(String)
port_segment_size = Column(Integer)
adapters = Column(Integer)
adapter_type = Column(String)
use_any_adapter = Column(Boolean)
headless = Column(Boolean)
on_close = Column(String)
console_type = Column(String)
console_auto_start = Column(Boolean)
custom_adapters = Column(PickleType)
__mapper_args__ = {
"polymorphic_identity": "vmware"
}
class VPCSTemplate(Template):
__tablename__ = "vpcs_templates"
template_id = Column(GUID, ForeignKey("templates.template_id"), primary_key=True)
base_script_file = Column(String)
console_type = Column(String)
console_auto_start = Column(Boolean, default=False)
__mapper_args__ = {
"polymorphic_identity": "vpcs"
}
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python
#
# Copyright (C) 2021 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, DateTime, func
from sqlalchemy.orm import relationship
from .base import BaseTable, generate_uuid, GUID
class User(BaseTable):
__tablename__ = "users"
user_id = Column(GUID, primary_key=True, default=generate_uuid)
username = Column(String, unique=True, index=True)
email = Column(String, unique=True, index=True)
full_name = Column(String)
hashed_password = Column(String)
is_active = Column(Boolean, default=True)
is_superuser = Column(Boolean, default=False)
# items = relationship("Item", back_populates="owner")
#
#
# class Item(Base):
# __tablename__ = "items"
#
# id = Column(Integer, primary_key=True, index=True)
# title = Column(String, index=True)
# description = Column(String, index=True)
# owner_id = Column(Integer, ForeignKey("users.id"))
#
# owner = relationship("User", back_populates="items")
+2
View File
@@ -16,6 +16,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sqlalchemy.ext.asyncio import AsyncSession
from gns3server.controller import Controller
class BaseRepository:
@@ -23,3 +24,4 @@ class BaseRepository:
def __init__(self, db_session: AsyncSession) -> None:
self._db_session = db_session
self._controller = Controller.instance()
+243
View File
@@ -0,0 +1,243 @@
#!/usr/bin/env python
#
# Copyright (C) 2021 GNS3 Technologies Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import uuid
from uuid import UUID
from typing import List
from fastapi.encoders import jsonable_encoder
from sqlalchemy import select, update, delete
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm.session import make_transient
from .base import BaseRepository
import gns3server.db.models as models
from gns3server import schemas
TEMPLATE_TYPE_TO_SHEMA = {
"cloud": schemas.CloudTemplate,
"ethernet_hub": schemas.EthernetHubTemplate,
"ethernet_switch": schemas.EthernetSwitchTemplate,
"docker": schemas.DockerTemplate,
"dynamips": schemas.DynamipsTemplate,
"vpcs": schemas.VPCSTemplate,
"virtualbox": schemas.VirtualBoxTemplate,
"vmware": schemas.VMwareTemplate,
"iou": schemas.IOUTemplate,
"qemu": schemas.QemuTemplate
}
DYNAMIPS_PLATFORM_TO_SHEMA = {
"c7200": schemas.C7200DynamipsTemplate,
"c3745": schemas.C3745DynamipsTemplate,
"c3725": schemas.C3725DynamipsTemplate,
"c3600": schemas.C3600DynamipsTemplate,
"c2691": schemas.C2691DynamipsTemplate,
"c2600": schemas.C2600DynamipsTemplate,
"c1700": schemas.C1700DynamipsTemplate
}
TEMPLATE_TYPE_TO_MODEL = {
"cloud": models.CloudTemplate,
"docker": models.DockerTemplate,
"dynamips": models.DynamipsTemplate,
"ethernet_hub": models.EthernetHubTemplate,
"ethernet_switch": models.EthernetSwitchTemplate,
"iou": models.IOUTemplate,
"qemu": models.QemuTemplate,
"virtualbox": models.VirtualBoxTemplate,
"vmware": models.VMwareTemplate,
"vpcs": models.VPCSTemplate
}
# built-in templates have their compute_id set to None to tell clients to select a compute
BUILTIN_TEMPLATES = [
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "cloud"),
"template_type": "cloud",
"name": "Cloud",
"default_name_format": "Cloud{0}",
"category": "guest",
"symbol": ":/symbols/cloud.svg",
"compute_id": None,
"builtin": True
},
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "nat"),
"template_type": "nat",
"name": "NAT",
"default_name_format": "NAT{0}",
"category": "guest",
"symbol": ":/symbols/cloud.svg",
"compute_id": None,
"builtin": True
},
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "vpcs"),
"template_type": "vpcs",
"name": "VPCS",
"default_name_format": "PC{0}",
"category": "guest",
"symbol": ":/symbols/vpcs_guest.svg",
"base_script_file": "vpcs_base_config.txt",
"compute_id": None,
"builtin": True
},
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_switch"),
"template_type": "ethernet_switch",
"name": "Ethernet switch",
"console_type": "none",
"default_name_format": "Switch{0}",
"category": "switch",
"symbol": ":/symbols/ethernet_switch.svg",
"compute_id": None,
"builtin": True
},
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_hub"),
"template_type": "ethernet_hub",
"name": "Ethernet hub",
"default_name_format": "Hub{0}",
"category": "switch",
"symbol": ":/symbols/hub.svg",
"compute_id": None,
"builtin": True
},
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "frame_relay_switch"),
"template_type": "frame_relay_switch",
"name": "Frame Relay switch",
"default_name_format": "FRSW{0}",
"category": "switch",
"symbol": ":/symbols/frame_relay_switch.svg",
"compute_id": None,
"builtin": True
},
{
"template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "atm_switch"),
"template_type": "atm_switch",
"name": "ATM switch",
"default_name_format": "ATMSW{0}",
"category": "switch",
"symbol": ":/symbols/atm_switch.svg",
"compute_id": None,
"builtin": True
},
]
class TemplatesRepository(BaseRepository):
def __init__(self, db_session: AsyncSession) -> None:
super().__init__(db_session)
def get_builtin_template(self, template_id: UUID) -> dict:
for builtin_template in BUILTIN_TEMPLATES:
if builtin_template["template_id"] == template_id:
return jsonable_encoder(builtin_template)
async def get_template(self, template_id: UUID) -> dict:
query = select(models.Template).where(models.Template.template_id == template_id)
result = (await self._db_session.execute(query)).scalars().first()
if result:
return result._asjson()
else:
return self.get_builtin_template(template_id)
async def get_templates(self) -> List[dict]:
templates = []
query = select(models.Template)
result = await self._db_session.execute(query)
for db_template in result.scalars().all():
templates.append(db_template._asjson())
for builtin_template in BUILTIN_TEMPLATES:
templates.append(jsonable_encoder(builtin_template))
return templates
async def create_template(self, template_create: schemas.TemplateCreate) -> dict:
# get the default template settings
template_settings = jsonable_encoder(template_create, exclude_unset=True)
template_schema = TEMPLATE_TYPE_TO_SHEMA[template_create.template_type]
template_settings_with_defaults = template_schema.parse_obj(template_settings)
settings = template_settings_with_defaults.dict()
if template_create.template_type == "dynamips":
# special case for Dynamips to cover all platform types that contain specific settings
dynamips_template_schema = DYNAMIPS_PLATFORM_TO_SHEMA[settings["platform"]]
dynamips_template_settings_with_defaults = dynamips_template_schema.parse_obj(template_settings)
settings = dynamips_template_settings_with_defaults.dict()
model = TEMPLATE_TYPE_TO_MODEL[template_create.template_type]
db_template = model(**settings)
self._db_session.add(db_template)
await self._db_session.commit()
await self._db_session.refresh(db_template)
template = db_template._asjson()
self._controller.notification.controller_emit("template.created", template)
return template
async def update_template(
self,
template_id: UUID,
template_update: schemas.TemplateUpdate) -> dict:
update_values = template_update.dict(exclude_unset=True)
query = update(models.Template) \
.where(models.Template.template_id == template_id) \
.values(update_values)
await self._db_session.execute(query)
await self._db_session.commit()
template = await self.get_template(template_id)
if template:
self._controller.notification.controller_emit("template.updated", template)
return template
async def delete_template(self, template_id: UUID) -> bool:
query = delete(models.Template).where(models.Template.template_id == template_id)
result = await self._db_session.execute(query)
await self._db_session.commit()
if result.rowcount > 0:
self._controller.notification.controller_emit("template.deleted", {"template_id": str(template_id)})
return True
return False
async def duplicate_template(self, template_id: UUID) -> dict:
query = select(models.Template).where(models.Template.template_id == template_id)
db_template = (await self._db_session.execute(query)).scalars().first()
if not db_template:
return db_template
# duplicate db object with new primary key (template_id)
self._db_session.expunge(db_template)
make_transient(db_template)
db_template.template_id = None
self._db_session.add(db_template)
await self._db_session.commit()
await self._db_session.refresh(db_template)
template = db_template._asjson()
self._controller.notification.controller_emit("template.created", template)
return template
+8 -4
View File
@@ -36,22 +36,26 @@ class UsersRepository(BaseRepository):
async def get_user(self, user_id: UUID) -> Optional[models.User]:
result = await self._db_session.execute(select(models.User).where(models.User.user_id == user_id))
query = select(models.User).where(models.User.user_id == user_id)
result = await self._db_session.execute(query)
return result.scalars().first()
async def get_user_by_username(self, username: str) -> Optional[models.User]:
result = await self._db_session.execute(select(models.User).where(models.User.username == username))
query = select(models.User).where(models.User.username == username)
result = await self._db_session.execute(query)
return result.scalars().first()
async def get_user_by_email(self, email: str) -> Optional[models.User]:
result = await self._db_session.execute(select(models.User).where(models.User.email == email))
query = select(models.User).where(models.User.email == email)
result = await self._db_session.execute(query)
return result.scalars().first()
async def get_users(self) -> List[models.User]:
result = await self._db_session.execute(select(models.User))
query = select(models.User)
result = await self._db_session.execute(query)
return result.scalars().all()
async def create_user(self, user: schemas.UserCreate) -> models.User:
+1 -1
View File
@@ -31,7 +31,7 @@ log = logging.getLogger(__name__)
async def connect_to_db(app: FastAPI) -> None:
db_path = os.path.join(Config.instance().config_dir, "gns3_controller.db")
db_url = os.environ.get("GNS3_DATABASE_URI", f"sqlite:///{db_path}")
db_url = os.environ.get("GNS3_DATABASE_URI", f"sqlite+pysqlite:///{db_path}")
engine = create_async_engine(db_url, connect_args={"check_same_thread": False}, future=True)
try:
async with engine.begin() as conn: