Release v2.2.43

Correctly install built-in appliances

.github/workflows/publish-api-documentation.yml

@@ -12,11 +12,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
         ref: "gh-pages"
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v3
       with:
         python-version: 3.7
     - name: Merge changes from 3.0 branch
@@ -24,13 +24,10 @@ jobs:
         git config user.name github-actions
         git config user.email github-actions@github.com
         git merge origin/3.0 -X theirs
-    - name: Install dependencies
+    - name: Install GNS3 server and dependencies
       run: |
         python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Install GNS3 server
-      run: |
-        python setup.py install
+        python -m pip install .
     - name: Generate the API documentation
       run: |
         cd scripts

CHANGELOG

@@ -1,5 +1,15 @@
 # Change Log
 
+## 2.2.43 19/09/2023
+
+* Force English output for VBoxManage. Fixes #2266
+* Automatically add vboxnet and DHCP server if not present for VirtualBox GNS3 VM. Ref #2266
+* Fix issue with controller config saved before checking current version with previous one
+* Prevent X11 socket file to be modified by Docker container
+* Use the user data dir to store built-in appliances
+* Catch ConnectionResetError exception when client disconnects
+* Upgrade to PyQt 5.15.9 and pywin32
+
 ## 2.2.42 09/08/2023
 
 * Bundle web-ui v2.2.42

gns3server/appliances/open-media-vault.gns3a

@@ -26,6 +26,14 @@
         "kvm": "require"
     },
     "images": [
+        {
+            "filename": "openmediavault_6.5.0-amd64.iso",
+            "version": "6.5.0",
+            "md5sum": "aa40e5ca50748b139cba2f4ac704a72d",
+            "filesize": 941621248,
+            "download_url": "https://www.openmediavault.org/download.html",
+            "direct_download_url": "https://sourceforge.net/projects/openmediavault/files/6.5.0/openmediavault_6.5.0-amd64.iso"
+        },
         {
             "filename": "openmediavault_6.0.24-amd64.iso",
             "version": "6.0.24",
@@ -60,6 +68,14 @@
         }
     ],
     "versions": [
+        {
+            "name": "6.5.0",
+            "images": {
+                "hda_disk_image": "empty30G.qcow2",
+                "hdb_disk_image": "empty30G.qcow2",
+                "cdrom_image": "openmediavault_6.5.0-amd64.iso"
+            }
+        },
         {
             "name": "6.0.24",
             "images": {

gns3server/appliances/openwrt.gns3a

@@ -170,7 +170,7 @@
         {
             "filename": "openwrt-18.06.5-x86-64-combined-ext4.img",
             "version": "18.06.5",
-            "md5sum": "6fce24c15f0bc75af16c133b839aea30",
+            "md5sum": "a0f72f4e75e15bef06396fa31eb1bc82",
             "filesize": 285736960,
             "download_url": "https://downloads.openwrt.org/releases/18.06.5/targets/x86/64/",
             "direct_download_url": "https://downloads.openwrt.org/releases/18.06.5/targets/x86/64/openwrt-18.06.5-x86-64-combined-ext4.img.gz",
@@ -179,7 +179,7 @@
         {
             "filename": "openwrt-18.06.2-x86-64-combined-ext4.img",
             "version": "18.06.2",
-            "md5sum": "d112cd432bf51e2ddadbf9513f272fd9",
+            "md5sum": "9996a3c070b3e2ea582d28293bd78055",
             "filesize": 285736960,
             "download_url": "https://downloads.openwrt.org/releases/18.06.2/targets/x86/64/",
             "direct_download_url": "https://downloads.openwrt.org/releases/18.06.2/targets/x86/64/openwrt-18.06.2-x86-64-combined-ext4.img.gz",

gns3server/appliances/vyos.gns3a

@@ -59,32 +59,28 @@
             "version": "1.2.9-S1",
             "md5sum": "3fece6363f9766f862e26d292d0ed5a3",
             "filesize": 430964736,
-            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-s1-generic-iso-image",
-            "direct_download_url": "https://s3-us.vyos.io/1.2.9-S1/vyos-1.2.9-S1-amd64.iso"
+            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-s1-generic-iso-image"
         },
         {
             "filename": "vyos-1.2.9-S1-10G-qemu.qcow2",
             "version": "1.2.9-S1-KVM",
             "md5sum": "0a70d78b80a3716d42487c02ef44f41f",
             "filesize": 426967040,
-            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-s1-for-kvm",
-            "direct_download_url": "https://s3-us.vyos.io/1.2.9-S1/vyos-1.2.9-S1-10G-qemu.qcow2"
+            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-s1-for-kvm"
         },
         {
             "filename": "vyos-1.2.9-amd64.iso",
             "version": "1.2.9",
             "md5sum": "586be23b6256173e174c82d8f1f699a1",
             "filesize": 430964736,
-            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-generic-iso-image",
-            "direct_download_url": "https://s3-us.vyos.io/1.2.9/vyos-1.2.9-amd64.iso"
+            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-generic-iso-image"
         },
         {
             "filename": "vyos-1.2.9-10G-qemu.qcow2",
             "version": "1.2.9-KVM",
             "md5sum": "76871c7b248c32f75177c419128257ac",
             "filesize": 427360256,
-            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-10g-qemu-qcow2",
-            "direct_download_url": "https://s3-us.vyos.io/1.2.9/vyos-1.2.9-10G-qemu.qcow2"
+            "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-9-10g-qemu-qcow2"
         },
         {
             "filename": "vyos-1.2.8-amd64.iso",
@@ -93,13 +89,6 @@
             "filesize": 429916160,
             "download_url": "https://support.vyos.io/en/downloads/files/vyos-1-2-8-generic-iso-image"
         },
-        {
-            "filename": "vyos-1.1.8-amd64.iso",
-            "version": "1.1.8",
-            "md5sum": "95a141d4b592b81c803cdf7e9b11d8ea",
-            "filesize": 241172480,
-            "direct_download_url": "https://s3-us.vyos.io/vyos-1.1.8-amd64.iso"
-        },
         {
             "filename": "empty8G.qcow2",
             "version": "1.0",
@@ -170,13 +159,6 @@
                 "hda_disk_image": "empty8G.qcow2",
                 "cdrom_image": "vyos-1.2.8-amd64.iso"
             }
-        },
-        {
-            "name": "1.1.8",
-            "images": {
-                "hda_disk_image": "empty8G.qcow2",
-                "cdrom_image": "vyos-1.1.8-amd64.iso"
-            }
         }
     ]
 }

gns3server/appliances/windows-11-dev-env.gns3a

@@ -29,6 +29,14 @@
         "kvm": "require"
     },
     "images": [
+        {
+            "filename": "WinDev2308Eval-disk1.vmdk",
+            "version": "2308",
+            "md5sum": "6a9b4ed6d7481f7bbf8a054c797b1eee",
+            "filesize": 24945341952,
+            "download_url": "https://download.microsoft.com/download/7/1/3/7135f2ab-8528-49fc-9252-8d5d94c697ef/WinDev2308Eval.VMWare.zip",
+            "compression": "zip"
+        },
         {
             "filename": "WinDev2212Eval-disk1.vmdk",
             "version": "2212",
@@ -48,6 +56,13 @@
         }
     ],
     "versions": [
+        {
+            "name": "2308",
+            "images": {
+                "bios_image": "OVMF-edk2-stable202305.fd",
+                "hda_disk_image": "WinDev2308Eval-disk1.vmdk"
+            }
+        },
         {
             "name": "2212",
             "images": {

gns3server/compute/docker/docker_vm.py

@@ -406,7 +406,7 @@ class DockerVM(BaseNode):
             await self._start_vnc()
             params["Env"].append("QT_GRAPHICSSYSTEM=native")  # To fix a Qt issue: https://github.com/GNS3/gns3-server/issues/556
             params["Env"].append("DISPLAY=:{}".format(self._display))
-            params["HostConfig"]["Binds"].append("/tmp/.X11-unix/:/tmp/.X11-unix/")
+            params["HostConfig"]["Binds"].append("/tmp/.X11-unix/X{0}:/tmp/.X11-unix/X{0}:ro".format(self._display))
 
         if self._extra_hosts:
             extra_hosts = self._format_extra_hosts(self._extra_hosts)

gns3server/compute/virtualbox/__init__.py

@@ -109,9 +109,16 @@ class VirtualBox(BaseManager):
             command = [vboxmanage_path, "--nologo", subcommand]
             command.extend(args)
             command_string = " ".join(command)
+            env = os.environ.copy()
+            env["LANG"] = "en"  # force english output because we rely on it to parse the output
             log.info("Executing VBoxManage with command: {}".format(command_string))
             try:
-                process = await asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+                process = await asyncio.create_subprocess_exec(
+                    *command,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                    env=env
+                )
             except (OSError, subprocess.SubprocessError) as e:
                 raise VirtualBoxError("Could not execute VBoxManage: {}".format(e))
 

gns3server/controller/__init__.py

@@ -191,29 +191,28 @@ class Controller:
         Save the controller configuration on disk
         """
 
-        if self._config_loaded is False:
-            return
+        controller_settings = dict()
+        if self._config_loaded:
+            controller_settings = {"computes": [],
+                                   "templates": [],
+                                   "gns3vm": self.gns3vm.__json__(),
+                                   "iou_license": self._iou_license_settings,
+                                   "appliances_etag": self._appliance_manager.appliances_etag,
+                                   "version": __version__}
 
-        controller_settings = {"computes": [],
-                               "templates": [],
-                               "gns3vm": self.gns3vm.__json__(),
-                               "iou_license": self._iou_license_settings,
-                               "appliances_etag": self._appliance_manager.appliances_etag,
-                               "version": __version__}
+            for template in self._template_manager.templates.values():
+                if not template.builtin:
+                    controller_settings["templates"].append(template.__json__())
 
-        for template in self._template_manager.templates.values():
-            if not template.builtin:
-                controller_settings["templates"].append(template.__json__())
-
-        for compute in self._computes.values():
-            if compute.id != "local" and compute.id != "vm":
-                controller_settings["computes"].append({"host": compute.host,
-                                                        "name": compute.name,
-                                                        "port": compute.port,
-                                                        "protocol": compute.protocol,
-                                                        "user": compute.user,
-                                                        "password": compute.password,
-                                                        "compute_id": compute.id})
+            for compute in self._computes.values():
+                if compute.id != "local" and compute.id != "vm":
+                    controller_settings["computes"].append({"host": compute.host,
+                                                            "name": compute.name,
+                                                            "port": compute.port,
+                                                            "protocol": compute.protocol,
+                                                            "user": compute.user,
+                                                            "password": compute.password,
+                                                            "compute_id": compute.id})
 
         try:
             os.makedirs(os.path.dirname(self._config_file), exist_ok=True)
@@ -229,8 +228,7 @@ class Controller:
 
         try:
             if not os.path.exists(self._config_file):
-                self._config_loaded = True
-                self.save()
+                self.save()  # this will create the config file
             with open(self._config_file) as f:
                 controller_settings = json.load(f)
         except (OSError, ValueError) as e:
@@ -255,6 +253,8 @@ class Controller:
         if not previous_version or \
                 parse_version(__version__.split("+")[0]) > parse_version(previous_version.split("+")[0]):
             self._appliance_manager.install_builtin_appliances()
+        elif not os.listdir(self._appliance_manager.builtin_appliances_path()):
+            self._appliance_manager.install_builtin_appliances()
 
         self._appliance_manager.appliances_etag = controller_settings.get("appliances_etag")
         self._appliance_manager.load_appliances()

gns3server/controller/appliance_manager.py

@@ -21,6 +21,7 @@ import uuid
 import asyncio
 import aiohttp
 import shutil
+import platformdirs
 
 
 try:
@@ -81,13 +82,13 @@ class ApplianceManager:
         os.makedirs(appliances_path, exist_ok=True)
         return appliances_path
 
-    def _builtin_appliances_path(self, delete_first=False):
+    def builtin_appliances_path(self, delete_first=False):
         """
         Get the built-in appliance storage directory
         """
 
-        config = Config.instance()
-        appliances_dir = os.path.join(config.config_dir, "appliances")
+        appname = vendor = "GNS3"
+        appliances_dir = os.path.join(platformdirs.user_data_dir(appname, vendor, roaming=True), "appliances")
         if delete_first:
             shutil.rmtree(appliances_dir, ignore_errors=True)
         os.makedirs(appliances_dir, exist_ok=True)
@@ -98,7 +99,7 @@ class ApplianceManager:
         At startup we copy the built-in appliances files.
         """
 
-        dst_path = self._builtin_appliances_path(delete_first=True)
+        dst_path = self.builtin_appliances_path(delete_first=True)
         log.info(f"Installing built-in appliances in '{dst_path}'")
         from . import Controller
         try:
@@ -112,7 +113,7 @@ class ApplianceManager:
         """
 
         self._appliances = {}
-        for directory, builtin in ((self._builtin_appliances_path(), True,), (self._custom_appliances_path(), False,)):
+        for directory, builtin in ((self.builtin_appliances_path(), True,), (self._custom_appliances_path(), False,)):
             if directory and os.path.isdir(directory):
                 for file in os.listdir(directory):
                     if not file.endswith('.gns3a') and not file.endswith('.gns3appliance'):
@@ -215,7 +216,7 @@ class ApplianceManager:
                         from . import Controller
                         Controller.instance().save()
                     json_data = await response.json()
-                appliances_dir = self._builtin_appliances_path()
+                appliances_dir = self.builtin_appliances_path()
                 downloaded_appliance_files = []
                 for appliance in json_data:
                     if appliance["type"] == "file":

gns3server/controller/gns3vm/virtualbox_gns3_vm.py

@@ -15,11 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import re
 import sys
 import aiohttp
 import logging
 import asyncio
 import socket
+import ipaddress
 
 from .base_gns3_vm import BaseGNS3VM
 from .gns3_vm_error import GNS3VMError
@@ -80,9 +82,6 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
             except ValueError:
                 continue
             self._system_properties[name.strip()] = value.strip()
-        if "API Version" in self._system_properties:
-            # API version is not consistent between VirtualBox versions, the key is named "API Version" in VirtualBox 7
-            self._system_properties["API version"] = self._system_properties.pop("API Version")
 
     async def _check_requirements(self):
         """
@@ -92,16 +91,16 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
         if not self._system_properties:
             await self._get_system_properties()
         if "API version" not in self._system_properties:
-            raise VirtualBoxError("Can't access to VirtualBox API version:\n{}".format(self._system_properties))
+            raise GNS3VMError("Can't access to VirtualBox API version:\n{}".format(self._system_properties))
         from cpuinfo import get_cpu_info
         cpu_info = await wait_run_in_executor(get_cpu_info)
         vendor_id = cpu_info.get('vendor_id_raw')
         if vendor_id == "GenuineIntel":
             if parse_version(self._system_properties["API version"]) < parse_version("6_1"):
-                raise VirtualBoxError("VirtualBox version 6.1 or above is required to run the GNS3 VM with nested virtualization enabled on Intel processors")
+                raise GNS3VMError("VirtualBox version 6.1 or above is required to run the GNS3 VM with nested virtualization enabled on Intel processors")
         elif vendor_id == "AuthenticAMD":
             if parse_version(self._system_properties["API version"]) < parse_version("6_0"):
-                raise VirtualBoxError("VirtualBox version 6.0 or above is required to run the GNS3 VM with nested virtualization enabled on AMD processors")
+                raise GNS3VMError("VirtualBox version 6.0 or above is required to run the GNS3 VM with nested virtualization enabled on AMD processors")
         else:
             log.warning("Could not determine CPU vendor: {}".format(vendor_id))
 
@@ -162,6 +161,44 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
                     return True
         return False
 
+    async def _add_dhcp_server(self, vboxnet):
+        """
+        Add a DHCP server for vboxnet.
+
+        :param vboxnet: vboxnet name
+        """
+
+        hostonlyifs = await self._execute("list", ["hostonlyifs"])
+        pattern = r"IPAddress:\s+(\d+\.\d+\.\d+\.\d+)\nNetworkMask:\s+(\d+\.\d+\.\d+\.\d+)"
+        match = re.search(pattern, hostonlyifs)
+
+        if match:
+            ip_address = match.group(1)
+            netmask = match.group(2)
+        else:
+            raise GNS3VMError("Could not find IP address and netmask for vboxnet {}".format(vboxnet))
+
+        try:
+            interface = ipaddress.IPv4Interface(f"{ip_address}/{netmask}")
+            subnet = ipaddress.IPv4Network(str(interface.network))
+            dhcp_server_ip = str(interface.ip + 1)
+            netmask = str(subnet.netmask)
+            lower_ip = str(interface.ip + 2)
+            upper_ip = str(subnet.network_address + subnet.num_addresses - 2)
+        except ValueError:
+            raise GNS3VMError("Invalid IP address and netmask for vboxnet {}: {}/{}".format(vboxnet, ip_address, netmask))
+
+        dhcp_server_args = [
+            "add",
+            "--network=HostInterfaceNetworking-{}".format(vboxnet),
+            "--server-ip={}".format(dhcp_server_ip),
+            "--netmask={}".format(netmask),
+            "--lower-ip={}".format(lower_ip),
+            "--upper-ip={}".format(upper_ip),
+            "--enable"
+        ]
+        await self._execute("dhcpserver", dhcp_server_args)
+
     async def _check_vboxnet_exists(self, vboxnet, vboxnet_type):
         """
         Check if the vboxnet interface exists
@@ -264,12 +301,20 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
                 await self.set_hostonly_network(interface_number, first_available_vboxnet)
                 vboxnet = first_available_vboxnet
             else:
-                raise GNS3VMError('VirtualBox host-only network "{}" does not exist, please make the sure the network adapter {} configuration is valid for "{}"'.format(vboxnet,
-                                                                                                                                                                         interface_number,
-                                                                                                                                                                         self._vmname))
+                try:
+                    await self._execute("hostonlyif", ["create"])
+                except GNS3VMError:
+                    raise GNS3VMError('VirtualBox host-only network "{}" does not exist and could not be automatically created, please make the sure the network adapter {} configuration is valid for "{}"'.format(
+                        vboxnet,
+                        interface_number,
+                        self._vmname
+                    ))
 
         if backend_type == "hostonlyadapter" and not (await self._check_dhcp_server(vboxnet)):
-            raise GNS3VMError('DHCP must be enabled on VirtualBox host-only network "{}"'.format(vboxnet))
+            try:
+                await self._add_dhcp_server(vboxnet)
+            except GNS3VMError as e:
+                raise GNS3VMError("Could not add DHCP server for vboxnet {}: {}, please configure manually".format(vboxnet, e))
 
         vm_state = await self._get_state()
         log.info('"{}" state is {}'.format(self._vmname, vm_state))
@@ -302,7 +347,7 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
         except OSError as e:
             raise GNS3VMError("Error while getting random port: {}".format(e))
 
-        if (await self._check_vbox_port_forwarding()):
+        if await self._check_vbox_port_forwarding():
             # delete the GNS3VM NAT port forwarding rule if it exists
             log.info("Removing GNS3VM NAT port forwarding rule from interface {}".format(nat_interface_number))
             await self._execute("controlvm", [self._vmname, "natpf{}".format(nat_interface_number), "delete", "GNS3VM"])

gns3server/crash_report.py

@@ -57,7 +57,7 @@ class CrashReport:
     Report crash to a third party service
     """
 
-    DSN = "https://226eee142b22cc399d1566b3dd4cbc86@o19455.ingest.sentry.io/38482"
+    DSN = "https://8dcaf668c2f31af6028fb4130bf2f58e@o19455.ingest.sentry.io/38482"
     _instance = None
 
     def __init__(self):

gns3server/version.py

@@ -23,8 +23,8 @@
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
 
-__version__ = "2.2.42"
-__version_info__ = (2, 2, 42, 0)
+__version__ = "2.2.43"
+__version_info__ = (2, 2, 43, 0)
 
 if "dev" in __version__:
     try:

gns3server/web/route.py

@@ -232,8 +232,7 @@ class Route(object):
                     response.set_status(408)
                     response.json({"message": "Request canceled", "status": 408})
                     raise  # must raise to let aiohttp know the connection has been closed
-                except aiohttp.ClientError:
-                    log.warning("Client error")
+                except (ConnectionResetError, aiohttp.ClientError):
                     response = Response(request=request, route=route)
                     response.set_status(408)
                     response.json({"message": "Client error", "status": 408})

requirements.txt

@@ -1,17 +1,18 @@
-jsonschema>=4.17.3,<4.18; python_version >= '3.7'
+jsonschema>=4.17.3,<4.18; python_version >= '3.7' # v4.17.3 is the last version to support Python 3.7
 jsonschema==3.2.0; python_version < '3.7'  # v3.2.0 is the last version to support Python 3.6
-aiohttp>=3.8.4,<3.9
+aiohttp>=3.8.5,<3.9
 aiohttp-cors>=0.7.0,<0.8
-aiofiles>=23.1.0,<23.2; python_version >= '3.7'
+aiofiles>=23.2.1,<23.3; python_version >= '3.7'
 aiofiles==0.8.0; python_version < '3.7'  # v0.8.0 is the last version to support Python 3.6
 Jinja2>=3.1.2,<3.2; python_version >= '3.7'
 Jinja2==3.0.3; python_version < '3.7'  # v3.0.3 is the last version to support Python 3.6
-sentry-sdk==1.29.2,<1.30
+sentry-sdk==1.31.0,<1.32
 psutil==5.9.5
 async-timeout>=4.0.2,<4.1
 distro>=1.8.0
 py-cpuinfo>=9.0.0,<10.0
+platformdirs>=2.4.0
 importlib-resources>=1.3; python_version <= '3.9'
-truststore>=0.7.0; python_version >= '3.10'
+truststore>=0.8.0; python_version >= '3.10'
 setuptools>=60.8.1; python_version >= '3.7'
 setuptools==59.6.0; python_version < '3.7'  # v59.6.0 is the last version to support Python 3.6

tests/compute/docker/test_docker_vm.py

@@ -182,7 +182,7 @@ async def test_create_vnc(compute_project, manager):
                         "Binds": [
                             "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
                             "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")),
-                            '/tmp/.X11-unix/:/tmp/.X11-unix/'
+                            "/tmp/.X11-unix/X{0}:/tmp/.X11-unix/X{0}:ro".format(vm._display)
                         ],
                         "Privileged": True
                     },

win-requirements.txt

@@ -1,4 +1,4 @@
 -rrequirements.txt
 
-pywin32==305
+pywin32==306
 wmi==1.5.1