Merge branch '3.0' into busybox-docker

gns3server/compute/builtin/nodes/cloud.py

@@ -430,6 +430,7 @@ class Cloud(BaseNode):
             await self._add_ubridge_connection(nio, port_number)
             self._nios[port_number] = nio
         except (NodeError, UbridgeError) as e:
+            log.error('Cannot add NIO on cloud "{name}": {error}'.format(name=self._name, error=e))
             await self._stop_ubridge()
             self.status = "stopped"
             self._nios[port_number] = nio

gns3server/compute/qemu/qemu_vm.py

@@ -669,7 +669,7 @@ class QemuVM(BaseNode):
 
         if not mac_address:
             # use the node UUID to generate a random MAC address
-            self._mac_address = f"0c:{self.project.id[-4:-2]}:{self.project.id[-2:]}:{self.id[-4:-2]}:{self.id[-2:]}:00"
+            self._mac_address = f"0c:{self.id[2:4]}:{self.id[4:6]}:{self.id[6:8]}:00:00"
         else:
             self._mac_address = mac_address
 
@@ -912,20 +912,26 @@ class QemuVM(BaseNode):
             )
         )
 
-        if not sys.platform.startswith("linux"):
-            if "-no-kvm" in options:
-                options = options.replace("-no-kvm", "")
-            if "-enable-kvm" in options:
+        # "-no-kvm" and "-no-hax' are deprecated since Qemu v5.2
+        if "-no-kvm" in options:
+            options = options.replace("-no-kvm", "-machine accel=tcg")
+        if "-no-hax" in options:
+            options = options.replace("-no-hax", "-machine accel=tcg")
+
+        if "-enable-kvm" in options:
+            if not sys.platform.startswith("linux"):
+                # KVM can only be enabled on Linux
                 options = options.replace("-enable-kvm", "")
-        else:
-            if "-no-hax" in options:
-                options = options.replace("-no-hax", "")
-            if "-enable-hax" in options:
+            else:
+                options = options.replace("-enable-kvm", "-machine accel=kvm")
+
+        if "-enable-hax" in options:
+            if not sys.platform.startswith("win"):
+                # HAXM is only available on Windows
                 options = options.replace("-enable-hax", "")
-            if "-icount" in options and ("-no-kvm" not in options):
-                # automatically add the -no-kvm option if -icount is detected
-                # to help with the migration of ASA VMs created before version 1.4
-                options = "-no-kvm " + options
+            else:
+                options = options.replace("-enable-hax", "-machine accel=hax")
+
         self._options = options.strip()
 
     @property
@@ -1862,6 +1868,24 @@ class QemuVM(BaseNode):
         try:
             qemu_img_path = self._get_qemu_img()
             command = [qemu_img_path, "create", "-o", f"backing_file={disk_image}", "-f", "qcow2", disk]
+            try:
+                base_qcow2 = Qcow2(disk_image)
+                if base_qcow2.crypt_method:
+                    # Workaround for https://gitlab.com/qemu-project/qemu/-/issues/441
+                    # Also embed a secret name so it doesn't have to be passed to qemu -drive ...
+                    options = {
+                        "encrypt.key-secret": os.path.basename(disk_image),
+                        "driver": "qcow2",
+                        "file": {
+                            "driver": "file",
+                            "filename": disk_image,
+                        },
+                    }
+                    command = [qemu_img_path, "create", "-b", "json:"+json.dumps(options, separators=(',', ':')),
+                               "-f", "qcow2", "-u", disk, str(base_qcow2.size)]
+            except Qcow2Error:
+                pass  # non-qcow2 base images are acceptable (e.g. vmdk, raw image)
+
             retcode = await self._qemu_img_exec(command)
             if retcode:
                 stdout = self.read_qemu_img_stdout()
@@ -2070,7 +2094,7 @@ class QemuVM(BaseNode):
                     # The disk exists we check if the clone works
                     try:
                         qcow2 = Qcow2(disk)
-                        await qcow2.rebase(qemu_img_path, disk_image)
+                        await qcow2.validate(qemu_img_path)
                     except (Qcow2Error, OSError) as e:
                         raise QemuError(f"Could not use qcow2 disk image '{disk_image}' for {disk_name} {e}")
 
@@ -2310,7 +2334,7 @@ class QemuVM(BaseNode):
 
         enable_hardware_accel = self.manager.config.settings.Qemu.enable_hardware_acceleration
         require_hardware_accel = self.manager.config.settings.Qemu.require_hardware_acceleration
-        if enable_hardware_accel and "-no-kvm" not in options and "-no-hax" not in options:
+        if enable_hardware_accel and "-machine accel=tcg" not in options:
             # Turn OFF hardware acceleration for non x86 architectures
             if sys.platform.startswith("win"):
                 supported_binaries = [

gns3server/compute/qemu/utils/qcow2.py

@@ -58,13 +58,12 @@ class Qcow2:
         #     uint64_t snapshots_offset;
         # } QCowHeader;
 
-        struct_format = ">IIQi"
-        with open(self._path, "rb") as f:
+        struct_format = ">IIQiiQi"
+        with open(self._path, 'rb') as f:
             content = f.read(struct.calcsize(struct_format))
             try:
-                self.magic, self.version, self.backing_file_offset, self.backing_file_size = struct.unpack_from(
-                    struct_format, content
-                )
+                (self.magic, self.version, self.backing_file_offset, self.backing_file_size,
+                    self.cluster_bits, self.size, self.crypt_method) = struct.unpack_from(struct_format, content)
 
             except struct.error:
                 raise Qcow2Error(f"Invalid file header for {self._path}")
@@ -105,3 +104,15 @@ class Qcow2:
         if retcode != 0:
             raise Qcow2Error("Could not rebase the image")
         self._reload()
+
+    async def validate(self, qemu_img):
+        """
+        Run qemu-img info to validate the file and its backing images
+
+        :param qemu_img: Path to the qemu-img binary
+        """
+        command = [qemu_img, "info", "--backing-chain", self._path]
+        process = await asyncio.create_subprocess_exec(*command)
+        retcode = await process.wait()
+        if retcode != 0:
+            raise Qcow2Error("Could not validate the image")

gns3server/compute/vmware/__init__.py

@@ -26,6 +26,7 @@ import asyncio
 import subprocess
 import logging
 import codecs
+import ipaddress
 
 from collections import OrderedDict
 from gns3server.utils.interfaces import interfaces
@@ -50,6 +51,7 @@ class VMware(BaseManager):
         self._vmrun_path = None
         self._host_type = None
         self._vmnets = []
+        self._vmnets_info = {}
         self._vmnet_start_range = 2
         if sys.platform.startswith("win"):
             self._vmnet_end_range = 19
@@ -277,7 +279,7 @@ class VMware(BaseManager):
         else:
             # location on Linux
             vmware_networking_file = "/etc/vmware/networking"
-        vmnet_interfaces = []
+        vmnet_interfaces = {}
         try:
             with open(vmware_networking_file, encoding="utf-8") as f:
                 for line in f.read().splitlines():
@@ -285,7 +287,20 @@ class VMware(BaseManager):
                     if match:
                         vmnet = f"vmnet{match.group(1)}"
                         if vmnet not in ("vmnet0", "vmnet1", "vmnet8"):
-                            vmnet_interfaces.append(vmnet)
+                            vmnet_interfaces[vmnet] = {}
+            with open(vmware_networking_file, "r", encoding="utf-8") as f:
+                for line in f.read().splitlines():
+                    match = re.search(r"VNET_([0-9]+)_HOSTONLY_SUBNET\s+(.*)", line)
+                    if match:
+                        vmnet = "vmnet{}".format(match.group(1))
+                        if vmnet in vmnet_interfaces.keys():
+                            vmnet_interfaces[vmnet]["subnet"] = match.group(2)
+                    match = re.search(r"VNET_([0-9]+)_HOSTONLY_NETMASK\s+(.*)", line)
+                    if match:
+                        vmnet = "vmnet{}".format(match.group(1))
+                        if vmnet in vmnet_interfaces.keys():
+                            vmnet_interfaces[vmnet]["netmask"] = match.group(2)
+
         except OSError as e:
             raise VMwareError(f"Cannot open {vmware_networking_file}: {e}")
         return vmnet_interfaces
@@ -330,6 +345,25 @@ class VMware(BaseManager):
             )
         return self._vmnets.pop(0)
 
+    def find_bridge_interface(self, vmnet_interface):
+        """
+        Find the bridge interface that is used for the vmnet interface in VMware.
+        """
+
+        if vmnet_interface in self._vmnets_info.keys():
+            subnet = self._vmnets_info[vmnet_interface].get("subnet", None)
+            netmask = self._vmnets_info[vmnet_interface].get("netmask", None)
+            if subnet and netmask:
+                for interface in interfaces():
+                    try:
+                        network = ipaddress.ip_network(f"{subnet}/{netmask}")
+                        ip = ipaddress.ip_address(interface["ip_address"])
+                    except ValueError:
+                        continue
+                    if ip in network:
+                        return interface["name"]
+        return None
+
     def refresh_vmnet_list(self, ubridge=True):
 
         if ubridge:
@@ -337,6 +371,8 @@ class VMware(BaseManager):
             vmnet_interfaces = self._get_vmnet_interfaces_ubridge()
         else:
             vmnet_interfaces = self._get_vmnet_interfaces()
+            vmnet_interfaces = list(vmnet_interfaces.keys())
+            self._vmnets_info = vmnet_interfaces.copy()
 
         # remove vmnets already in use
         for vmware_vm in self._nodes.values():
@@ -754,5 +790,4 @@ class VMware(BaseManager):
 if __name__ == "__main__":
     loop = asyncio.get_event_loop()
     vmware = VMware.instance()
-    print("=> Check version")
     loop.run_until_complete(asyncio.ensure_future(vmware.check_vmware_version()))

gns3server/compute/vmware/vmware_vm.py

@@ -22,9 +22,11 @@ import sys
 import os
 import asyncio
 import tempfile
+import platform
 
 from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer
 from gns3server.utils.asyncio.serial import asyncio_open_serial
+from gns3server.utils import parse_version
 from gns3server.utils.asyncio import locking
 from collections import OrderedDict
 from .vmware_error import VMwareError
@@ -260,8 +262,13 @@ class VMwareVM(BaseNode):
             if self._get_vmx_setting(connected):
                 del self._vmx_pairs[connected]
 
+        use_ubridge = True
+        # use alternative method to find vmnet interfaces on macOS >= 11.0 (BigSur)
+        # because "bridge" interfaces are used instead and they are only created on the VM starts
+        if sys.platform.startswith("darwin") and parse_version(platform.mac_ver()[0]) >= parse_version("11.0.0"):
+            use_ubridge = False
+        self.manager.refresh_vmnet_list(ubridge=use_ubridge)
         # then configure VMware network adapters
-        self.manager.refresh_vmnet_list()
         for adapter_number in range(0, self._adapters):
 
             custom_adapter = self._get_custom_adapter_settings(adapter_number)
@@ -347,8 +354,17 @@ class VMwareVM(BaseNode):
         vmnet_interface = os.path.basename(self._vmx_pairs[vnet])
 
         if sys.platform.startswith("darwin"):
-            # special case on OSX, we cannot bind VMnet interfaces using the libpcap
-            await self._ubridge_send(f'bridge add_nio_fusion_vmnet {vnet} "{vmnet_interface}"')
+            if parse_version(platform.mac_ver()[0]) >= parse_version("11.0.0"):
+                # a bridge interface (bridge100, bridge101 etc.) is used instead of a vmnet interface
+                # on macOS >= 11.0 (Big Sur)
+                vmnet_interface = self.manager.find_bridge_interface(vmnet_interface)
+                if not vmnet_interface:
+                    raise VMwareError(f"Could not find bridge interface linked with {vmnet_interface}")
+                block_host_traffic = self.manager.config.get_section_config("VMware").getboolean("block_host_traffic", False)
+                await self._add_ubridge_ethernet_connection(vnet, vmnet_interface, block_host_traffic)
+            else:
+                # special case on macOS, we cannot bind VMnet interfaces using the libpcap
+                await self._ubridge_send('bridge add_nio_fusion_vmnet {name} "{interface}"'.format(name=vnet, interface=vmnet_interface))
         else:
             block_host_traffic = self.manager.config.VMware.block_host_traffic
             await self._add_ubridge_ethernet_connection(vnet, vmnet_interface, block_host_traffic)