diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 0f2222f77..214238408 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -63,6 +63,31 @@ log = logging.getLogger(__name__) FORBIDDEN_OPTIONS = {"-blockdev", "-drive", "-hda", "-hdb", "-hdc", "-hdd", "-fsdev", "-virtfs", "-nic", "-netdev"} FORBIDDEN_OPTIONS |= {"-" + opt for opt in FORBIDDEN_OPTIONS if opt.startswith("-") and not opt.startswith("--")} +CLOCKSOURCE_PATH = "/sys/devices/system/clocksource/clocksource0/current_clocksource" + +# Clocksources of a Linux host running on Hyper-V without a usable TSC. The guest +# timekeeping KVM derives from the TSC there is broken with recent kernels: kvmclock +# runs thousands of times too fast (Linux guests panic with soft lockups and reboot +# in a loop) and TSC deadline timers fire far too late (guests hang while booting). +HYPERV_CLOCKSOURCES = ("hyperv_clocksource_tsc_page", "hyperv_clocksource_msr") + +# guest CPU features turned off on such hosts +KVMCLOCK_OFF_FEATURES = ("kvmclock", "tsc-deadline") + + +def host_clocksource(): + """ + Returns the clocksource used by the host kernel (Linux only). + + :returns: clocksource name or None + """ + + try: + with open(CLOCKSOURCE_PATH, encoding="utf-8") as f: + return f.read().strip() + except OSError: + return None + class QemuVM(KernelDatapathMixin, BaseNode): module_name = "qemu" @@ -2763,6 +2788,44 @@ class QemuVM(KernelDatapathMixin, BaseNode): return True return False + def _kvmclock_off_reason(self): + """ + Returns why kvmclock and the TSC deadline timer must be turned off for this KVM guest. + + :returns: reason (string) or None to keep them + """ + + kvmclock = self.manager.config.settings.Qemu.kvmclock + if kvmclock == "disabled": + return "kvmclock = disabled in the [Qemu] server settings" + if kvmclock == "auto": + clocksource = host_clocksource() + if clocksource in HYPERV_CLOCKSOURCES: + return ( + f"the host runs on Hyper-V without a usable TSC (clocksource {clocksource}), " + "set kvmclock = enabled in the [Qemu] server settings to keep them" + ) + return None + + def _kvmclock_off_options(self, additional_options): + """ + Turns kvmclock and the TSC deadline timer off on the -cpu of the additional + options (except the features it sets itself) or, without one, on QEMU's + default CPU model. + + :param additional_options: additional options (list), updated in place + :returns: options to add to the command line + """ + + for index in range(len(additional_options) - 2, -1, -1): + if additional_options[index] in ("-cpu", "--cpu"): + for feature in KVMCLOCK_OFF_FEATURES: + if feature not in additional_options[index + 1]: + additional_options[index + 1] += f",{feature}=off" + return [] + cpu_model = "qemu32" if self._platform == "i386" else "qemu64" + return ["-cpu", ",".join([cpu_model] + [f"{feature}=off" for feature in KVMCLOCK_OFF_FEATURES])] + async def _clear_save_vm_stated(self, snapshot_name="GNS3_SAVED_STATE"): drives = ["a", "b", "c", "d"] @@ -2862,9 +2925,11 @@ class QemuVM(KernelDatapathMixin, BaseNode): if self._cpus > maxcpus: maxcpus = self._cpus command.extend(["-smp", f"cpus={self._cpus},maxcpus={maxcpus},sockets=1"]) + kvm = False if await self._run_with_hardware_acceleration(self.qemu_path, self._options): if sys.platform.startswith("linux"): command.extend(["-enable-kvm"]) + kvm = True elif sys.platform.startswith("darwin"): command.extend(["-enable-hax"]) command.extend(["-boot", f"order={self._boot_priority}"]) @@ -2887,6 +2952,7 @@ class QemuVM(KernelDatapathMixin, BaseNode): command.extend(await self._disable_graphics()) if self._tpm: command.extend(self._tpm_options()) + additional_opt_list = [] if additional_options: try: additional_opt_list = shlex.split(additional_options) @@ -2897,7 +2963,15 @@ class QemuVM(KernelDatapathMixin, BaseNode): for opt in additional_opt_list: if opt in FORBIDDEN_OPTIONS: raise QemuError(f"Forbidden additional option: {opt}") - command.extend(additional_opt_list) + if kvm: + kvmclock_off_reason = self._kvmclock_off_reason() + if kvmclock_off_reason: + log.info( + f'QEMU VM "{self._name}" [{self._id}]: kvmclock and TSC deadline timer turned off, ' + f"{kvmclock_off_reason}" + ) + command.extend(self._kvmclock_off_options(additional_opt_list)) + command.extend(additional_opt_list) # avoiding mouse offset (see https://github.com/GNS3/gns3-server/issues/2335) if self._console_type == "vnc": diff --git a/gns3server/config_samples/gns3_server.conf b/gns3server/config_samples/gns3_server.conf index 275f26358..b200029dd 100644 --- a/gns3server/config_samples/gns3_server.conf +++ b/gns3server/config_samples/gns3_server.conf @@ -187,6 +187,9 @@ monitor_host = 127.0.0.1 enable_hardware_acceleration = True ; Require hardware acceleration in order to start VMs require_hardware_acceleration = False +; kvmclock and TSC deadline timer of KVM guests: "auto" turns them off only when this server runs on +; Hyper-V without a usable TSC (guests panic with soft lockups or hang), "enabled" or "disabled" forces them +kvmclock = auto ; Allow unsafe additional command line options allow_unsafe_options = False ; Path to the OVMF firmware directory diff --git a/gns3server/schemas/config.py b/gns3server/schemas/config.py index 74dba2dde..1f20eeffe 100644 --- a/gns3server/schemas/config.py +++ b/gns3server/schemas/config.py @@ -79,6 +79,13 @@ class IOUSettings(BaseModel): model_config = ConfigDict(validate_assignment=True, str_strip_whitespace=True) +class QemuKvmclock(StrEnum): + # turned off only when the host runs on Hyper-V without a usable TSC + auto = "auto" + enabled = "enabled" + disabled = "disabled" + + class QemuSettings(BaseModel): enable_monitor: bool = Field(True, description="Use the Qemu monitor feature to communicate with Qemu VMs") monitor_host: str = Field("127.0.0.1", description="IP used to listen for the monitor") @@ -86,6 +93,12 @@ class QemuSettings(BaseModel): require_hardware_acceleration: bool = Field( False, description="Require hardware acceleration in order to start VMs" ) + kvmclock: QemuKvmclock = Field( + QemuKvmclock.auto, + description="kvmclock and TSC deadline timer of KVM guests: 'auto' turns them off when this server runs on " + "Hyper-V without a usable TSC (guest clocks run away there, Linux guests panic with soft lockups or hang), " + "'enabled' or 'disabled' forces them", + ) allow_unsafe_options: bool = Field(False, description="Allow unsafe additional command line options") ovmf_firmware_dir: str = Field("/usr/share/OVMF", description="Path to the OVMF firmware directory") model_config = ConfigDict(validate_assignment=True, str_strip_whitespace=True) diff --git a/gns3server/schemas/controller/settings.py b/gns3server/schemas/controller/settings.py index 600b41ba7..6bfd1e4a2 100644 --- a/gns3server/schemas/controller/settings.py +++ b/gns3server/schemas/controller/settings.py @@ -35,6 +35,7 @@ from ..config import ( ControllerSettings, DynamipsSettings, IOUSettings, + QemuKvmclock, QemuSettings, ServerProtocol, ServerSettings, @@ -194,6 +195,7 @@ class QemuSettingsUpdate(BaseModel): monitor_host: Optional[str] = None enable_hardware_acceleration: Optional[bool] = None require_hardware_acceleration: Optional[bool] = None + kvmclock: Optional[QemuKvmclock] = None allow_unsafe_options: Optional[bool] = None ovmf_firmware_dir: Optional[str] = None diff --git a/tests/api/routes/controller/test_settings.py b/tests/api/routes/controller/test_settings.py index f7500a295..e64b12712 100644 --- a/tests/api/routes/controller/test_settings.py +++ b/tests/api/routes/controller/test_settings.py @@ -204,6 +204,20 @@ class TestSettingsRoutes: response = await client.put(app.url_path_for("update_server_settings"), json={"Server": {"prot": "http"}}) assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + async def test_put_settings_qemu_kvmclock( + self, app: FastAPI, client: AsyncClient, config: Config, stable_jwt_secret: str + ) -> None: + + response = await client.put(app.url_path_for("update_server_settings"), json={"Qemu": {"kvmclock": "disabled"}}) + assert response.status_code == status.HTTP_200_OK + assert response.json()["Qemu"]["kvmclock"] == "disabled" + parsed = configparser.ConfigParser() + parsed.read(config._main_config_file) + assert parsed["Qemu"]["kvmclock"] == "disabled" + + response = await client.put(app.url_path_for("update_server_settings"), json={"Qemu": {"kvmclock": "off"}}) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + async def test_put_settings_deprecated_sections_rejected( self, app: FastAPI, client: AsyncClient, stable_jwt_secret: str ) -> None: diff --git a/tests/compute/qemu/test_qemu_vm.py b/tests/compute/qemu/test_qemu_vm.py index f48418d4b..954507e07 100644 --- a/tests/compute/qemu/test_qemu_vm.py +++ b/tests/compute/qemu/test_qemu_vm.py @@ -29,7 +29,7 @@ import pytest_asyncio from gns3server.compute.notification_manager import NotificationManager from gns3server.compute.qemu import Qemu from gns3server.compute.qemu.qemu_error import QemuError -from gns3server.compute.qemu.qemu_vm import QemuVM +from gns3server.compute.qemu.qemu_vm import QemuVM, host_clocksource from gns3server.utils import force_unix_path, int_to_macaddress, macaddress_to_int from tests.utils import AsyncioMagicMock, asyncio_patch @@ -815,6 +815,110 @@ async def test_build_command_with_forbidden_options(vm): await vm._build_command() +async def _build_command_with_kvm(vm, clocksource): + """ + Builds the command line as on a KVM host whose kernel uses the given clocksource. + """ + + with asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._run_with_hardware_acceleration", return_value=True): + with patch("gns3server.compute.qemu.qemu_vm.host_clocksource", return_value=clocksource): + with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()): + return await vm._build_command() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("clocksource", ["hyperv_clocksource_tsc_page", "hyperv_clocksource_msr"]) +async def test_build_command_kvmclock_off_on_hyperv_host(linux_platform, vm, clocksource): + """ + A host running on Hyper-V without a usable TSC falls back to a Hyper-V clocksource: + kvmclock runs away in KVM guests there (Linux guests panic with soft lockups) and + TSC deadline timers fire far too late (guests hang), so both are turned off on + QEMU's default CPU model. + """ + + cmd = await _build_command_with_kvm(vm, clocksource) + assert "-enable-kvm" in cmd + assert cmd[cmd.index("-cpu") + 1] == "qemu64,kvmclock=off,tsc-deadline=off" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("clocksource", ["tsc", None]) +async def test_build_command_kvmclock_kept_on_other_hosts(linux_platform, vm, clocksource): + + cmd = await _build_command_with_kvm(vm, clocksource) + assert "-enable-kvm" in cmd + assert "-cpu" not in cmd + + +@pytest.mark.asyncio +async def test_build_command_kvmclock_kept_without_kvm(vm): + + with patch("gns3server.compute.qemu.qemu_vm.host_clocksource", return_value="hyperv_clocksource_tsc_page"): + with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()): + cmd = await vm._build_command() + assert "-cpu" not in cmd + + +@pytest.mark.asyncio +async def test_build_command_kvmclock_off_on_user_cpu(linux_platform, vm): + """ + A -cpu given in the additional options keeps its model and gets the features turned off. + """ + + vm.options = "-nographic -cpu host" + cmd = await _build_command_with_kvm(vm, "hyperv_clocksource_tsc_page") + assert cmd.count("-cpu") == 1 + assert cmd[cmd.index("-cpu") + 1] == "host,kvmclock=off,tsc-deadline=off" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cpu,expected", + [ + ("host,kvmclock=on", "host,kvmclock=on,tsc-deadline=off"), + ("max,-kvmclock,+tsc-deadline", "max,-kvmclock,+tsc-deadline"), + ], +) +async def test_build_command_kvmclock_user_choice_kept(linux_platform, vm, cpu, expected): + + vm.options = f"-cpu {cpu}" + cmd = await _build_command_with_kvm(vm, "hyperv_clocksource_tsc_page") + assert cmd.count("-cpu") == 1 + assert cmd[cmd.index("-cpu") + 1] == expected + + +@pytest.mark.asyncio +async def test_build_command_kvmclock_off_i386(linux_platform, vm): + + vm._platform = "i386" + cmd = await _build_command_with_kvm(vm, "hyperv_clocksource_tsc_page") + assert cmd[cmd.index("-cpu") + 1] == "qemu32,kvmclock=off,tsc-deadline=off" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "setting,clocksource,kvmclock_off", + [ + ("disabled", "tsc", True), + ("enabled", "hyperv_clocksource_tsc_page", False), + ], +) +async def test_build_command_kvmclock_setting(linux_platform, vm, setting, clocksource, kvmclock_off): + + vm.manager.config.settings.Qemu.kvmclock = setting + cmd = await _build_command_with_kvm(vm, clocksource) + assert ("qemu64,kvmclock=off,tsc-deadline=off" in cmd) is kvmclock_off + + +def test_host_clocksource(monkeypatch, tmpdir): + + path = tmpdir / "current_clocksource" + monkeypatch.setattr("gns3server.compute.qemu.qemu_vm.CLOCKSOURCE_PATH", str(path)) + assert host_clocksource() is None + path.write_text("hyperv_clocksource_tsc_page\n", encoding="utf-8") + assert host_clocksource() == "hyperv_clocksource_tsc_page" + + def test_hda_disk_image(vm, images_dir): open(os.path.join(images_dir, "test1"), "w+").close()