Merge pull request #3025 from tikD1337/fix/qemu-kvmclock-hyperv

fix(qemu): turn off kvmclock and TSC deadline timer on Hyper-V hosts without a usable TSC

gns3server/compute/qemu/qemu_vm.py

@@ -63,6 +63,31 @@ log = logging.getLogger(__name__)
 FORBIDDEN_OPTIONS = {"-blockdev", "-drive", "-hda", "-hdb", "-hdc", "-hdd", "-fsdev", "-virtfs", "-nic", "-netdev"}
 FORBIDDEN_OPTIONS |= {"-" + opt for opt in FORBIDDEN_OPTIONS if opt.startswith("-") and not opt.startswith("--")}
 
+CLOCKSOURCE_PATH = "/sys/devices/system/clocksource/clocksource0/current_clocksource"
+
+# Clocksources of a Linux host running on Hyper-V without a usable TSC. The guest
+# timekeeping KVM derives from the TSC there is broken with recent kernels: kvmclock
+# runs thousands of times too fast (Linux guests panic with soft lockups and reboot
+# in a loop) and TSC deadline timers fire far too late (guests hang while booting).
+HYPERV_CLOCKSOURCES = ("hyperv_clocksource_tsc_page", "hyperv_clocksource_msr")
+
+# guest CPU features turned off on such hosts
+KVMCLOCK_OFF_FEATURES = ("kvmclock", "tsc-deadline")
+
+
+def host_clocksource():
+    """
+    Returns the clocksource used by the host kernel (Linux only).
+
+    :returns: clocksource name or None
+    """
+
+    try:
+        with open(CLOCKSOURCE_PATH, encoding="utf-8") as f:
+            return f.read().strip()
+    except OSError:
+        return None
+
 
 class QemuVM(KernelDatapathMixin, BaseNode):
     module_name = "qemu"
@@ -2763,6 +2788,44 @@ class QemuVM(KernelDatapathMixin, BaseNode):
             return True
         return False
 
+    def _kvmclock_off_reason(self):
+        """
+        Returns why kvmclock and the TSC deadline timer must be turned off for this KVM guest.
+
+        :returns: reason (string) or None to keep them
+        """
+
+        kvmclock = self.manager.config.settings.Qemu.kvmclock
+        if kvmclock == "disabled":
+            return "kvmclock = disabled in the [Qemu] server settings"
+        if kvmclock == "auto":
+            clocksource = host_clocksource()
+            if clocksource in HYPERV_CLOCKSOURCES:
+                return (
+                    f"the host runs on Hyper-V without a usable TSC (clocksource {clocksource}), "
+                    "set kvmclock = enabled in the [Qemu] server settings to keep them"
+                )
+        return None
+
+    def _kvmclock_off_options(self, additional_options):
+        """
+        Turns kvmclock and the TSC deadline timer off on the -cpu of the additional
+        options (except the features it sets itself) or, without one, on QEMU's
+        default CPU model.
+
+        :param additional_options: additional options (list), updated in place
+        :returns: options to add to the command line
+        """
+
+        for index in range(len(additional_options) - 2, -1, -1):
+            if additional_options[index] in ("-cpu", "--cpu"):
+                for feature in KVMCLOCK_OFF_FEATURES:
+                    if feature not in additional_options[index + 1]:
+                        additional_options[index + 1] += f",{feature}=off"
+                return []
+        cpu_model = "qemu32" if self._platform == "i386" else "qemu64"
+        return ["-cpu", ",".join([cpu_model] + [f"{feature}=off" for feature in KVMCLOCK_OFF_FEATURES])]
+
     async def _clear_save_vm_stated(self, snapshot_name="GNS3_SAVED_STATE"):
 
         drives = ["a", "b", "c", "d"]
@@ -2862,9 +2925,11 @@ class QemuVM(KernelDatapathMixin, BaseNode):
         if self._cpus > maxcpus:
             maxcpus = self._cpus
         command.extend(["-smp", f"cpus={self._cpus},maxcpus={maxcpus},sockets=1"])
+        kvm = False
         if await self._run_with_hardware_acceleration(self.qemu_path, self._options):
             if sys.platform.startswith("linux"):
                 command.extend(["-enable-kvm"])
+                kvm = True
             elif sys.platform.startswith("darwin"):
                 command.extend(["-enable-hax"])
         command.extend(["-boot", f"order={self._boot_priority}"])
@@ -2887,6 +2952,7 @@ class QemuVM(KernelDatapathMixin, BaseNode):
             command.extend(await self._disable_graphics())
         if self._tpm:
             command.extend(self._tpm_options())
+        additional_opt_list = []
         if additional_options:
             try:
                 additional_opt_list = shlex.split(additional_options)
@@ -2897,7 +2963,15 @@ class QemuVM(KernelDatapathMixin, BaseNode):
                 for opt in additional_opt_list:
                     if opt in FORBIDDEN_OPTIONS:
                         raise QemuError(f"Forbidden additional option: {opt}")
-            command.extend(additional_opt_list)
+        if kvm:
+            kvmclock_off_reason = self._kvmclock_off_reason()
+            if kvmclock_off_reason:
+                log.info(
+                    f'QEMU VM "{self._name}" [{self._id}]: kvmclock and TSC deadline timer turned off, '
+                    f"{kvmclock_off_reason}"
+                )
+                command.extend(self._kvmclock_off_options(additional_opt_list))
+        command.extend(additional_opt_list)
 
         # avoiding mouse offset (see https://github.com/GNS3/gns3-server/issues/2335)
         if self._console_type == "vnc":

gns3server/config_samples/gns3_server.conf

@@ -187,6 +187,9 @@ monitor_host = 127.0.0.1
 enable_hardware_acceleration = True
 ; Require hardware acceleration in order to start VMs
 require_hardware_acceleration = False
+; kvmclock and TSC deadline timer of KVM guests: "auto" turns them off only when this server runs on
+; Hyper-V without a usable TSC (guests panic with soft lockups or hang), "enabled" or "disabled" forces them
+kvmclock = auto
 ; Allow unsafe additional command line options
 allow_unsafe_options = False
 ; Path to the OVMF firmware directory

gns3server/schemas/config.py

@@ -79,6 +79,13 @@ class IOUSettings(BaseModel):
     model_config = ConfigDict(validate_assignment=True, str_strip_whitespace=True)
 
 
+class QemuKvmclock(StrEnum):
+    # turned off only when the host runs on Hyper-V without a usable TSC
+    auto = "auto"
+    enabled = "enabled"
+    disabled = "disabled"
+
+
 class QemuSettings(BaseModel):
     enable_monitor: bool = Field(True, description="Use the Qemu monitor feature to communicate with Qemu VMs")
     monitor_host: str = Field("127.0.0.1", description="IP used to listen for the monitor")
@@ -86,6 +93,12 @@ class QemuSettings(BaseModel):
     require_hardware_acceleration: bool = Field(
         False, description="Require hardware acceleration in order to start VMs"
     )
+    kvmclock: QemuKvmclock = Field(
+        QemuKvmclock.auto,
+        description="kvmclock and TSC deadline timer of KVM guests: 'auto' turns them off when this server runs on "
+        "Hyper-V without a usable TSC (guest clocks run away there, Linux guests panic with soft lockups or hang), "
+        "'enabled' or 'disabled' forces them",
+    )
     allow_unsafe_options: bool = Field(False, description="Allow unsafe additional command line options")
     ovmf_firmware_dir: str = Field("/usr/share/OVMF", description="Path to the OVMF firmware directory")
     model_config = ConfigDict(validate_assignment=True, str_strip_whitespace=True)

gns3server/schemas/controller/settings.py

@@ -35,6 +35,7 @@ from ..config import (
     ControllerSettings,
     DynamipsSettings,
     IOUSettings,
+    QemuKvmclock,
     QemuSettings,
     ServerProtocol,
     ServerSettings,
@@ -194,6 +195,7 @@ class QemuSettingsUpdate(BaseModel):
     monitor_host: Optional[str] = None
     enable_hardware_acceleration: Optional[bool] = None
     require_hardware_acceleration: Optional[bool] = None
+    kvmclock: Optional[QemuKvmclock] = None
     allow_unsafe_options: Optional[bool] = None
     ovmf_firmware_dir: Optional[str] = None
 

tests/api/routes/controller/test_settings.py

@@ -204,6 +204,20 @@ class TestSettingsRoutes:
         response = await client.put(app.url_path_for("update_server_settings"), json={"Server": {"prot": "http"}})
         assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT
 
+    async def test_put_settings_qemu_kvmclock(
+        self, app: FastAPI, client: AsyncClient, config: Config, stable_jwt_secret: str
+    ) -> None:
+
+        response = await client.put(app.url_path_for("update_server_settings"), json={"Qemu": {"kvmclock": "disabled"}})
+        assert response.status_code == status.HTTP_200_OK
+        assert response.json()["Qemu"]["kvmclock"] == "disabled"
+        parsed = configparser.ConfigParser()
+        parsed.read(config._main_config_file)
+        assert parsed["Qemu"]["kvmclock"] == "disabled"
+
+        response = await client.put(app.url_path_for("update_server_settings"), json={"Qemu": {"kvmclock": "off"}})
+        assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT
+
     async def test_put_settings_deprecated_sections_rejected(
         self, app: FastAPI, client: AsyncClient, stable_jwt_secret: str
     ) -> None:

tests/compute/qemu/test_qemu_vm.py

@@ -29,7 +29,7 @@ import pytest_asyncio
 from gns3server.compute.notification_manager import NotificationManager
 from gns3server.compute.qemu import Qemu
 from gns3server.compute.qemu.qemu_error import QemuError
-from gns3server.compute.qemu.qemu_vm import QemuVM
+from gns3server.compute.qemu.qemu_vm import QemuVM, host_clocksource
 from gns3server.utils import force_unix_path, int_to_macaddress, macaddress_to_int
 from tests.utils import AsyncioMagicMock, asyncio_patch
 
@@ -815,6 +815,110 @@ async def test_build_command_with_forbidden_options(vm):
         await vm._build_command()
 
 
+async def _build_command_with_kvm(vm, clocksource):
+    """
+    Builds the command line as on a KVM host whose kernel uses the given clocksource.
+    """
+
+    with asyncio_patch("gns3server.compute.qemu.qemu_vm.QemuVM._run_with_hardware_acceleration", return_value=True):
+        with patch("gns3server.compute.qemu.qemu_vm.host_clocksource", return_value=clocksource):
+            with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()):
+                return await vm._build_command()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("clocksource", ["hyperv_clocksource_tsc_page", "hyperv_clocksource_msr"])
+async def test_build_command_kvmclock_off_on_hyperv_host(linux_platform, vm, clocksource):
+    """
+    A host running on Hyper-V without a usable TSC falls back to a Hyper-V clocksource:
+    kvmclock runs away in KVM guests there (Linux guests panic with soft lockups) and
+    TSC deadline timers fire far too late (guests hang), so both are turned off on
+    QEMU's default CPU model.
+    """
+
+    cmd = await _build_command_with_kvm(vm, clocksource)
+    assert "-enable-kvm" in cmd
+    assert cmd[cmd.index("-cpu") + 1] == "qemu64,kvmclock=off,tsc-deadline=off"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("clocksource", ["tsc", None])
+async def test_build_command_kvmclock_kept_on_other_hosts(linux_platform, vm, clocksource):
+
+    cmd = await _build_command_with_kvm(vm, clocksource)
+    assert "-enable-kvm" in cmd
+    assert "-cpu" not in cmd
+
+
+@pytest.mark.asyncio
+async def test_build_command_kvmclock_kept_without_kvm(vm):
+
+    with patch("gns3server.compute.qemu.qemu_vm.host_clocksource", return_value="hyperv_clocksource_tsc_page"):
+        with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()):
+            cmd = await vm._build_command()
+    assert "-cpu" not in cmd
+
+
+@pytest.mark.asyncio
+async def test_build_command_kvmclock_off_on_user_cpu(linux_platform, vm):
+    """
+    A -cpu given in the additional options keeps its model and gets the features turned off.
+    """
+
+    vm.options = "-nographic -cpu host"
+    cmd = await _build_command_with_kvm(vm, "hyperv_clocksource_tsc_page")
+    assert cmd.count("-cpu") == 1
+    assert cmd[cmd.index("-cpu") + 1] == "host,kvmclock=off,tsc-deadline=off"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "cpu,expected",
+    [
+        ("host,kvmclock=on", "host,kvmclock=on,tsc-deadline=off"),
+        ("max,-kvmclock,+tsc-deadline", "max,-kvmclock,+tsc-deadline"),
+    ],
+)
+async def test_build_command_kvmclock_user_choice_kept(linux_platform, vm, cpu, expected):
+
+    vm.options = f"-cpu {cpu}"
+    cmd = await _build_command_with_kvm(vm, "hyperv_clocksource_tsc_page")
+    assert cmd.count("-cpu") == 1
+    assert cmd[cmd.index("-cpu") + 1] == expected
+
+
+@pytest.mark.asyncio
+async def test_build_command_kvmclock_off_i386(linux_platform, vm):
+
+    vm._platform = "i386"
+    cmd = await _build_command_with_kvm(vm, "hyperv_clocksource_tsc_page")
+    assert cmd[cmd.index("-cpu") + 1] == "qemu32,kvmclock=off,tsc-deadline=off"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "setting,clocksource,kvmclock_off",
+    [
+        ("disabled", "tsc", True),
+        ("enabled", "hyperv_clocksource_tsc_page", False),
+    ],
+)
+async def test_build_command_kvmclock_setting(linux_platform, vm, setting, clocksource, kvmclock_off):
+
+    vm.manager.config.settings.Qemu.kvmclock = setting
+    cmd = await _build_command_with_kvm(vm, clocksource)
+    assert ("qemu64,kvmclock=off,tsc-deadline=off" in cmd) is kvmclock_off
+
+
+def test_host_clocksource(monkeypatch, tmpdir):
+
+    path = tmpdir / "current_clocksource"
+    monkeypatch.setattr("gns3server.compute.qemu.qemu_vm.CLOCKSOURCE_PATH", str(path))
+    assert host_clocksource() is None
+    path.write_text("hyperv_clocksource_tsc_page\n", encoding="utf-8")
+    assert host_clocksource() == "hyperv_clocksource_tsc_page"
+
+
 def test_hda_disk_image(vm, images_dir):
 
     open(os.path.join(images_dir, "test1"), "w+").close()