| | commit e1c37ea4d4a74fb6addcfeede9a34d521de44fa5 |
| | Author: Anees Iqbal hello@aneesiqbal.ai |
| | Date: Thu Oct 1 05:31:30 2026 +0300 |
| | hw/vmapple: boot macOS 14 and later guests |
| | macOS 14 and later guests stop in their kernel's first instructions: |
| | the kernel spins with "hvc #0; cbnz x0, ." on each of its pointer |
| | authentication hypercalls, which the public Hypervisor.framework API |
| | fails without an exit. macOS 15 and later also panic when APFS mounts |
| | their unencrypted data volume ("unencrypted data volume is not |
| | allowed"), unless the device tree has /filesystems/DoNotUseSEP, and |
| | reset in a loop. |
| | Make two changes to the guest when iBoot enters the kernel: |
| | - Replace every "cbnz x0, ." that follows an "hvc #0" in the kernel's |
| | memory, from the boot arguments' physBase to their topOfKernelData, |
| | with a nop, so the kernel runs on the firmware's keys, as macOS 13's |
| | does. 14.6.1, 15.6.1 and 26.6.2 have 19 such sites; 13.6 has none. |
| | - If the device tree has no /filesystems node, append one that has |
| | DoNotUseSEP as the root's last child, in the zeroes that pad the tree |
| | to its last 16 KiB page, and cover it with the tree's length in the |
| | boot arguments. 13.6's tree has the node, and is left as it is. |
| | To catch the kernel's entry, RAM is not executable from reset. Each page |
| | iBoot runs becomes executable on its first fetch, until a fetch lands in |
| | the kernel's memory with x0 at the boot arguments; only the boot CPU |
| | runs then. HVF reports these faults through a new hook, |
| | hvf_arm_exec_fault, which a machine can set to handle a stage 2 fault |
| | on an instruction fetch itself. |
| | With this, 13.6, 14.6.1, 15.6.1 and 26.6.2 guests reach their desktops. |
| | diff --git a/docs/system/arm/vmapple.rst b/docs/system/arm/vmapple.rst |
| | index 0994d6d0f9..b60ecf7942 100644 |
| | --- a/docs/system/arm/vmapple.rst |
| | +++ b/docs/system/arm/vmapple.rst |
| | @@ -14,10 +14,9 @@ To run the vmapple machine model, you need to |
| | * Run on Apple Silicon |
| | * Run on macOS 12.0 or above |
| | - * Have an already installed copy of a Virtualization.Framework macOS 12 or 13 |
| | - virtual machine. Newer versions are NOT supported on the guest side (see |
| | - Guest versions). I will assume that you installed it using the |
| | - macosvm <https://github.com/s-u/macosvm>__ CLI. |
| | + * Have an already installed copy of a Virtualization.Framework macOS 12 or |
| | + later virtual machine (see Guest versions). I will assume that you |
| | + installed it using the macosvm <https://github.com/s-u/macosvm>__ CLI. |
| | * Build QEMU with apple-gfx, which the machine's display needs. apple-gfx |
| | drives reims-vgpu's C API, a library that pkg-config finds as |
| | reims-vgpu: pass --enable-pvg with PKG_CONFIG_PATH naming the |
| | @@ -73,21 +72,43 @@ to get better interactive access into the target system: |
| | Guest versions |
| | -------------- |
| | -macOS 12 and 13 guests boot. A vmapple guest kernel asks the hypervisor to |
| | -manage its pointer authentication keys through paravirtualized hypercalls: |
| | -hvc #0 with SMCCC function IDs 0xC1000000 to 0xC10000FF. |
| | +macOS 12 and 13 guests boot without changes. macOS 14 and later need the |
| | +two below, which the machine makes when iBoot enters the kernel. |
| | + |
| | +A vmapple guest kernel asks the hypervisor to manage its pointer |
| | +authentication keys through paravirtualized hypercalls: hvc #0 with |
| | +SMCCC function IDs 0xC1000000 to 0xC10000FF. |
| | Hypervisor.framework answers these calls itself, without an exit to QEMU, |
| | and for a VM created through its public API it answers each of them as not |
| | supported. The macOS 12 and 13 kernels carry on with the keys the firmware |
| | set up, which all processes then share. |
| | From macOS 14 on, the kernel checks the result of every one of these calls |
| | -and loops forever when one fails. Such a guest stops right after iBoot hands |
| | -over to the kernel. The serial console ends with iBoot's End of | | | -iBootStage2 serial output. line, and the boot CPU keeps a host core busy |
| | -without exiting to QEMU. Hypervisor.framework implements the calls only for |
| | -VMs created with a private ISA setting, which needs an entitlement that only |
| | -Apple-signed binaries can hold. QEMU does not use it. |
| | +and loops forever when one fails, with hvc #0; cbnz x0, .. |
| | +Hypervisor.framework implements the calls only for VMs created with a |
| | +private ISA setting, which needs an entitlement that only Apple-signed |
| | +binaries can hold. QEMU does not use it. Instead, when iBoot enters the |
| | +kernel, the machine replaces each of those loops with a nop in the memory |
| | +iBoot loaded the kernel into, so the kernel carries on as macOS 13's does. |
| | +This takes the per-process keys away from the guest's pointer |
| | +authentication. Without it, such a guest stops right after iBoot hands over |
| | +to the kernel: the serial console ends with iBoot's End of iBootStage2 | | | +serial output. line, and the boot CPU keeps a host core busy without |
| | +exiting to QEMU. |
| | + |
| | +The machine has no Secure Enclave. From macOS 15 on, the guest's APFS then |
| | +panics on its unencrypted data volume ("unencrypted data volume is not |
| | +allowed") unless the device tree's /filesystems node has a |
| | +DoNotUseSEP property, and the guest resets and boots again, over and |
| | +over. When iBoot enters the kernel, the machine also adds such a node to the |
| | +device tree iBoot hands the kernel, if the tree has no /filesystems |
| | +node. macOS 13's tree has one, which is left as it is. |
| | + |
| | +To catch iBoot entering the kernel, guest RAM is not executable from reset. |
| | +Each page iBoot runs becomes executable on its first instruction fetch, until |
| | +a fetch lands between the boot arguments' physBase and |
| | +topOfKernelData with x0 at those boot arguments, as iBoot enters the |
| | +kernel. All of RAM is executable from then on. |
| | Display |
| | ------- |
| | diff --git a/hw/vmapple/vmapple.c b/hw/vmapple/vmapple.c |
| | index c0306bcf52..da11b91132 100644 |
| | --- a/hw/vmapple/vmapple.c |
| | +++ b/hw/vmapple/vmapple.c |
| | @@ -16,6 +16,8 @@ |
| | #include "qemu/osdep.h" |
| | #include "qemu/bitops.h" |
| | +#include "qemu/cacheflush.h" |
| | +#include "qemu/cutils.h" |
| | #include "qemu/datadir.h" |
| | #include "qemu/error-report.h" |
| | #include "qemu/guest-random.h" |
| | @@ -47,11 +49,13 @@ |
| | #include "qobject/qlist.h" |
| | #include "standard-headers/linux/input.h" |
| | #include "system/hvf.h" |
| | +#include "system/hvf_int.h" |
| | #include "system/reset.h" |
| | #include "system/runstate.h" |
| | #include "system/system.h" |
| | #include "target/arm/gtimer.h" |
| | #include "target/arm/cpu.h" |
| | +#include "target/arm/hvf_arm.h" |
| | struct VMAppleMachineState { |
| | MachineState parent; |
| | @@ -457,12 +461,157 @@ static void create_pcie(VMAppleMachineState vms) |
| | } |
| | } |
| | +/ |
| | + * macOS 14 and later need two changes to the kernel's memory to boot here, |
| | + * made when iBoot enters the kernel. From reset, RAM is not executable: each |
| | + * page iBoot runs becomes executable on its first instruction fetch, until a |
| | + * fetch lands in the kernel's memory with x0 at the boot arguments. Only the |
| | + * boot CPU runs then. The changes leave macOS 13 as it was. |
| | + / |
| | + |
| | +/ A host pointer to [pa, pa + len), if guest RAM holds all of it. */ |
| | +static uint8_t *vmapple_ram(VMAppleMachineState *vms, uint64_t pa, |
| | + uint64_t len) |
| | +{ |
| | + MachineState *ms = MACHINE(vms); |
| | + uint64_t off = pa - vms->memmap[VMAPPLE_MEM].base; |
| | + |
| | + if (pa < vms->memmap[VMAPPLE_MEM].base \|\| off > ms->ram_size \|\| |
| | + len > ms->ram_size - off) { |
| | + return NULL; | | | + } | | | + return (uint8_t )memory_region_get_ram_ptr(ms->ram) + off; | | | +} | | | + | | | +/ | | | + * Hypervisor.framework fails the kernel's pointer authentication hypercalls, | | | + * and from macOS 14 on the kernel spins on each failure with | | | + * "hvc #0; cbnz x0, .". Take out the spins: the kernel then runs on the | | | + * firmware's keys, as macOS 13's does. | | | + */ | | | +static void vmapple_skip_pac_waits(uint8_t *mem, uint64_t size) | | | +{ |
| | + for (uint64_t off = 4; off + 4 <= size; off += 4) { |
| | + if (ldl_le_p(mem + off) == 0xb5000000 && /* cbnz x0, . */ |
| | + ldl_le_p(mem + off - 4) == 0xd4000002) { /* hvc #0 */ |
| | + stl_le_p(mem + off, 0xd503201f); /* nop */ |
| | + flush_idcache_range((uintptr_t)mem + off, (uintptr_t)mem + off, |
| | + 4); |
| | + } | | | + } | | | +} | | | + | | | +/* | | | + * A device tree node: a property count, a child count, its properties, then | | | + * its children. A property is a 32-byte name, a 32-bit length whose top bit | | | + * is a flag, and its value, padded to 4 bytes. | | | + */ | | | +static const struct QEMU_PACKED { | | | + uint32_t props, children; | | | + struct QEMU_PACKED { | | | + char name[32]; | | | + uint32_t len; |
| | + char value[12]; |
| | + } name; |
| | + struct QEMU_PACKED { | | | + char name[32]; | | | + uint32_t len, value; |
| | + } no_sep; |
| | +} vmapple_filesystems = { |
| | + const_le32(2), 0, |
| | + { "name", const_le32(12), "filesystems" }, |
| | + { "DoNotUseSEP", const_le32(4), const_le32(1) }, |
| | +}; |
| | + | | | +/* | | | + * The machine has no Secure Enclave, and from macOS 15 on APFS panics on the | | | + * unencrypted data volume unless the device tree has /filesystems/DoNotUseSEP. | | | + * If the tree has no /filesystems, append that node as the root's last child, | | | + * in the zeroes that pad the tree to its last 16 KiB page, and cover it with | | | + * the tree's length in the boot arguments. | | | + */ | | | +static void vmapple_no_sep(VMAppleMachineState *vms, uint8_t args) | | | +{ | | | + / deviceTreeP - virtBase + physBase, and deviceTreeLength */ |
| | + uint64_t pa = ldq_le_p(args + 0x60) - ldq_le_p(args + 8) + |
| | + ldq_le_p(args + 16); |
| | + uint64_t len = ldl_le_p(args + 0x68), end = 0, nodes, props; |
| | + uint64_t room = ROUND_UP(pa + len, 16 * KiB) - pa; |
| | + uint64_t size = sizeof(vmapple_filesystems); |
| | + uint8_t *dt = vmapple_ram(vms, pa, room); |
| | + | | | + if (!dt) { | | | + return; | | | + } | | | + /* The tree's end: its nodes come in order, each before its children */ |
| | + for (nodes = 1; nodes; nodes--) { |
| | + if (end + 8 > len) { |
| | + return; | | | + } |
| | + props = ldl_le_p(dt + end); |
| | + nodes += ldl_le_p(dt + end + 4); |
| | + for (end += 8; props; props--) { |
| | + if (end + 36 > len) { |
| | + return; | | | + } | | | + end += 36 + ROUND_UP(ldl_le_p(dt + end + 32) & INT32_MAX, 4); | | | + } | | | + } | | | + if (end + size > room || !buffer_is_zero(dt + end, size) || | | | + memmem(dt, end, &vmapple_filesystems.name, | | | + sizeof(vmapple_filesystems.name))) { | | | + return; | | | + } |
| | + memcpy(dt + end, &vmapple_filesystems, size); |
| | + stl_le_p(dt + 4, ldl_le_p(dt + 4) + 1); |
| | + stl_le_p(args + 0x68, MAX(len, end + size)); |
| | +} | | | + | | | +static bool vmapple_exec_fault(CPUState *cpu, hwaddr ipa) | | | +{ |
| | + VMAppleMachineState *vms = VMAPPLE_MACHINE(qdev_get_machine()); |
| | + uint64_t x0, base = 0, top = 0; |
| | + uint8_t *args, kernel; | | | + | | | + if (!vmapple_ram(vms, ipa, 1)) { | | | + return false; | | | + } | | | + / XNU's boot_args, version 2: physBase and topOfKernelData */ |
| | + assert_hvf_ok(hv_vcpu_get_reg(cpu->accel->fd, HV_REG_X0, &x0)); |
| | + args = vmapple_ram(vms, x0, 0x70); |
| | + if (args && lduw_le_p(args + 2) == 2) { |
| | + base = ldq_le_p(args + 16); |
| | + top = ldq_le_p(args + 32); |
| | + } |
| | + kernel = vmapple_ram(vms, base, top - base); |
| | + if (ipa < base \|\| ipa >= top \|\| !kernel) { |
| | + /* Still in iBoot */ |
| | + assert_hvf_ok(hv_vm_protect(ipa & qemu_real_host_page_mask(), |
| | + qemu_real_host_page_size(), |
| | + HV_MEMORY_READ | HV_MEMORY_WRITE | | | | + HV_MEMORY_EXEC)); | | | + return true; | | | + } |
| | + vmapple_no_sep(vms, args); |
| | + vmapple_skip_pac_waits(kernel, top - base); |
| | + hvf_arm_exec_fault = NULL; |
| | + assert_hvf_ok(hv_vm_protect(vms->memmap[VMAPPLE_MEM].base, |
| | + MACHINE(vms)->ram_size, |
| | + HV_MEMORY_READ | HV_MEMORY_WRITE | | | | + HV_MEMORY_EXEC)); | | | + return true; | | | +} | | | + | | | static void vmapple_reset(void *opaque) | | | { | | | VMAppleMachineState *vms = opaque; |
| | hwaddr base = vms->memmap[VMAPPLE_FIRMWARE].base; |
| | cpu_set_pc(first_cpu, base); |
| | + hvf_arm_exec_fault = vmapple_exec_fault; |
| | + assert_hvf_ok(hv_vm_protect(vms->memmap[VMAPPLE_MEM].base, |
| | + MACHINE(vms)->ram_size, |
| | + HV_MEMORY_READ \| HV_MEMORY_WRITE)); |
| | } | | | static void mach_vmapple_init(MachineState *machine) | | | diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c | | | index 17064abba8..febeba2a80 100644 | | | --- a/target/arm/hvf/hvf.c | | | +++ b/target/arm/hvf/hvf.c | | | @@ -332,6 +332,8 @@ typedef struct ARMHostCPUFeatures { | | | static ARMHostCPUFeatures arm_host_cpu_features; | | | +bool (*hvf_arm_exec_fault)(CPUState *cpu, hwaddr ipa); | | | + | | | struct hvf_reg_match { | | | int reg; | | | uint64_t offset; | | | @@ -2797,6 +2799,10 @@ static int hvf_handle_exception(CPUState *cpu, hv_vcpu_exit_exception_t excp) | | | trace_hvf_insn_abort(env->pc, set, fnv, ea, s1ptw, ifsc); | | | + if (hvf_arm_exec_fault && | | | + hvf_arm_exec_fault(cpu, excp->physical_address)) { | | | + break; | | | + } | | | / fall through */ | | | } | | | default: | | | diff --git a/target/arm/hvf_arm.h b/target/arm/hvf_arm.h | | | index 59e19f6e84..643455570a 100644 | | | --- a/target/arm/hvf_arm.h | | | +++ b/target/arm/hvf_arm.h | | | @@ -23,6 +23,12 @@ void hvf_arm_init_debug(void); | | | void hvf_arm_set_cpu_features_from_host(ARMCPU cpu); | | | +/ | | | + * If set, called on a stage 2 fault on an instruction fetch, with the fault's | | | + * IPA. Returns true if it handled the fault. | | | + */ | | | +extern bool (*hvf_arm_exec_fault)(CPUState cpu, hwaddr ipa); | | | + | | | / | | | * We need access to types from macOS SDK >=15.2, so expose stubs if the | | | * headers are not available until we raise our minimum macOS version. |