1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2021 Google LLC 4 * Author: Fuad Tabba <tabba@google.com> 5 */ 6 7#include <linux/kvm_host.h> 8#include <linux/mm.h> 9#include <nvhe/fixed_config.h> 10#include <nvhe/mem_protect.h> 11#include <nvhe/memory.h> 12#include <nvhe/pkvm.h> 13#include <nvhe/trap_handler.h> 14 15/* Used by icache_is_vpipt(). */ 16unsigned long __icache_flags; 17 18/* Used by kvm_get_vttbr(). */ 19unsigned int kvm_arm_vmid_bits; 20 21/* 22 * Set trap register values based on features in ID_AA64PFR0. 23 */ 24static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) 25{ 26 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR0_EL1); 27 u64 hcr_set = HCR_RW; 28 u64 hcr_clear = 0; 29 u64 cptr_set = 0; 30 u64 cptr_clear = 0; 31 32 /* Protected KVM does not support AArch32 guests. */ 33 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), 34 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY); 35 BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), 36 PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY); 37 38 /* 39 * Linux guests assume support for floating-point and Advanced SIMD. Do 40 * not change the trapping behavior for these from the KVM default. 41 */ 42 BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP), 43 PVM_ID_AA64PFR0_ALLOW)); 44 BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD), 45 PVM_ID_AA64PFR0_ALLOW)); 46 47 if (has_hvhe()) 48 hcr_set |= HCR_E2H; 49 50 /* Trap RAS unless all current versions are supported */ 51 if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) < 52 ID_AA64PFR0_EL1_RAS_V1P1) { 53 hcr_set |= HCR_TERR | HCR_TEA; 54 hcr_clear |= HCR_FIEN; 55 } 56 57 /* Trap AMU */ 58 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) { 59 hcr_clear |= HCR_AMVOFFEN; 60 cptr_set |= CPTR_EL2_TAM; 61 } 62 63 /* Trap SVE */ 64 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) { 65 if (has_hvhe()) 66 cptr_clear |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; 67 else 68 cptr_set |= CPTR_EL2_TZ; 69 } 70 71 vcpu->arch.hcr_el2 |= hcr_set; 72 vcpu->arch.hcr_el2 &= ~hcr_clear; 73 vcpu->arch.cptr_el2 |= cptr_set; 74 vcpu->arch.cptr_el2 &= ~cptr_clear; 75} 76 77/* 78 * Set trap register values based on features in ID_AA64PFR1. 79 */ 80static void pvm_init_traps_aa64pfr1(struct kvm_vcpu *vcpu) 81{ 82 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64PFR1_EL1); 83 u64 hcr_set = 0; 84 u64 hcr_clear = 0; 85 86 /* Memory Tagging: Trap and Treat as Untagged if not supported. */ 87 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE), feature_ids)) { 88 hcr_set |= HCR_TID5; 89 hcr_clear |= HCR_DCT | HCR_ATA; 90 } 91 92 vcpu->arch.hcr_el2 |= hcr_set; 93 vcpu->arch.hcr_el2 &= ~hcr_clear; 94} 95 96/* 97 * Set trap register values based on features in ID_AA64DFR0. 98 */ 99static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) 100{ 101 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64DFR0_EL1); 102 u64 mdcr_set = 0; 103 u64 mdcr_clear = 0; 104 u64 cptr_set = 0; 105 106 /* Trap/constrain PMU */ 107 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) { 108 mdcr_set |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; 109 mdcr_clear |= MDCR_EL2_HPME | MDCR_EL2_MTPME | 110 MDCR_EL2_HPMN_MASK; 111 } 112 113 /* Trap Debug */ 114 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), feature_ids)) 115 mdcr_set |= MDCR_EL2_TDRA | MDCR_EL2_TDA | MDCR_EL2_TDE; 116 117 /* Trap OS Double Lock */ 118 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DoubleLock), feature_ids)) 119 mdcr_set |= MDCR_EL2_TDOSA; 120 121 /* Trap SPE */ 122 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer), feature_ids)) { 123 mdcr_set |= MDCR_EL2_TPMS; 124 mdcr_clear |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; 125 } 126 127 /* Trap Trace Filter */ 128 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids)) 129 mdcr_set |= MDCR_EL2_TTRF; 130 131 /* Trap Trace */ 132 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) { 133 if (has_hvhe()) 134 cptr_set |= CPACR_EL1_TTA; 135 else 136 cptr_set |= CPTR_EL2_TTA; 137 } 138 139 vcpu->arch.mdcr_el2 |= mdcr_set; 140 vcpu->arch.mdcr_el2 &= ~mdcr_clear; 141 vcpu->arch.cptr_el2 |= cptr_set; 142} 143 144/* 145 * Set trap register values based on features in ID_AA64MMFR0. 146 */ 147static void pvm_init_traps_aa64mmfr0(struct kvm_vcpu *vcpu) 148{ 149 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR0_EL1); 150 u64 mdcr_set = 0; 151 152 /* Trap Debug Communications Channel registers */ 153 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_FGT), feature_ids)) 154 mdcr_set |= MDCR_EL2_TDCC; 155 156 vcpu->arch.mdcr_el2 |= mdcr_set; 157} 158 159/* 160 * Set trap register values based on features in ID_AA64MMFR1. 161 */ 162static void pvm_init_traps_aa64mmfr1(struct kvm_vcpu *vcpu) 163{ 164 const u64 feature_ids = pvm_read_id_reg(vcpu, SYS_ID_AA64MMFR1_EL1); 165 u64 hcr_set = 0; 166 167 /* Trap LOR */ 168 if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), feature_ids)) 169 hcr_set |= HCR_TLOR; 170 171 vcpu->arch.hcr_el2 |= hcr_set; 172} 173 174/* 175 * Set baseline trap register values. 176 */ 177static void pvm_init_trap_regs(struct kvm_vcpu *vcpu) 178{ 179 const u64 hcr_trap_feat_regs = HCR_TID3; 180 const u64 hcr_trap_impdef = HCR_TACR | HCR_TIDCP | HCR_TID1; 181 182 /* 183 * Always trap: 184 * - Feature id registers: to control features exposed to guests 185 * - Implementation-defined features 186 */ 187 vcpu->arch.hcr_el2 |= hcr_trap_feat_regs | hcr_trap_impdef; 188 189 /* Clear res0 and set res1 bits to trap potential new features. */ 190 vcpu->arch.hcr_el2 &= ~(HCR_RES0); 191 vcpu->arch.mdcr_el2 &= ~(MDCR_EL2_RES0); 192 if (!has_hvhe()) { 193 vcpu->arch.cptr_el2 |= CPTR_NVHE_EL2_RES1; 194 vcpu->arch.cptr_el2 &= ~(CPTR_NVHE_EL2_RES0); 195 } 196} 197 198/* 199 * Initialize trap register values for protected VMs. 200 */ 201void __pkvm_vcpu_init_traps(struct kvm_vcpu *vcpu) 202{ 203 pvm_init_trap_regs(vcpu); 204 pvm_init_traps_aa64pfr0(vcpu); 205 pvm_init_traps_aa64pfr1(vcpu); 206 pvm_init_traps_aa64dfr0(vcpu); 207 pvm_init_traps_aa64mmfr0(vcpu); 208 pvm_init_traps_aa64mmfr1(vcpu); 209} 210 211/* 212 * Start the VM table handle at the offset defined instead of at 0. 213 * Mainly for sanity checking and debugging. 214 */ 215#define HANDLE_OFFSET 0x1000 216 217static unsigned int vm_handle_to_idx(pkvm_handle_t handle) 218{ 219 return handle - HANDLE_OFFSET; 220} 221 222static pkvm_handle_t idx_to_vm_handle(unsigned int idx) 223{ 224 return idx + HANDLE_OFFSET; 225} 226 227/* 228 * Spinlock for protecting state related to the VM table. Protects writes 229 * to 'vm_table' and 'nr_table_entries' as well as reads and writes to 230 * 'last_hyp_vcpu_lookup'. 231 */ 232static DEFINE_HYP_SPINLOCK(vm_table_lock); 233 234/* 235 * The table of VM entries for protected VMs in hyp. 236 * Allocated at hyp initialization and setup. 237 */ 238static struct pkvm_hyp_vm **vm_table; 239 240void pkvm_hyp_vm_table_init(void *tbl) 241{ 242 WARN_ON(vm_table); 243 vm_table = tbl; 244} 245 246/* 247 * Return the hyp vm structure corresponding to the handle. 248 */ 249static struct pkvm_hyp_vm *get_vm_by_handle(pkvm_handle_t handle) 250{ 251 unsigned int idx = vm_handle_to_idx(handle); 252 253 if (unlikely(idx >= KVM_MAX_PVMS)) 254 return NULL; 255 256 return vm_table[idx]; 257} 258 259struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, 260 unsigned int vcpu_idx) 261{ 262 struct pkvm_hyp_vcpu *hyp_vcpu = NULL; 263 struct pkvm_hyp_vm *hyp_vm; 264 265 hyp_spin_lock(&vm_table_lock); 266 hyp_vm = get_vm_by_handle(handle); 267 if (!hyp_vm || hyp_vm->nr_vcpus <= vcpu_idx) 268 goto unlock; 269 270 hyp_vcpu = hyp_vm->vcpus[vcpu_idx]; 271 hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); 272unlock: 273 hyp_spin_unlock(&vm_table_lock); 274 return hyp_vcpu; 275} 276 277void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) 278{ 279 struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); 280 281 hyp_spin_lock(&vm_table_lock); 282 hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); 283 hyp_spin_unlock(&vm_table_lock); 284} 285 286static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) 287{ 288 if (host_vcpu) 289 hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); 290} 291 292static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], 293 unsigned int nr_vcpus) 294{ 295 int i; 296 297 for (i = 0; i < nr_vcpus; i++) 298 unpin_host_vcpu(hyp_vcpus[i]->host_vcpu); 299} 300 301static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, 302 unsigned int nr_vcpus) 303{ 304 hyp_vm->host_kvm = host_kvm; 305 hyp_vm->kvm.created_vcpus = nr_vcpus; 306 hyp_vm->kvm.arch.vtcr = host_mmu.arch.vtcr; 307} 308 309static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, 310 struct pkvm_hyp_vm *hyp_vm, 311 struct kvm_vcpu *host_vcpu, 312 unsigned int vcpu_idx) 313{ 314 int ret = 0; 315 316 if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) 317 return -EBUSY; 318 319 if (host_vcpu->vcpu_idx != vcpu_idx) { 320 ret = -EINVAL; 321 goto done; 322 } 323 324 hyp_vcpu->host_vcpu = host_vcpu; 325 326 hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; 327 hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id); 328 hyp_vcpu->vcpu.vcpu_idx = vcpu_idx; 329 330 hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; 331 hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); 332done: 333 if (ret) 334 unpin_host_vcpu(host_vcpu); 335 return ret; 336} 337 338static int find_free_vm_table_entry(struct kvm *host_kvm) 339{ 340 int i; 341 342 for (i = 0; i < KVM_MAX_PVMS; ++i) { 343 if (!vm_table[i]) 344 return i; 345 } 346 347 return -ENOMEM; 348} 349 350/* 351 * Allocate a VM table entry and insert a pointer to the new vm. 352 * 353 * Return a unique handle to the protected VM on success, 354 * negative error code on failure. 355 */ 356static pkvm_handle_t insert_vm_table_entry(struct kvm *host_kvm, 357 struct pkvm_hyp_vm *hyp_vm) 358{ 359 struct kvm_s2_mmu *mmu = &hyp_vm->kvm.arch.mmu; 360 int idx; 361 362 hyp_assert_lock_held(&vm_table_lock); 363 364 /* 365 * Initializing protected state might have failed, yet a malicious 366 * host could trigger this function. Thus, ensure that 'vm_table' 367 * exists. 368 */ 369 if (unlikely(!vm_table)) 370 return -EINVAL; 371 372 idx = find_free_vm_table_entry(host_kvm); 373 if (idx < 0) 374 return idx; 375 376 hyp_vm->kvm.arch.pkvm.handle = idx_to_vm_handle(idx); 377 378 /* VMID 0 is reserved for the host */ 379 atomic64_set(&mmu->vmid.id, idx + 1); 380 381 mmu->arch = &hyp_vm->kvm.arch; 382 mmu->pgt = &hyp_vm->pgt; 383 384 vm_table[idx] = hyp_vm; 385 return hyp_vm->kvm.arch.pkvm.handle; 386} 387 388/* 389 * Deallocate and remove the VM table entry corresponding to the handle. 390 */ 391static void remove_vm_table_entry(pkvm_handle_t handle) 392{ 393 hyp_assert_lock_held(&vm_table_lock); 394 vm_table[vm_handle_to_idx(handle)] = NULL; 395} 396 397static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) 398{ 399 return size_add(sizeof(struct pkvm_hyp_vm), 400 size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); 401} 402 403static void *map_donated_memory_noclear(unsigned long host_va, size_t size) 404{ 405 void *va = (void *)kern_hyp_va(host_va); 406 407 if (!PAGE_ALIGNED(va)) 408 return NULL; 409 410 if (__pkvm_host_donate_hyp(hyp_virt_to_pfn(va), 411 PAGE_ALIGN(size) >> PAGE_SHIFT)) 412 return NULL; 413 414 return va; 415} 416 417static void *map_donated_memory(unsigned long host_va, size_t size) 418{ 419 void *va = map_donated_memory_noclear(host_va, size); 420 421 if (va) 422 memset(va, 0, size); 423 424 return va; 425} 426 427static void __unmap_donated_memory(void *va, size_t size) 428{ 429 WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(va), 430 PAGE_ALIGN(size) >> PAGE_SHIFT)); 431} 432 433static void unmap_donated_memory(void *va, size_t size) 434{ 435 if (!va) 436 return; 437 438 memset(va, 0, size); 439 __unmap_donated_memory(va, size); 440} 441 442static void unmap_donated_memory_noclear(void *va, size_t size) 443{ 444 if (!va) 445 return; 446 447 __unmap_donated_memory(va, size); 448} 449 450/* 451 * Initialize the hypervisor copy of the protected VM state using the 452 * memory donated by the host. 453 * 454 * Unmaps the donated memory from the host at stage 2. 455 * 456 * host_kvm: A pointer to the host's struct kvm. 457 * vm_hva: The host va of the area being donated for the VM state. 458 * Must be page aligned. 459 * pgd_hva: The host va of the area being donated for the stage-2 PGD for 460 * the VM. Must be page aligned. Its size is implied by the VM's 461 * VTCR. 462 * 463 * Return a unique handle to the protected VM on success, 464 * negative error code on failure. 465 */ 466int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, 467 unsigned long pgd_hva) 468{ 469 struct pkvm_hyp_vm *hyp_vm = NULL; 470 size_t vm_size, pgd_size; 471 unsigned int nr_vcpus; 472 void *pgd = NULL; 473 int ret; 474 475 ret = hyp_pin_shared_mem(host_kvm, host_kvm + 1); 476 if (ret) 477 return ret; 478 479 nr_vcpus = READ_ONCE(host_kvm->created_vcpus); 480 if (nr_vcpus < 1) { 481 ret = -EINVAL; 482 goto err_unpin_kvm; 483 } 484 485 vm_size = pkvm_get_hyp_vm_size(nr_vcpus); 486 pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr); 487 488 ret = -ENOMEM; 489 490 hyp_vm = map_donated_memory(vm_hva, vm_size); 491 if (!hyp_vm) 492 goto err_remove_mappings; 493 494 pgd = map_donated_memory_noclear(pgd_hva, pgd_size); 495 if (!pgd) 496 goto err_remove_mappings; 497 498 init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus); 499 500 hyp_spin_lock(&vm_table_lock); 501 ret = insert_vm_table_entry(host_kvm, hyp_vm); 502 if (ret < 0) 503 goto err_unlock; 504 505 ret = kvm_guest_prepare_stage2(hyp_vm, pgd); 506 if (ret) 507 goto err_remove_vm_table_entry; 508 hyp_spin_unlock(&vm_table_lock); 509 510 return hyp_vm->kvm.arch.pkvm.handle; 511 512err_remove_vm_table_entry: 513 remove_vm_table_entry(hyp_vm->kvm.arch.pkvm.handle); 514err_unlock: 515 hyp_spin_unlock(&vm_table_lock); 516err_remove_mappings: 517 unmap_donated_memory(hyp_vm, vm_size); 518 unmap_donated_memory(pgd, pgd_size); 519err_unpin_kvm: 520 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 521 return ret; 522} 523 524/* 525 * Initialize the hypervisor copy of the protected vCPU state using the 526 * memory donated by the host. 527 * 528 * handle: The handle for the protected vm. 529 * host_vcpu: A pointer to the corresponding host vcpu. 530 * vcpu_hva: The host va of the area being donated for the vcpu state. 531 * Must be page aligned. The size of the area must be equal to 532 * the page-aligned size of 'struct pkvm_hyp_vcpu'. 533 * Return 0 on success, negative error code on failure. 534 */ 535int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, 536 unsigned long vcpu_hva) 537{ 538 struct pkvm_hyp_vcpu *hyp_vcpu; 539 struct pkvm_hyp_vm *hyp_vm; 540 unsigned int idx; 541 int ret; 542 543 hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu)); 544 if (!hyp_vcpu) 545 return -ENOMEM; 546 547 hyp_spin_lock(&vm_table_lock); 548 549 hyp_vm = get_vm_by_handle(handle); 550 if (!hyp_vm) { 551 ret = -ENOENT; 552 goto unlock; 553 } 554 555 idx = hyp_vm->nr_vcpus; 556 if (idx >= hyp_vm->kvm.created_vcpus) { 557 ret = -EINVAL; 558 goto unlock; 559 } 560 561 ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx); 562 if (ret) 563 goto unlock; 564 565 hyp_vm->vcpus[idx] = hyp_vcpu; 566 hyp_vm->nr_vcpus++; 567unlock: 568 hyp_spin_unlock(&vm_table_lock); 569 570 if (ret) 571 unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu)); 572 573 return ret; 574} 575 576static void 577teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size) 578{ 579 size = PAGE_ALIGN(size); 580 memset(addr, 0, size); 581 582 for (void *start = addr; start < addr + size; start += PAGE_SIZE) 583 push_hyp_memcache(mc, start, hyp_virt_to_phys); 584 585 unmap_donated_memory_noclear(addr, size); 586} 587 588int __pkvm_teardown_vm(pkvm_handle_t handle) 589{ 590 struct kvm_hyp_memcache *mc; 591 struct pkvm_hyp_vm *hyp_vm; 592 struct kvm *host_kvm; 593 unsigned int idx; 594 size_t vm_size; 595 int err; 596 597 hyp_spin_lock(&vm_table_lock); 598 hyp_vm = get_vm_by_handle(handle); 599 if (!hyp_vm) { 600 err = -ENOENT; 601 goto err_unlock; 602 } 603 604 if (WARN_ON(hyp_page_count(hyp_vm))) { 605 err = -EBUSY; 606 goto err_unlock; 607 } 608 609 host_kvm = hyp_vm->host_kvm; 610 611 /* Ensure the VMID is clean before it can be reallocated */ 612 __kvm_tlb_flush_vmid(&hyp_vm->kvm.arch.mmu); 613 remove_vm_table_entry(handle); 614 hyp_spin_unlock(&vm_table_lock); 615 616 /* Reclaim guest pages (including page-table pages) */ 617 mc = &host_kvm->arch.pkvm.teardown_mc; 618 reclaim_guest_pages(hyp_vm, mc); 619 unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus); 620 621 /* Push the metadata pages to the teardown memcache */ 622 for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) { 623 struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; 624 625 teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); 626 } 627 628 vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus); 629 teardown_donated_memory(mc, hyp_vm, vm_size); 630 hyp_unpin_shared_mem(host_kvm, host_kvm + 1); 631 return 0; 632 633err_unlock: 634 hyp_spin_unlock(&vm_table_lock); 635 return err; 636} 637