1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM-SEV support 6 * 7 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 8 */ 9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11#include <linux/kvm_types.h> 12#include <linux/kvm_host.h> 13#include <linux/kernel.h> 14#include <linux/highmem.h> 15#include <linux/psp.h> 16#include <linux/psp-sev.h> 17#include <linux/pagemap.h> 18#include <linux/swap.h> 19#include <linux/misc_cgroup.h> 20#include <linux/processor.h> 21#include <linux/trace_events.h> 22 23#include <asm/pkru.h> 24#include <asm/trapnr.h> 25#include <asm/fpu/xcr.h> 26#include <asm/debugreg.h> 27 28#include "mmu.h" 29#include "x86.h" 30#include "svm.h" 31#include "svm_ops.h" 32#include "cpuid.h" 33#include "trace.h" 34 35#ifndef CONFIG_KVM_AMD_SEV 36/* 37 * When this config is not defined, SEV feature is not supported and APIs in 38 * this file are not used but this file still gets compiled into the KVM AMD 39 * module. 40 * 41 * We will not have MISC_CG_RES_SEV and MISC_CG_RES_SEV_ES entries in the enum 42 * misc_res_type {} defined in linux/misc_cgroup.h. 43 * 44 * Below macros allow compilation to succeed. 45 */ 46#define MISC_CG_RES_SEV MISC_CG_RES_TYPES 47#define MISC_CG_RES_SEV_ES MISC_CG_RES_TYPES 48#endif 49 50#ifdef CONFIG_KVM_AMD_SEV 51/* enable/disable SEV support */ 52static bool sev_enabled = true; 53module_param_named(sev, sev_enabled, bool, 0444); 54 55/* enable/disable SEV-ES support */ 56static bool sev_es_enabled = true; 57module_param_named(sev_es, sev_es_enabled, bool, 0444); 58 59/* enable/disable SEV-ES DebugSwap support */ 60static bool sev_es_debug_swap_enabled = true; 61module_param_named(debug_swap, sev_es_debug_swap_enabled, bool, 0444); 62#else 63#define sev_enabled false 64#define sev_es_enabled false 65#define sev_es_debug_swap_enabled false 66#endif /* CONFIG_KVM_AMD_SEV */ 67 68static u8 sev_enc_bit; 69static DECLARE_RWSEM(sev_deactivate_lock); 70static DEFINE_MUTEX(sev_bitmap_lock); 71unsigned int max_sev_asid; 72static unsigned int min_sev_asid; 73static unsigned long sev_me_mask; 74static unsigned int nr_asids; 75static unsigned long *sev_asid_bitmap; 76static unsigned long *sev_reclaim_asid_bitmap; 77 78struct enc_region { 79 struct list_head list; 80 unsigned long npages; 81 struct page **pages; 82 unsigned long uaddr; 83 unsigned long size; 84}; 85 86/* Called with the sev_bitmap_lock held, or on shutdown */ 87static int sev_flush_asids(int min_asid, int max_asid) 88{ 89 int ret, asid, error = 0; 90 91 /* Check if there are any ASIDs to reclaim before performing a flush */ 92 asid = find_next_bit(sev_reclaim_asid_bitmap, nr_asids, min_asid); 93 if (asid > max_asid) 94 return -EBUSY; 95 96 /* 97 * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail, 98 * so it must be guarded. 99 */ 100 down_write(&sev_deactivate_lock); 101 102 wbinvd_on_all_cpus(); 103 ret = sev_guest_df_flush(&error); 104 105 up_write(&sev_deactivate_lock); 106 107 if (ret) 108 pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error); 109 110 return ret; 111} 112 113static inline bool is_mirroring_enc_context(struct kvm *kvm) 114{ 115 return !!to_kvm_svm(kvm)->sev_info.enc_context_owner; 116} 117 118/* Must be called with the sev_bitmap_lock held */ 119static bool __sev_recycle_asids(int min_asid, int max_asid) 120{ 121 if (sev_flush_asids(min_asid, max_asid)) 122 return false; 123 124 /* The flush process will flush all reclaimable SEV and SEV-ES ASIDs */ 125 bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap, 126 nr_asids); 127 bitmap_zero(sev_reclaim_asid_bitmap, nr_asids); 128 129 return true; 130} 131 132static int sev_misc_cg_try_charge(struct kvm_sev_info *sev) 133{ 134 enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; 135 return misc_cg_try_charge(type, sev->misc_cg, 1); 136} 137 138static void sev_misc_cg_uncharge(struct kvm_sev_info *sev) 139{ 140 enum misc_res_type type = sev->es_active ? MISC_CG_RES_SEV_ES : MISC_CG_RES_SEV; 141 misc_cg_uncharge(type, sev->misc_cg, 1); 142} 143 144static int sev_asid_new(struct kvm_sev_info *sev) 145{ 146 int asid, min_asid, max_asid, ret; 147 bool retry = true; 148 149 WARN_ON(sev->misc_cg); 150 sev->misc_cg = get_current_misc_cg(); 151 ret = sev_misc_cg_try_charge(sev); 152 if (ret) { 153 put_misc_cg(sev->misc_cg); 154 sev->misc_cg = NULL; 155 return ret; 156 } 157 158 mutex_lock(&sev_bitmap_lock); 159 160 /* 161 * SEV-enabled guests must use asid from min_sev_asid to max_sev_asid. 162 * SEV-ES-enabled guest can use from 1 to min_sev_asid - 1. 163 */ 164 min_asid = sev->es_active ? 1 : min_sev_asid; 165 max_asid = sev->es_active ? min_sev_asid - 1 : max_sev_asid; 166again: 167 asid = find_next_zero_bit(sev_asid_bitmap, max_asid + 1, min_asid); 168 if (asid > max_asid) { 169 if (retry && __sev_recycle_asids(min_asid, max_asid)) { 170 retry = false; 171 goto again; 172 } 173 mutex_unlock(&sev_bitmap_lock); 174 ret = -EBUSY; 175 goto e_uncharge; 176 } 177 178 __set_bit(asid, sev_asid_bitmap); 179 180 mutex_unlock(&sev_bitmap_lock); 181 182 return asid; 183e_uncharge: 184 sev_misc_cg_uncharge(sev); 185 put_misc_cg(sev->misc_cg); 186 sev->misc_cg = NULL; 187 return ret; 188} 189 190static int sev_get_asid(struct kvm *kvm) 191{ 192 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 193 194 return sev->asid; 195} 196 197static void sev_asid_free(struct kvm_sev_info *sev) 198{ 199 struct svm_cpu_data *sd; 200 int cpu; 201 202 mutex_lock(&sev_bitmap_lock); 203 204 __set_bit(sev->asid, sev_reclaim_asid_bitmap); 205 206 for_each_possible_cpu(cpu) { 207 sd = per_cpu_ptr(&svm_data, cpu); 208 sd->sev_vmcbs[sev->asid] = NULL; 209 } 210 211 mutex_unlock(&sev_bitmap_lock); 212 213 sev_misc_cg_uncharge(sev); 214 put_misc_cg(sev->misc_cg); 215 sev->misc_cg = NULL; 216} 217 218static void sev_decommission(unsigned int handle) 219{ 220 struct sev_data_decommission decommission; 221 222 if (!handle) 223 return; 224 225 decommission.handle = handle; 226 sev_guest_decommission(&decommission, NULL); 227} 228 229static void sev_unbind_asid(struct kvm *kvm, unsigned int handle) 230{ 231 struct sev_data_deactivate deactivate; 232 233 if (!handle) 234 return; 235 236 deactivate.handle = handle; 237 238 /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */ 239 down_read(&sev_deactivate_lock); 240 sev_guest_deactivate(&deactivate, NULL); 241 up_read(&sev_deactivate_lock); 242 243 sev_decommission(handle); 244} 245 246static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp) 247{ 248 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 249 int asid, ret; 250 251 if (kvm->created_vcpus) 252 return -EINVAL; 253 254 ret = -EBUSY; 255 if (unlikely(sev->active)) 256 return ret; 257 258 sev->active = true; 259 sev->es_active = argp->id == KVM_SEV_ES_INIT; 260 asid = sev_asid_new(sev); 261 if (asid < 0) 262 goto e_no_asid; 263 sev->asid = asid; 264 265 ret = sev_platform_init(&argp->error); 266 if (ret) 267 goto e_free; 268 269 INIT_LIST_HEAD(&sev->regions_list); 270 INIT_LIST_HEAD(&sev->mirror_vms); 271 272 kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); 273 274 return 0; 275 276e_free: 277 sev_asid_free(sev); 278 sev->asid = 0; 279e_no_asid: 280 sev->es_active = false; 281 sev->active = false; 282 return ret; 283} 284 285static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error) 286{ 287 struct sev_data_activate activate; 288 int asid = sev_get_asid(kvm); 289 int ret; 290 291 /* activate ASID on the given handle */ 292 activate.handle = handle; 293 activate.asid = asid; 294 ret = sev_guest_activate(&activate, error); 295 296 return ret; 297} 298 299static int __sev_issue_cmd(int fd, int id, void *data, int *error) 300{ 301 struct fd f; 302 int ret; 303 304 f = fdget(fd); 305 if (!f.file) 306 return -EBADF; 307 308 ret = sev_issue_cmd_external_user(f.file, id, data, error); 309 310 fdput(f); 311 return ret; 312} 313 314static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error) 315{ 316 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 317 318 return __sev_issue_cmd(sev->fd, id, data, error); 319} 320 321static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 322{ 323 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 324 struct sev_data_launch_start start; 325 struct kvm_sev_launch_start params; 326 void *dh_blob, *session_blob; 327 int *error = &argp->error; 328 int ret; 329 330 if (!sev_guest(kvm)) 331 return -ENOTTY; 332 333 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) 334 return -EFAULT; 335 336 memset(&start, 0, sizeof(start)); 337 338 dh_blob = NULL; 339 if (params.dh_uaddr) { 340 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len); 341 if (IS_ERR(dh_blob)) 342 return PTR_ERR(dh_blob); 343 344 start.dh_cert_address = __sme_set(__pa(dh_blob)); 345 start.dh_cert_len = params.dh_len; 346 } 347 348 session_blob = NULL; 349 if (params.session_uaddr) { 350 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len); 351 if (IS_ERR(session_blob)) { 352 ret = PTR_ERR(session_blob); 353 goto e_free_dh; 354 } 355 356 start.session_address = __sme_set(__pa(session_blob)); 357 start.session_len = params.session_len; 358 } 359 360 start.handle = params.handle; 361 start.policy = params.policy; 362 363 /* create memory encryption context */ 364 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, &start, error); 365 if (ret) 366 goto e_free_session; 367 368 /* Bind ASID to this guest */ 369 ret = sev_bind_asid(kvm, start.handle, error); 370 if (ret) { 371 sev_decommission(start.handle); 372 goto e_free_session; 373 } 374 375 /* return handle to userspace */ 376 params.handle = start.handle; 377 if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) { 378 sev_unbind_asid(kvm, start.handle); 379 ret = -EFAULT; 380 goto e_free_session; 381 } 382 383 sev->handle = start.handle; 384 sev->fd = argp->sev_fd; 385 386e_free_session: 387 kfree(session_blob); 388e_free_dh: 389 kfree(dh_blob); 390 return ret; 391} 392 393static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, 394 unsigned long ulen, unsigned long *n, 395 int write) 396{ 397 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 398 unsigned long npages, size; 399 int npinned; 400 unsigned long locked, lock_limit; 401 struct page **pages; 402 unsigned long first, last; 403 int ret; 404 405 lockdep_assert_held(&kvm->lock); 406 407 if (ulen == 0 || uaddr + ulen < uaddr) 408 return ERR_PTR(-EINVAL); 409 410 /* Calculate number of pages. */ 411 first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; 412 last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; 413 npages = (last - first + 1); 414 415 locked = sev->pages_locked + npages; 416 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 417 if (locked > lock_limit && !capable(CAP_IPC_LOCK)) { 418 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit); 419 return ERR_PTR(-ENOMEM); 420 } 421 422 if (WARN_ON_ONCE(npages > INT_MAX)) 423 return ERR_PTR(-EINVAL); 424 425 /* Avoid using vmalloc for smaller buffers. */ 426 size = npages * sizeof(struct page *); 427 if (size > PAGE_SIZE) 428 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); 429 else 430 pages = kmalloc(size, GFP_KERNEL_ACCOUNT); 431 432 if (!pages) 433 return ERR_PTR(-ENOMEM); 434 435 /* Pin the user virtual address. */ 436 npinned = pin_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages); 437 if (npinned != npages) { 438 pr_err("SEV: Failure locking %lu pages.\n", npages); 439 ret = -ENOMEM; 440 goto err; 441 } 442 443 *n = npages; 444 sev->pages_locked = locked; 445 446 return pages; 447 448err: 449 if (npinned > 0) 450 unpin_user_pages(pages, npinned); 451 452 kvfree(pages); 453 return ERR_PTR(ret); 454} 455 456static void sev_unpin_memory(struct kvm *kvm, struct page **pages, 457 unsigned long npages) 458{ 459 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 460 461 unpin_user_pages(pages, npages); 462 kvfree(pages); 463 sev->pages_locked -= npages; 464} 465 466static void sev_clflush_pages(struct page *pages[], unsigned long npages) 467{ 468 uint8_t *page_virtual; 469 unsigned long i; 470 471 if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 || 472 pages == NULL) 473 return; 474 475 for (i = 0; i < npages; i++) { 476 page_virtual = kmap_local_page(pages[i]); 477 clflush_cache_range(page_virtual, PAGE_SIZE); 478 kunmap_local(page_virtual); 479 cond_resched(); 480 } 481} 482 483static unsigned long get_num_contig_pages(unsigned long idx, 484 struct page **inpages, unsigned long npages) 485{ 486 unsigned long paddr, next_paddr; 487 unsigned long i = idx + 1, pages = 1; 488 489 /* find the number of contiguous pages starting from idx */ 490 paddr = __sme_page_pa(inpages[idx]); 491 while (i < npages) { 492 next_paddr = __sme_page_pa(inpages[i++]); 493 if ((paddr + PAGE_SIZE) == next_paddr) { 494 pages++; 495 paddr = next_paddr; 496 continue; 497 } 498 break; 499 } 500 501 return pages; 502} 503 504static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) 505{ 506 unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i; 507 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 508 struct kvm_sev_launch_update_data params; 509 struct sev_data_launch_update_data data; 510 struct page **inpages; 511 int ret; 512 513 if (!sev_guest(kvm)) 514 return -ENOTTY; 515 516 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) 517 return -EFAULT; 518 519 vaddr = params.uaddr; 520 size = params.len; 521 vaddr_end = vaddr + size; 522 523 /* Lock the user memory. */ 524 inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1); 525 if (IS_ERR(inpages)) 526 return PTR_ERR(inpages); 527 528 /* 529 * Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in 530 * place; the cache may contain the data that was written unencrypted. 531 */ 532 sev_clflush_pages(inpages, npages); 533 534 data.reserved = 0; 535 data.handle = sev->handle; 536 537 for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) { 538 int offset, len; 539 540 /* 541 * If the user buffer is not page-aligned, calculate the offset 542 * within the page. 543 */ 544 offset = vaddr & (PAGE_SIZE - 1); 545 546 /* Calculate the number of pages that can be encrypted in one go. */ 547 pages = get_num_contig_pages(i, inpages, npages); 548 549 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size); 550 551 data.len = len; 552 data.address = __sme_page_pa(inpages[i]) + offset; 553 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, &data, &argp->error); 554 if (ret) 555 goto e_unpin; 556 557 size -= len; 558 next_vaddr = vaddr + len; 559 } 560 561e_unpin: 562 /* content of memory is updated, mark pages dirty */ 563 for (i = 0; i < npages; i++) { 564 set_page_dirty_lock(inpages[i]); 565 mark_page_accessed(inpages[i]); 566 } 567 /* unlock the user pages */ 568 sev_unpin_memory(kvm, inpages, npages); 569 return ret; 570} 571 572static int sev_es_sync_vmsa(struct vcpu_svm *svm) 573{ 574 struct sev_es_save_area *save = svm->sev_es.vmsa; 575 576 /* Check some debug related fields before encrypting the VMSA */ 577 if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1)) 578 return -EINVAL; 579 580 /* 581 * SEV-ES will use a VMSA that is pointed to by the VMCB, not 582 * the traditional VMSA that is part of the VMCB. Copy the 583 * traditional VMSA as it has been built so far (in prep 584 * for LAUNCH_UPDATE_VMSA) to be the initial SEV-ES state. 585 */ 586 memcpy(save, &svm->vmcb->save, sizeof(svm->vmcb->save)); 587 588 /* Sync registgers */ 589 save->rax = svm->vcpu.arch.regs[VCPU_REGS_RAX]; 590 save->rbx = svm->vcpu.arch.regs[VCPU_REGS_RBX]; 591 save->rcx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 592 save->rdx = svm->vcpu.arch.regs[VCPU_REGS_RDX]; 593 save->rsp = svm->vcpu.arch.regs[VCPU_REGS_RSP]; 594 save->rbp = svm->vcpu.arch.regs[VCPU_REGS_RBP]; 595 save->rsi = svm->vcpu.arch.regs[VCPU_REGS_RSI]; 596 save->rdi = svm->vcpu.arch.regs[VCPU_REGS_RDI]; 597#ifdef CONFIG_X86_64 598 save->r8 = svm->vcpu.arch.regs[VCPU_REGS_R8]; 599 save->r9 = svm->vcpu.arch.regs[VCPU_REGS_R9]; 600 save->r10 = svm->vcpu.arch.regs[VCPU_REGS_R10]; 601 save->r11 = svm->vcpu.arch.regs[VCPU_REGS_R11]; 602 save->r12 = svm->vcpu.arch.regs[VCPU_REGS_R12]; 603 save->r13 = svm->vcpu.arch.regs[VCPU_REGS_R13]; 604 save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14]; 605 save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15]; 606#endif 607 save->rip = svm->vcpu.arch.regs[VCPU_REGS_RIP]; 608 609 /* Sync some non-GPR registers before encrypting */ 610 save->xcr0 = svm->vcpu.arch.xcr0; 611 save->pkru = svm->vcpu.arch.pkru; 612 save->xss = svm->vcpu.arch.ia32_xss; 613 save->dr6 = svm->vcpu.arch.dr6; 614 615 if (sev_es_debug_swap_enabled) 616 save->sev_features |= SVM_SEV_FEAT_DEBUG_SWAP; 617 618 pr_debug("Virtual Machine Save Area (VMSA):\n"); 619 print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false); 620 621 return 0; 622} 623 624static int __sev_launch_update_vmsa(struct kvm *kvm, struct kvm_vcpu *vcpu, 625 int *error) 626{ 627 struct sev_data_launch_update_vmsa vmsa; 628 struct vcpu_svm *svm = to_svm(vcpu); 629 int ret; 630 631 if (vcpu->guest_debug) { 632 pr_warn_once("KVM_SET_GUEST_DEBUG for SEV-ES guest is not supported"); 633 return -EINVAL; 634 } 635 636 /* Perform some pre-encryption checks against the VMSA */ 637 ret = sev_es_sync_vmsa(svm); 638 if (ret) 639 return ret; 640 641 /* 642 * The LAUNCH_UPDATE_VMSA command will perform in-place encryption of 643 * the VMSA memory content (i.e it will write the same memory region 644 * with the guest's key), so invalidate it first. 645 */ 646 clflush_cache_range(svm->sev_es.vmsa, PAGE_SIZE); 647 648 vmsa.reserved = 0; 649 vmsa.handle = to_kvm_svm(kvm)->sev_info.handle; 650 vmsa.address = __sme_pa(svm->sev_es.vmsa); 651 vmsa.len = PAGE_SIZE; 652 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_VMSA, &vmsa, error); 653 if (ret) 654 return ret; 655 656 vcpu->arch.guest_state_protected = true; 657 return 0; 658} 659 660static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) 661{ 662 struct kvm_vcpu *vcpu; 663 unsigned long i; 664 int ret; 665 666 if (!sev_es_guest(kvm)) 667 return -ENOTTY; 668 669 kvm_for_each_vcpu(i, vcpu, kvm) { 670 ret = mutex_lock_killable(&vcpu->mutex); 671 if (ret) 672 return ret; 673 674 ret = __sev_launch_update_vmsa(kvm, vcpu, &argp->error); 675 676 mutex_unlock(&vcpu->mutex); 677 if (ret) 678 return ret; 679 } 680 681 return 0; 682} 683 684static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) 685{ 686 void __user *measure = (void __user *)(uintptr_t)argp->data; 687 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 688 struct sev_data_launch_measure data; 689 struct kvm_sev_launch_measure params; 690 void __user *p = NULL; 691 void *blob = NULL; 692 int ret; 693 694 if (!sev_guest(kvm)) 695 return -ENOTTY; 696 697 if (copy_from_user(¶ms, measure, sizeof(params))) 698 return -EFAULT; 699 700 memset(&data, 0, sizeof(data)); 701 702 /* User wants to query the blob length */ 703 if (!params.len) 704 goto cmd; 705 706 p = (void __user *)(uintptr_t)params.uaddr; 707 if (p) { 708 if (params.len > SEV_FW_BLOB_MAX_SIZE) 709 return -EINVAL; 710 711 blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT); 712 if (!blob) 713 return -ENOMEM; 714 715 data.address = __psp_pa(blob); 716 data.len = params.len; 717 } 718 719cmd: 720 data.handle = sev->handle; 721 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, &data, &argp->error); 722 723 /* 724 * If we query the session length, FW responded with expected data. 725 */ 726 if (!params.len) 727 goto done; 728 729 if (ret) 730 goto e_free_blob; 731 732 if (blob) { 733 if (copy_to_user(p, blob, params.len)) 734 ret = -EFAULT; 735 } 736 737done: 738 params.len = data.len; 739 if (copy_to_user(measure, ¶ms, sizeof(params))) 740 ret = -EFAULT; 741e_free_blob: 742 kfree(blob); 743 return ret; 744} 745 746static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 747{ 748 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 749 struct sev_data_launch_finish data; 750 751 if (!sev_guest(kvm)) 752 return -ENOTTY; 753 754 data.handle = sev->handle; 755 return sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, &data, &argp->error); 756} 757 758static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp) 759{ 760 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 761 struct kvm_sev_guest_status params; 762 struct sev_data_guest_status data; 763 int ret; 764 765 if (!sev_guest(kvm)) 766 return -ENOTTY; 767 768 memset(&data, 0, sizeof(data)); 769 770 data.handle = sev->handle; 771 ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, &data, &argp->error); 772 if (ret) 773 return ret; 774 775 params.policy = data.policy; 776 params.state = data.state; 777 params.handle = data.handle; 778 779 if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) 780 ret = -EFAULT; 781 782 return ret; 783} 784 785static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src, 786 unsigned long dst, int size, 787 int *error, bool enc) 788{ 789 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 790 struct sev_data_dbg data; 791 792 data.reserved = 0; 793 data.handle = sev->handle; 794 data.dst_addr = dst; 795 data.src_addr = src; 796 data.len = size; 797 798 return sev_issue_cmd(kvm, 799 enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT, 800 &data, error); 801} 802 803static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, 804 unsigned long dst_paddr, int sz, int *err) 805{ 806 int offset; 807 808 /* 809 * Its safe to read more than we are asked, caller should ensure that 810 * destination has enough space. 811 */ 812 offset = src_paddr & 15; 813 src_paddr = round_down(src_paddr, 16); 814 sz = round_up(sz + offset, 16); 815 816 return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false); 817} 818 819static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, 820 void __user *dst_uaddr, 821 unsigned long dst_paddr, 822 int size, int *err) 823{ 824 struct page *tpage = NULL; 825 int ret, offset; 826 827 /* if inputs are not 16-byte then use intermediate buffer */ 828 if (!IS_ALIGNED(dst_paddr, 16) || 829 !IS_ALIGNED(paddr, 16) || 830 !IS_ALIGNED(size, 16)) { 831 tpage = (void *)alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 832 if (!tpage) 833 return -ENOMEM; 834 835 dst_paddr = __sme_page_pa(tpage); 836 } 837 838 ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err); 839 if (ret) 840 goto e_free; 841 842 if (tpage) { 843 offset = paddr & 15; 844 if (copy_to_user(dst_uaddr, page_address(tpage) + offset, size)) 845 ret = -EFAULT; 846 } 847 848e_free: 849 if (tpage) 850 __free_page(tpage); 851 852 return ret; 853} 854 855static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr, 856 void __user *vaddr, 857 unsigned long dst_paddr, 858 void __user *dst_vaddr, 859 int size, int *error) 860{ 861 struct page *src_tpage = NULL; 862 struct page *dst_tpage = NULL; 863 int ret, len = size; 864 865 /* If source buffer is not aligned then use an intermediate buffer */ 866 if (!IS_ALIGNED((unsigned long)vaddr, 16)) { 867 src_tpage = alloc_page(GFP_KERNEL_ACCOUNT); 868 if (!src_tpage) 869 return -ENOMEM; 870 871 if (copy_from_user(page_address(src_tpage), vaddr, size)) { 872 __free_page(src_tpage); 873 return -EFAULT; 874 } 875 876 paddr = __sme_page_pa(src_tpage); 877 } 878 879 /* 880 * If destination buffer or length is not aligned then do read-modify-write: 881 * - decrypt destination in an intermediate buffer 882 * - copy the source buffer in an intermediate buffer 883 * - use the intermediate buffer as source buffer 884 */ 885 if (!IS_ALIGNED((unsigned long)dst_vaddr, 16) || !IS_ALIGNED(size, 16)) { 886 int dst_offset; 887 888 dst_tpage = alloc_page(GFP_KERNEL_ACCOUNT); 889 if (!dst_tpage) { 890 ret = -ENOMEM; 891 goto e_free; 892 } 893 894 ret = __sev_dbg_decrypt(kvm, dst_paddr, 895 __sme_page_pa(dst_tpage), size, error); 896 if (ret) 897 goto e_free; 898 899 /* 900 * If source is kernel buffer then use memcpy() otherwise 901 * copy_from_user(). 902 */ 903 dst_offset = dst_paddr & 15; 904 905 if (src_tpage) 906 memcpy(page_address(dst_tpage) + dst_offset, 907 page_address(src_tpage), size); 908 else { 909 if (copy_from_user(page_address(dst_tpage) + dst_offset, 910 vaddr, size)) { 911 ret = -EFAULT; 912 goto e_free; 913 } 914 } 915 916 paddr = __sme_page_pa(dst_tpage); 917 dst_paddr = round_down(dst_paddr, 16); 918 len = round_up(size, 16); 919 } 920 921 ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true); 922 923e_free: 924 if (src_tpage) 925 __free_page(src_tpage); 926 if (dst_tpage) 927 __free_page(dst_tpage); 928 return ret; 929} 930 931static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) 932{ 933 unsigned long vaddr, vaddr_end, next_vaddr; 934 unsigned long dst_vaddr; 935 struct page **src_p, **dst_p; 936 struct kvm_sev_dbg debug; 937 unsigned long n; 938 unsigned int size; 939 int ret; 940 941 if (!sev_guest(kvm)) 942 return -ENOTTY; 943 944 if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug))) 945 return -EFAULT; 946 947 if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr) 948 return -EINVAL; 949 if (!debug.dst_uaddr) 950 return -EINVAL; 951 952 vaddr = debug.src_uaddr; 953 size = debug.len; 954 vaddr_end = vaddr + size; 955 dst_vaddr = debug.dst_uaddr; 956 957 for (; vaddr < vaddr_end; vaddr = next_vaddr) { 958 int len, s_off, d_off; 959 960 /* lock userspace source and destination page */ 961 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0); 962 if (IS_ERR(src_p)) 963 return PTR_ERR(src_p); 964 965 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1); 966 if (IS_ERR(dst_p)) { 967 sev_unpin_memory(kvm, src_p, n); 968 return PTR_ERR(dst_p); 969 } 970 971 /* 972 * Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify 973 * the pages; flush the destination too so that future accesses do not 974 * see stale data. 975 */ 976 sev_clflush_pages(src_p, 1); 977 sev_clflush_pages(dst_p, 1); 978 979 /* 980 * Since user buffer may not be page aligned, calculate the 981 * offset within the page. 982 */ 983 s_off = vaddr & ~PAGE_MASK; 984 d_off = dst_vaddr & ~PAGE_MASK; 985 len = min_t(size_t, (PAGE_SIZE - s_off), size); 986 987 if (dec) 988 ret = __sev_dbg_decrypt_user(kvm, 989 __sme_page_pa(src_p[0]) + s_off, 990 (void __user *)dst_vaddr, 991 __sme_page_pa(dst_p[0]) + d_off, 992 len, &argp->error); 993 else 994 ret = __sev_dbg_encrypt_user(kvm, 995 __sme_page_pa(src_p[0]) + s_off, 996 (void __user *)vaddr, 997 __sme_page_pa(dst_p[0]) + d_off, 998 (void __user *)dst_vaddr, 999 len, &argp->error); 1000 1001 sev_unpin_memory(kvm, src_p, n); 1002 sev_unpin_memory(kvm, dst_p, n); 1003 1004 if (ret) 1005 goto err; 1006 1007 next_vaddr = vaddr + len; 1008 dst_vaddr = dst_vaddr + len; 1009 size -= len; 1010 } 1011err: 1012 return ret; 1013} 1014 1015static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp) 1016{ 1017 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1018 struct sev_data_launch_secret data; 1019 struct kvm_sev_launch_secret params; 1020 struct page **pages; 1021 void *blob, *hdr; 1022 unsigned long n, i; 1023 int ret, offset; 1024 1025 if (!sev_guest(kvm)) 1026 return -ENOTTY; 1027 1028 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) 1029 return -EFAULT; 1030 1031 pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1); 1032 if (IS_ERR(pages)) 1033 return PTR_ERR(pages); 1034 1035 /* 1036 * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in 1037 * place; the cache may contain the data that was written unencrypted. 1038 */ 1039 sev_clflush_pages(pages, n); 1040 1041 /* 1042 * The secret must be copied into contiguous memory region, lets verify 1043 * that userspace memory pages are contiguous before we issue command. 1044 */ 1045 if (get_num_contig_pages(0, pages, n) != n) { 1046 ret = -EINVAL; 1047 goto e_unpin_memory; 1048 } 1049 1050 memset(&data, 0, sizeof(data)); 1051 1052 offset = params.guest_uaddr & (PAGE_SIZE - 1); 1053 data.guest_address = __sme_page_pa(pages[0]) + offset; 1054 data.guest_len = params.guest_len; 1055 1056 blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len); 1057 if (IS_ERR(blob)) { 1058 ret = PTR_ERR(blob); 1059 goto e_unpin_memory; 1060 } 1061 1062 data.trans_address = __psp_pa(blob); 1063 data.trans_len = params.trans_len; 1064 1065 hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); 1066 if (IS_ERR(hdr)) { 1067 ret = PTR_ERR(hdr); 1068 goto e_free_blob; 1069 } 1070 data.hdr_address = __psp_pa(hdr); 1071 data.hdr_len = params.hdr_len; 1072 1073 data.handle = sev->handle; 1074 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, &data, &argp->error); 1075 1076 kfree(hdr); 1077 1078e_free_blob: 1079 kfree(blob); 1080e_unpin_memory: 1081 /* content of memory is updated, mark pages dirty */ 1082 for (i = 0; i < n; i++) { 1083 set_page_dirty_lock(pages[i]); 1084 mark_page_accessed(pages[i]); 1085 } 1086 sev_unpin_memory(kvm, pages, n); 1087 return ret; 1088} 1089 1090static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp) 1091{ 1092 void __user *report = (void __user *)(uintptr_t)argp->data; 1093 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1094 struct sev_data_attestation_report data; 1095 struct kvm_sev_attestation_report params; 1096 void __user *p; 1097 void *blob = NULL; 1098 int ret; 1099 1100 if (!sev_guest(kvm)) 1101 return -ENOTTY; 1102 1103 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params))) 1104 return -EFAULT; 1105 1106 memset(&data, 0, sizeof(data)); 1107 1108 /* User wants to query the blob length */ 1109 if (!params.len) 1110 goto cmd; 1111 1112 p = (void __user *)(uintptr_t)params.uaddr; 1113 if (p) { 1114 if (params.len > SEV_FW_BLOB_MAX_SIZE) 1115 return -EINVAL; 1116 1117 blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT); 1118 if (!blob) 1119 return -ENOMEM; 1120 1121 data.address = __psp_pa(blob); 1122 data.len = params.len; 1123 memcpy(data.mnonce, params.mnonce, sizeof(params.mnonce)); 1124 } 1125cmd: 1126 data.handle = sev->handle; 1127 ret = sev_issue_cmd(kvm, SEV_CMD_ATTESTATION_REPORT, &data, &argp->error); 1128 /* 1129 * If we query the session length, FW responded with expected data. 1130 */ 1131 if (!params.len) 1132 goto done; 1133 1134 if (ret) 1135 goto e_free_blob; 1136 1137 if (blob) { 1138 if (copy_to_user(p, blob, params.len)) 1139 ret = -EFAULT; 1140 } 1141 1142done: 1143 params.len = data.len; 1144 if (copy_to_user(report, ¶ms, sizeof(params))) 1145 ret = -EFAULT; 1146e_free_blob: 1147 kfree(blob); 1148 return ret; 1149} 1150 1151/* Userspace wants to query session length. */ 1152static int 1153__sev_send_start_query_session_length(struct kvm *kvm, struct kvm_sev_cmd *argp, 1154 struct kvm_sev_send_start *params) 1155{ 1156 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1157 struct sev_data_send_start data; 1158 int ret; 1159 1160 memset(&data, 0, sizeof(data)); 1161 data.handle = sev->handle; 1162 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error); 1163 1164 params->session_len = data.session_len; 1165 if (copy_to_user((void __user *)(uintptr_t)argp->data, params, 1166 sizeof(struct kvm_sev_send_start))) 1167 ret = -EFAULT; 1168 1169 return ret; 1170} 1171 1172static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 1173{ 1174 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1175 struct sev_data_send_start data; 1176 struct kvm_sev_send_start params; 1177 void *amd_certs, *session_data; 1178 void *pdh_cert, *plat_certs; 1179 int ret; 1180 1181 if (!sev_guest(kvm)) 1182 return -ENOTTY; 1183 1184 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, 1185 sizeof(struct kvm_sev_send_start))) 1186 return -EFAULT; 1187 1188 /* if session_len is zero, userspace wants to query the session length */ 1189 if (!params.session_len) 1190 return __sev_send_start_query_session_length(kvm, argp, 1191 ¶ms); 1192 1193 /* some sanity checks */ 1194 if (!params.pdh_cert_uaddr || !params.pdh_cert_len || 1195 !params.session_uaddr || params.session_len > SEV_FW_BLOB_MAX_SIZE) 1196 return -EINVAL; 1197 1198 /* allocate the memory to hold the session data blob */ 1199 session_data = kzalloc(params.session_len, GFP_KERNEL_ACCOUNT); 1200 if (!session_data) 1201 return -ENOMEM; 1202 1203 /* copy the certificate blobs from userspace */ 1204 pdh_cert = psp_copy_user_blob(params.pdh_cert_uaddr, 1205 params.pdh_cert_len); 1206 if (IS_ERR(pdh_cert)) { 1207 ret = PTR_ERR(pdh_cert); 1208 goto e_free_session; 1209 } 1210 1211 plat_certs = psp_copy_user_blob(params.plat_certs_uaddr, 1212 params.plat_certs_len); 1213 if (IS_ERR(plat_certs)) { 1214 ret = PTR_ERR(plat_certs); 1215 goto e_free_pdh; 1216 } 1217 1218 amd_certs = psp_copy_user_blob(params.amd_certs_uaddr, 1219 params.amd_certs_len); 1220 if (IS_ERR(amd_certs)) { 1221 ret = PTR_ERR(amd_certs); 1222 goto e_free_plat_cert; 1223 } 1224 1225 /* populate the FW SEND_START field with system physical address */ 1226 memset(&data, 0, sizeof(data)); 1227 data.pdh_cert_address = __psp_pa(pdh_cert); 1228 data.pdh_cert_len = params.pdh_cert_len; 1229 data.plat_certs_address = __psp_pa(plat_certs); 1230 data.plat_certs_len = params.plat_certs_len; 1231 data.amd_certs_address = __psp_pa(amd_certs); 1232 data.amd_certs_len = params.amd_certs_len; 1233 data.session_address = __psp_pa(session_data); 1234 data.session_len = params.session_len; 1235 data.handle = sev->handle; 1236 1237 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_START, &data, &argp->error); 1238 1239 if (!ret && copy_to_user((void __user *)(uintptr_t)params.session_uaddr, 1240 session_data, params.session_len)) { 1241 ret = -EFAULT; 1242 goto e_free_amd_cert; 1243 } 1244 1245 params.policy = data.policy; 1246 params.session_len = data.session_len; 1247 if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, 1248 sizeof(struct kvm_sev_send_start))) 1249 ret = -EFAULT; 1250 1251e_free_amd_cert: 1252 kfree(amd_certs); 1253e_free_plat_cert: 1254 kfree(plat_certs); 1255e_free_pdh: 1256 kfree(pdh_cert); 1257e_free_session: 1258 kfree(session_data); 1259 return ret; 1260} 1261 1262/* Userspace wants to query either header or trans length. */ 1263static int 1264__sev_send_update_data_query_lengths(struct kvm *kvm, struct kvm_sev_cmd *argp, 1265 struct kvm_sev_send_update_data *params) 1266{ 1267 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1268 struct sev_data_send_update_data data; 1269 int ret; 1270 1271 memset(&data, 0, sizeof(data)); 1272 data.handle = sev->handle; 1273 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error); 1274 1275 params->hdr_len = data.hdr_len; 1276 params->trans_len = data.trans_len; 1277 1278 if (copy_to_user((void __user *)(uintptr_t)argp->data, params, 1279 sizeof(struct kvm_sev_send_update_data))) 1280 ret = -EFAULT; 1281 1282 return ret; 1283} 1284 1285static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) 1286{ 1287 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1288 struct sev_data_send_update_data data; 1289 struct kvm_sev_send_update_data params; 1290 void *hdr, *trans_data; 1291 struct page **guest_page; 1292 unsigned long n; 1293 int ret, offset; 1294 1295 if (!sev_guest(kvm)) 1296 return -ENOTTY; 1297 1298 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, 1299 sizeof(struct kvm_sev_send_update_data))) 1300 return -EFAULT; 1301 1302 /* userspace wants to query either header or trans length */ 1303 if (!params.trans_len || !params.hdr_len) 1304 return __sev_send_update_data_query_lengths(kvm, argp, ¶ms); 1305 1306 if (!params.trans_uaddr || !params.guest_uaddr || 1307 !params.guest_len || !params.hdr_uaddr) 1308 return -EINVAL; 1309 1310 /* Check if we are crossing the page boundary */ 1311 offset = params.guest_uaddr & (PAGE_SIZE - 1); 1312 if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE) 1313 return -EINVAL; 1314 1315 /* Pin guest memory */ 1316 guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK, 1317 PAGE_SIZE, &n, 0); 1318 if (IS_ERR(guest_page)) 1319 return PTR_ERR(guest_page); 1320 1321 /* allocate memory for header and transport buffer */ 1322 ret = -ENOMEM; 1323 hdr = kzalloc(params.hdr_len, GFP_KERNEL_ACCOUNT); 1324 if (!hdr) 1325 goto e_unpin; 1326 1327 trans_data = kzalloc(params.trans_len, GFP_KERNEL_ACCOUNT); 1328 if (!trans_data) 1329 goto e_free_hdr; 1330 1331 memset(&data, 0, sizeof(data)); 1332 data.hdr_address = __psp_pa(hdr); 1333 data.hdr_len = params.hdr_len; 1334 data.trans_address = __psp_pa(trans_data); 1335 data.trans_len = params.trans_len; 1336 1337 /* The SEND_UPDATE_DATA command requires C-bit to be always set. */ 1338 data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset; 1339 data.guest_address |= sev_me_mask; 1340 data.guest_len = params.guest_len; 1341 data.handle = sev->handle; 1342 1343 ret = sev_issue_cmd(kvm, SEV_CMD_SEND_UPDATE_DATA, &data, &argp->error); 1344 1345 if (ret) 1346 goto e_free_trans_data; 1347 1348 /* copy transport buffer to user space */ 1349 if (copy_to_user((void __user *)(uintptr_t)params.trans_uaddr, 1350 trans_data, params.trans_len)) { 1351 ret = -EFAULT; 1352 goto e_free_trans_data; 1353 } 1354 1355 /* Copy packet header to userspace. */ 1356 if (copy_to_user((void __user *)(uintptr_t)params.hdr_uaddr, hdr, 1357 params.hdr_len)) 1358 ret = -EFAULT; 1359 1360e_free_trans_data: 1361 kfree(trans_data); 1362e_free_hdr: 1363 kfree(hdr); 1364e_unpin: 1365 sev_unpin_memory(kvm, guest_page, n); 1366 1367 return ret; 1368} 1369 1370static int sev_send_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 1371{ 1372 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1373 struct sev_data_send_finish data; 1374 1375 if (!sev_guest(kvm)) 1376 return -ENOTTY; 1377 1378 data.handle = sev->handle; 1379 return sev_issue_cmd(kvm, SEV_CMD_SEND_FINISH, &data, &argp->error); 1380} 1381 1382static int sev_send_cancel(struct kvm *kvm, struct kvm_sev_cmd *argp) 1383{ 1384 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1385 struct sev_data_send_cancel data; 1386 1387 if (!sev_guest(kvm)) 1388 return -ENOTTY; 1389 1390 data.handle = sev->handle; 1391 return sev_issue_cmd(kvm, SEV_CMD_SEND_CANCEL, &data, &argp->error); 1392} 1393 1394static int sev_receive_start(struct kvm *kvm, struct kvm_sev_cmd *argp) 1395{ 1396 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1397 struct sev_data_receive_start start; 1398 struct kvm_sev_receive_start params; 1399 int *error = &argp->error; 1400 void *session_data; 1401 void *pdh_data; 1402 int ret; 1403 1404 if (!sev_guest(kvm)) 1405 return -ENOTTY; 1406 1407 /* Get parameter from the userspace */ 1408 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, 1409 sizeof(struct kvm_sev_receive_start))) 1410 return -EFAULT; 1411 1412 /* some sanity checks */ 1413 if (!params.pdh_uaddr || !params.pdh_len || 1414 !params.session_uaddr || !params.session_len) 1415 return -EINVAL; 1416 1417 pdh_data = psp_copy_user_blob(params.pdh_uaddr, params.pdh_len); 1418 if (IS_ERR(pdh_data)) 1419 return PTR_ERR(pdh_data); 1420 1421 session_data = psp_copy_user_blob(params.session_uaddr, 1422 params.session_len); 1423 if (IS_ERR(session_data)) { 1424 ret = PTR_ERR(session_data); 1425 goto e_free_pdh; 1426 } 1427 1428 memset(&start, 0, sizeof(start)); 1429 start.handle = params.handle; 1430 start.policy = params.policy; 1431 start.pdh_cert_address = __psp_pa(pdh_data); 1432 start.pdh_cert_len = params.pdh_len; 1433 start.session_address = __psp_pa(session_data); 1434 start.session_len = params.session_len; 1435 1436 /* create memory encryption context */ 1437 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_RECEIVE_START, &start, 1438 error); 1439 if (ret) 1440 goto e_free_session; 1441 1442 /* Bind ASID to this guest */ 1443 ret = sev_bind_asid(kvm, start.handle, error); 1444 if (ret) { 1445 sev_decommission(start.handle); 1446 goto e_free_session; 1447 } 1448 1449 params.handle = start.handle; 1450 if (copy_to_user((void __user *)(uintptr_t)argp->data, 1451 ¶ms, sizeof(struct kvm_sev_receive_start))) { 1452 ret = -EFAULT; 1453 sev_unbind_asid(kvm, start.handle); 1454 goto e_free_session; 1455 } 1456 1457 sev->handle = start.handle; 1458 sev->fd = argp->sev_fd; 1459 1460e_free_session: 1461 kfree(session_data); 1462e_free_pdh: 1463 kfree(pdh_data); 1464 1465 return ret; 1466} 1467 1468static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) 1469{ 1470 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1471 struct kvm_sev_receive_update_data params; 1472 struct sev_data_receive_update_data data; 1473 void *hdr = NULL, *trans = NULL; 1474 struct page **guest_page; 1475 unsigned long n; 1476 int ret, offset; 1477 1478 if (!sev_guest(kvm)) 1479 return -EINVAL; 1480 1481 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, 1482 sizeof(struct kvm_sev_receive_update_data))) 1483 return -EFAULT; 1484 1485 if (!params.hdr_uaddr || !params.hdr_len || 1486 !params.guest_uaddr || !params.guest_len || 1487 !params.trans_uaddr || !params.trans_len) 1488 return -EINVAL; 1489 1490 /* Check if we are crossing the page boundary */ 1491 offset = params.guest_uaddr & (PAGE_SIZE - 1); 1492 if (params.guest_len > PAGE_SIZE || (params.guest_len + offset) > PAGE_SIZE) 1493 return -EINVAL; 1494 1495 hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len); 1496 if (IS_ERR(hdr)) 1497 return PTR_ERR(hdr); 1498 1499 trans = psp_copy_user_blob(params.trans_uaddr, params.trans_len); 1500 if (IS_ERR(trans)) { 1501 ret = PTR_ERR(trans); 1502 goto e_free_hdr; 1503 } 1504 1505 memset(&data, 0, sizeof(data)); 1506 data.hdr_address = __psp_pa(hdr); 1507 data.hdr_len = params.hdr_len; 1508 data.trans_address = __psp_pa(trans); 1509 data.trans_len = params.trans_len; 1510 1511 /* Pin guest memory */ 1512 guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK, 1513 PAGE_SIZE, &n, 1); 1514 if (IS_ERR(guest_page)) { 1515 ret = PTR_ERR(guest_page); 1516 goto e_free_trans; 1517 } 1518 1519 /* 1520 * Flush (on non-coherent CPUs) before RECEIVE_UPDATE_DATA, the PSP 1521 * encrypts the written data with the guest's key, and the cache may 1522 * contain dirty, unencrypted data. 1523 */ 1524 sev_clflush_pages(guest_page, n); 1525 1526 /* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */ 1527 data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset; 1528 data.guest_address |= sev_me_mask; 1529 data.guest_len = params.guest_len; 1530 data.handle = sev->handle; 1531 1532 ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data, 1533 &argp->error); 1534 1535 sev_unpin_memory(kvm, guest_page, n); 1536 1537e_free_trans: 1538 kfree(trans); 1539e_free_hdr: 1540 kfree(hdr); 1541 1542 return ret; 1543} 1544 1545static int sev_receive_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) 1546{ 1547 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1548 struct sev_data_receive_finish data; 1549 1550 if (!sev_guest(kvm)) 1551 return -ENOTTY; 1552 1553 data.handle = sev->handle; 1554 return sev_issue_cmd(kvm, SEV_CMD_RECEIVE_FINISH, &data, &argp->error); 1555} 1556 1557static bool is_cmd_allowed_from_mirror(u32 cmd_id) 1558{ 1559 /* 1560 * Allow mirrors VM to call KVM_SEV_LAUNCH_UPDATE_VMSA to enable SEV-ES 1561 * active mirror VMs. Also allow the debugging and status commands. 1562 */ 1563 if (cmd_id == KVM_SEV_LAUNCH_UPDATE_VMSA || 1564 cmd_id == KVM_SEV_GUEST_STATUS || cmd_id == KVM_SEV_DBG_DECRYPT || 1565 cmd_id == KVM_SEV_DBG_ENCRYPT) 1566 return true; 1567 1568 return false; 1569} 1570 1571static int sev_lock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) 1572{ 1573 struct kvm_sev_info *dst_sev = &to_kvm_svm(dst_kvm)->sev_info; 1574 struct kvm_sev_info *src_sev = &to_kvm_svm(src_kvm)->sev_info; 1575 int r = -EBUSY; 1576 1577 if (dst_kvm == src_kvm) 1578 return -EINVAL; 1579 1580 /* 1581 * Bail if these VMs are already involved in a migration to avoid 1582 * deadlock between two VMs trying to migrate to/from each other. 1583 */ 1584 if (atomic_cmpxchg_acquire(&dst_sev->migration_in_progress, 0, 1)) 1585 return -EBUSY; 1586 1587 if (atomic_cmpxchg_acquire(&src_sev->migration_in_progress, 0, 1)) 1588 goto release_dst; 1589 1590 r = -EINTR; 1591 if (mutex_lock_killable(&dst_kvm->lock)) 1592 goto release_src; 1593 if (mutex_lock_killable_nested(&src_kvm->lock, SINGLE_DEPTH_NESTING)) 1594 goto unlock_dst; 1595 return 0; 1596 1597unlock_dst: 1598 mutex_unlock(&dst_kvm->lock); 1599release_src: 1600 atomic_set_release(&src_sev->migration_in_progress, 0); 1601release_dst: 1602 atomic_set_release(&dst_sev->migration_in_progress, 0); 1603 return r; 1604} 1605 1606static void sev_unlock_two_vms(struct kvm *dst_kvm, struct kvm *src_kvm) 1607{ 1608 struct kvm_sev_info *dst_sev = &to_kvm_svm(dst_kvm)->sev_info; 1609 struct kvm_sev_info *src_sev = &to_kvm_svm(src_kvm)->sev_info; 1610 1611 mutex_unlock(&dst_kvm->lock); 1612 mutex_unlock(&src_kvm->lock); 1613 atomic_set_release(&dst_sev->migration_in_progress, 0); 1614 atomic_set_release(&src_sev->migration_in_progress, 0); 1615} 1616 1617/* vCPU mutex subclasses. */ 1618enum sev_migration_role { 1619 SEV_MIGRATION_SOURCE = 0, 1620 SEV_MIGRATION_TARGET, 1621 SEV_NR_MIGRATION_ROLES, 1622}; 1623 1624static int sev_lock_vcpus_for_migration(struct kvm *kvm, 1625 enum sev_migration_role role) 1626{ 1627 struct kvm_vcpu *vcpu; 1628 unsigned long i, j; 1629 1630 kvm_for_each_vcpu(i, vcpu, kvm) { 1631 if (mutex_lock_killable_nested(&vcpu->mutex, role)) 1632 goto out_unlock; 1633 1634#ifdef CONFIG_PROVE_LOCKING 1635 if (!i) 1636 /* 1637 * Reset the role to one that avoids colliding with 1638 * the role used for the first vcpu mutex. 1639 */ 1640 role = SEV_NR_MIGRATION_ROLES; 1641 else 1642 mutex_release(&vcpu->mutex.dep_map, _THIS_IP_); 1643#endif 1644 } 1645 1646 return 0; 1647 1648out_unlock: 1649 1650 kvm_for_each_vcpu(j, vcpu, kvm) { 1651 if (i == j) 1652 break; 1653 1654#ifdef CONFIG_PROVE_LOCKING 1655 if (j) 1656 mutex_acquire(&vcpu->mutex.dep_map, role, 0, _THIS_IP_); 1657#endif 1658 1659 mutex_unlock(&vcpu->mutex); 1660 } 1661 return -EINTR; 1662} 1663 1664static void sev_unlock_vcpus_for_migration(struct kvm *kvm) 1665{ 1666 struct kvm_vcpu *vcpu; 1667 unsigned long i; 1668 bool first = true; 1669 1670 kvm_for_each_vcpu(i, vcpu, kvm) { 1671 if (first) 1672 first = false; 1673 else 1674 mutex_acquire(&vcpu->mutex.dep_map, 1675 SEV_NR_MIGRATION_ROLES, 0, _THIS_IP_); 1676 1677 mutex_unlock(&vcpu->mutex); 1678 } 1679} 1680 1681static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) 1682{ 1683 struct kvm_sev_info *dst = &to_kvm_svm(dst_kvm)->sev_info; 1684 struct kvm_sev_info *src = &to_kvm_svm(src_kvm)->sev_info; 1685 struct kvm_vcpu *dst_vcpu, *src_vcpu; 1686 struct vcpu_svm *dst_svm, *src_svm; 1687 struct kvm_sev_info *mirror; 1688 unsigned long i; 1689 1690 dst->active = true; 1691 dst->asid = src->asid; 1692 dst->handle = src->handle; 1693 dst->pages_locked = src->pages_locked; 1694 dst->enc_context_owner = src->enc_context_owner; 1695 dst->es_active = src->es_active; 1696 1697 src->asid = 0; 1698 src->active = false; 1699 src->handle = 0; 1700 src->pages_locked = 0; 1701 src->enc_context_owner = NULL; 1702 src->es_active = false; 1703 1704 list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); 1705 1706 /* 1707 * If this VM has mirrors, "transfer" each mirror's refcount of the 1708 * source to the destination (this KVM). The caller holds a reference 1709 * to the source, so there's no danger of use-after-free. 1710 */ 1711 list_cut_before(&dst->mirror_vms, &src->mirror_vms, &src->mirror_vms); 1712 list_for_each_entry(mirror, &dst->mirror_vms, mirror_entry) { 1713 kvm_get_kvm(dst_kvm); 1714 kvm_put_kvm(src_kvm); 1715 mirror->enc_context_owner = dst_kvm; 1716 } 1717 1718 /* 1719 * If this VM is a mirror, remove the old mirror from the owners list 1720 * and add the new mirror to the list. 1721 */ 1722 if (is_mirroring_enc_context(dst_kvm)) { 1723 struct kvm_sev_info *owner_sev_info = 1724 &to_kvm_svm(dst->enc_context_owner)->sev_info; 1725 1726 list_del(&src->mirror_entry); 1727 list_add_tail(&dst->mirror_entry, &owner_sev_info->mirror_vms); 1728 } 1729 1730 kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) { 1731 dst_svm = to_svm(dst_vcpu); 1732 1733 sev_init_vmcb(dst_svm); 1734 1735 if (!dst->es_active) 1736 continue; 1737 1738 /* 1739 * Note, the source is not required to have the same number of 1740 * vCPUs as the destination when migrating a vanilla SEV VM. 1741 */ 1742 src_vcpu = kvm_get_vcpu(src_kvm, i); 1743 src_svm = to_svm(src_vcpu); 1744 1745 /* 1746 * Transfer VMSA and GHCB state to the destination. Nullify and 1747 * clear source fields as appropriate, the state now belongs to 1748 * the destination. 1749 */ 1750 memcpy(&dst_svm->sev_es, &src_svm->sev_es, sizeof(src_svm->sev_es)); 1751 dst_svm->vmcb->control.ghcb_gpa = src_svm->vmcb->control.ghcb_gpa; 1752 dst_svm->vmcb->control.vmsa_pa = src_svm->vmcb->control.vmsa_pa; 1753 dst_vcpu->arch.guest_state_protected = true; 1754 1755 memset(&src_svm->sev_es, 0, sizeof(src_svm->sev_es)); 1756 src_svm->vmcb->control.ghcb_gpa = INVALID_PAGE; 1757 src_svm->vmcb->control.vmsa_pa = INVALID_PAGE; 1758 src_vcpu->arch.guest_state_protected = false; 1759 } 1760} 1761 1762static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) 1763{ 1764 struct kvm_vcpu *src_vcpu; 1765 unsigned long i; 1766 1767 if (!sev_es_guest(src)) 1768 return 0; 1769 1770 if (atomic_read(&src->online_vcpus) != atomic_read(&dst->online_vcpus)) 1771 return -EINVAL; 1772 1773 kvm_for_each_vcpu(i, src_vcpu, src) { 1774 if (!src_vcpu->arch.guest_state_protected) 1775 return -EINVAL; 1776 } 1777 1778 return 0; 1779} 1780 1781int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) 1782{ 1783 struct kvm_sev_info *dst_sev = &to_kvm_svm(kvm)->sev_info; 1784 struct kvm_sev_info *src_sev, *cg_cleanup_sev; 1785 struct fd f = fdget(source_fd); 1786 struct kvm *source_kvm; 1787 bool charged = false; 1788 int ret; 1789 1790 if (!f.file) 1791 return -EBADF; 1792 1793 if (!file_is_kvm(f.file)) { 1794 ret = -EBADF; 1795 goto out_fput; 1796 } 1797 1798 source_kvm = f.file->private_data; 1799 ret = sev_lock_two_vms(kvm, source_kvm); 1800 if (ret) 1801 goto out_fput; 1802 1803 if (sev_guest(kvm) || !sev_guest(source_kvm)) { 1804 ret = -EINVAL; 1805 goto out_unlock; 1806 } 1807 1808 src_sev = &to_kvm_svm(source_kvm)->sev_info; 1809 1810 dst_sev->misc_cg = get_current_misc_cg(); 1811 cg_cleanup_sev = dst_sev; 1812 if (dst_sev->misc_cg != src_sev->misc_cg) { 1813 ret = sev_misc_cg_try_charge(dst_sev); 1814 if (ret) 1815 goto out_dst_cgroup; 1816 charged = true; 1817 } 1818 1819 ret = sev_lock_vcpus_for_migration(kvm, SEV_MIGRATION_SOURCE); 1820 if (ret) 1821 goto out_dst_cgroup; 1822 ret = sev_lock_vcpus_for_migration(source_kvm, SEV_MIGRATION_TARGET); 1823 if (ret) 1824 goto out_dst_vcpu; 1825 1826 ret = sev_check_source_vcpus(kvm, source_kvm); 1827 if (ret) 1828 goto out_source_vcpu; 1829 1830 sev_migrate_from(kvm, source_kvm); 1831 kvm_vm_dead(source_kvm); 1832 cg_cleanup_sev = src_sev; 1833 ret = 0; 1834 1835out_source_vcpu: 1836 sev_unlock_vcpus_for_migration(source_kvm); 1837out_dst_vcpu: 1838 sev_unlock_vcpus_for_migration(kvm); 1839out_dst_cgroup: 1840 /* Operates on the source on success, on the destination on failure. */ 1841 if (charged) 1842 sev_misc_cg_uncharge(cg_cleanup_sev); 1843 put_misc_cg(cg_cleanup_sev->misc_cg); 1844 cg_cleanup_sev->misc_cg = NULL; 1845out_unlock: 1846 sev_unlock_two_vms(kvm, source_kvm); 1847out_fput: 1848 fdput(f); 1849 return ret; 1850} 1851 1852int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp) 1853{ 1854 struct kvm_sev_cmd sev_cmd; 1855 int r; 1856 1857 if (!sev_enabled) 1858 return -ENOTTY; 1859 1860 if (!argp) 1861 return 0; 1862 1863 if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd))) 1864 return -EFAULT; 1865 1866 mutex_lock(&kvm->lock); 1867 1868 /* Only the enc_context_owner handles some memory enc operations. */ 1869 if (is_mirroring_enc_context(kvm) && 1870 !is_cmd_allowed_from_mirror(sev_cmd.id)) { 1871 r = -EINVAL; 1872 goto out; 1873 } 1874 1875 switch (sev_cmd.id) { 1876 case KVM_SEV_ES_INIT: 1877 if (!sev_es_enabled) { 1878 r = -ENOTTY; 1879 goto out; 1880 } 1881 fallthrough; 1882 case KVM_SEV_INIT: 1883 r = sev_guest_init(kvm, &sev_cmd); 1884 break; 1885 case KVM_SEV_LAUNCH_START: 1886 r = sev_launch_start(kvm, &sev_cmd); 1887 break; 1888 case KVM_SEV_LAUNCH_UPDATE_DATA: 1889 r = sev_launch_update_data(kvm, &sev_cmd); 1890 break; 1891 case KVM_SEV_LAUNCH_UPDATE_VMSA: 1892 r = sev_launch_update_vmsa(kvm, &sev_cmd); 1893 break; 1894 case KVM_SEV_LAUNCH_MEASURE: 1895 r = sev_launch_measure(kvm, &sev_cmd); 1896 break; 1897 case KVM_SEV_LAUNCH_FINISH: 1898 r = sev_launch_finish(kvm, &sev_cmd); 1899 break; 1900 case KVM_SEV_GUEST_STATUS: 1901 r = sev_guest_status(kvm, &sev_cmd); 1902 break; 1903 case KVM_SEV_DBG_DECRYPT: 1904 r = sev_dbg_crypt(kvm, &sev_cmd, true); 1905 break; 1906 case KVM_SEV_DBG_ENCRYPT: 1907 r = sev_dbg_crypt(kvm, &sev_cmd, false); 1908 break; 1909 case KVM_SEV_LAUNCH_SECRET: 1910 r = sev_launch_secret(kvm, &sev_cmd); 1911 break; 1912 case KVM_SEV_GET_ATTESTATION_REPORT: 1913 r = sev_get_attestation_report(kvm, &sev_cmd); 1914 break; 1915 case KVM_SEV_SEND_START: 1916 r = sev_send_start(kvm, &sev_cmd); 1917 break; 1918 case KVM_SEV_SEND_UPDATE_DATA: 1919 r = sev_send_update_data(kvm, &sev_cmd); 1920 break; 1921 case KVM_SEV_SEND_FINISH: 1922 r = sev_send_finish(kvm, &sev_cmd); 1923 break; 1924 case KVM_SEV_SEND_CANCEL: 1925 r = sev_send_cancel(kvm, &sev_cmd); 1926 break; 1927 case KVM_SEV_RECEIVE_START: 1928 r = sev_receive_start(kvm, &sev_cmd); 1929 break; 1930 case KVM_SEV_RECEIVE_UPDATE_DATA: 1931 r = sev_receive_update_data(kvm, &sev_cmd); 1932 break; 1933 case KVM_SEV_RECEIVE_FINISH: 1934 r = sev_receive_finish(kvm, &sev_cmd); 1935 break; 1936 default: 1937 r = -EINVAL; 1938 goto out; 1939 } 1940 1941 if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd))) 1942 r = -EFAULT; 1943 1944out: 1945 mutex_unlock(&kvm->lock); 1946 return r; 1947} 1948 1949int sev_mem_enc_register_region(struct kvm *kvm, 1950 struct kvm_enc_region *range) 1951{ 1952 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 1953 struct enc_region *region; 1954 int ret = 0; 1955 1956 if (!sev_guest(kvm)) 1957 return -ENOTTY; 1958 1959 /* If kvm is mirroring encryption context it isn't responsible for it */ 1960 if (is_mirroring_enc_context(kvm)) 1961 return -EINVAL; 1962 1963 if (range->addr > ULONG_MAX || range->size > ULONG_MAX) 1964 return -EINVAL; 1965 1966 region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT); 1967 if (!region) 1968 return -ENOMEM; 1969 1970 mutex_lock(&kvm->lock); 1971 region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, 1); 1972 if (IS_ERR(region->pages)) { 1973 ret = PTR_ERR(region->pages); 1974 mutex_unlock(&kvm->lock); 1975 goto e_free; 1976 } 1977 1978 region->uaddr = range->addr; 1979 region->size = range->size; 1980 1981 list_add_tail(®ion->list, &sev->regions_list); 1982 mutex_unlock(&kvm->lock); 1983 1984 /* 1985 * The guest may change the memory encryption attribute from C=0 -> C=1 1986 * or vice versa for this memory range. Lets make sure caches are 1987 * flushed to ensure that guest data gets written into memory with 1988 * correct C-bit. 1989 */ 1990 sev_clflush_pages(region->pages, region->npages); 1991 1992 return ret; 1993 1994e_free: 1995 kfree(region); 1996 return ret; 1997} 1998 1999static struct enc_region * 2000find_enc_region(struct kvm *kvm, struct kvm_enc_region *range) 2001{ 2002 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 2003 struct list_head *head = &sev->regions_list; 2004 struct enc_region *i; 2005 2006 list_for_each_entry(i, head, list) { 2007 if (i->uaddr == range->addr && 2008 i->size == range->size) 2009 return i; 2010 } 2011 2012 return NULL; 2013} 2014 2015static void __unregister_enc_region_locked(struct kvm *kvm, 2016 struct enc_region *region) 2017{ 2018 sev_unpin_memory(kvm, region->pages, region->npages); 2019 list_del(®ion->list); 2020 kfree(region); 2021} 2022 2023int sev_mem_enc_unregister_region(struct kvm *kvm, 2024 struct kvm_enc_region *range) 2025{ 2026 struct enc_region *region; 2027 int ret; 2028 2029 /* If kvm is mirroring encryption context it isn't responsible for it */ 2030 if (is_mirroring_enc_context(kvm)) 2031 return -EINVAL; 2032 2033 mutex_lock(&kvm->lock); 2034 2035 if (!sev_guest(kvm)) { 2036 ret = -ENOTTY; 2037 goto failed; 2038 } 2039 2040 region = find_enc_region(kvm, range); 2041 if (!region) { 2042 ret = -EINVAL; 2043 goto failed; 2044 } 2045 2046 /* 2047 * Ensure that all guest tagged cache entries are flushed before 2048 * releasing the pages back to the system for use. CLFLUSH will 2049 * not do this, so issue a WBINVD. 2050 */ 2051 wbinvd_on_all_cpus(); 2052 2053 __unregister_enc_region_locked(kvm, region); 2054 2055 mutex_unlock(&kvm->lock); 2056 return 0; 2057 2058failed: 2059 mutex_unlock(&kvm->lock); 2060 return ret; 2061} 2062 2063int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) 2064{ 2065 struct fd f = fdget(source_fd); 2066 struct kvm *source_kvm; 2067 struct kvm_sev_info *source_sev, *mirror_sev; 2068 int ret; 2069 2070 if (!f.file) 2071 return -EBADF; 2072 2073 if (!file_is_kvm(f.file)) { 2074 ret = -EBADF; 2075 goto e_source_fput; 2076 } 2077 2078 source_kvm = f.file->private_data; 2079 ret = sev_lock_two_vms(kvm, source_kvm); 2080 if (ret) 2081 goto e_source_fput; 2082 2083 /* 2084 * Mirrors of mirrors should work, but let's not get silly. Also 2085 * disallow out-of-band SEV/SEV-ES init if the target is already an 2086 * SEV guest, or if vCPUs have been created. KVM relies on vCPUs being 2087 * created after SEV/SEV-ES initialization, e.g. to init intercepts. 2088 */ 2089 if (sev_guest(kvm) || !sev_guest(source_kvm) || 2090 is_mirroring_enc_context(source_kvm) || kvm->created_vcpus) { 2091 ret = -EINVAL; 2092 goto e_unlock; 2093 } 2094 2095 /* 2096 * The mirror kvm holds an enc_context_owner ref so its asid can't 2097 * disappear until we're done with it 2098 */ 2099 source_sev = &to_kvm_svm(source_kvm)->sev_info; 2100 kvm_get_kvm(source_kvm); 2101 mirror_sev = &to_kvm_svm(kvm)->sev_info; 2102 list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms); 2103 2104 /* Set enc_context_owner and copy its encryption context over */ 2105 mirror_sev->enc_context_owner = source_kvm; 2106 mirror_sev->active = true; 2107 mirror_sev->asid = source_sev->asid; 2108 mirror_sev->fd = source_sev->fd; 2109 mirror_sev->es_active = source_sev->es_active; 2110 mirror_sev->handle = source_sev->handle; 2111 INIT_LIST_HEAD(&mirror_sev->regions_list); 2112 INIT_LIST_HEAD(&mirror_sev->mirror_vms); 2113 ret = 0; 2114 2115 /* 2116 * Do not copy ap_jump_table. Since the mirror does not share the same 2117 * KVM contexts as the original, and they may have different 2118 * memory-views. 2119 */ 2120 2121e_unlock: 2122 sev_unlock_two_vms(kvm, source_kvm); 2123e_source_fput: 2124 fdput(f); 2125 return ret; 2126} 2127 2128void sev_vm_destroy(struct kvm *kvm) 2129{ 2130 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info; 2131 struct list_head *head = &sev->regions_list; 2132 struct list_head *pos, *q; 2133 2134 if (!sev_guest(kvm)) 2135 return; 2136 2137 WARN_ON(!list_empty(&sev->mirror_vms)); 2138 2139 /* If this is a mirror_kvm release the enc_context_owner and skip sev cleanup */ 2140 if (is_mirroring_enc_context(kvm)) { 2141 struct kvm *owner_kvm = sev->enc_context_owner; 2142 2143 mutex_lock(&owner_kvm->lock); 2144 list_del(&sev->mirror_entry); 2145 mutex_unlock(&owner_kvm->lock); 2146 kvm_put_kvm(owner_kvm); 2147 return; 2148 } 2149 2150 /* 2151 * Ensure that all guest tagged cache entries are flushed before 2152 * releasing the pages back to the system for use. CLFLUSH will 2153 * not do this, so issue a WBINVD. 2154 */ 2155 wbinvd_on_all_cpus(); 2156 2157 /* 2158 * if userspace was terminated before unregistering the memory regions 2159 * then lets unpin all the registered memory. 2160 */ 2161 if (!list_empty(head)) { 2162 list_for_each_safe(pos, q, head) { 2163 __unregister_enc_region_locked(kvm, 2164 list_entry(pos, struct enc_region, list)); 2165 cond_resched(); 2166 } 2167 } 2168 2169 sev_unbind_asid(kvm, sev->handle); 2170 sev_asid_free(sev); 2171} 2172 2173void __init sev_set_cpu_caps(void) 2174{ 2175 if (!sev_enabled) 2176 kvm_cpu_cap_clear(X86_FEATURE_SEV); 2177 if (!sev_es_enabled) 2178 kvm_cpu_cap_clear(X86_FEATURE_SEV_ES); 2179} 2180 2181void __init sev_hardware_setup(void) 2182{ 2183#ifdef CONFIG_KVM_AMD_SEV 2184 unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count; 2185 bool sev_es_supported = false; 2186 bool sev_supported = false; 2187 2188 if (!sev_enabled || !npt_enabled || !nrips) 2189 goto out; 2190 2191 /* 2192 * SEV must obviously be supported in hardware. Sanity check that the 2193 * CPU supports decode assists, which is mandatory for SEV guests to 2194 * support instruction emulation. 2195 */ 2196 if (!boot_cpu_has(X86_FEATURE_SEV) || 2197 WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_DECODEASSISTS))) 2198 goto out; 2199 2200 /* Retrieve SEV CPUID information */ 2201 cpuid(0x8000001f, &eax, &ebx, &ecx, &edx); 2202 2203 /* Set encryption bit location for SEV-ES guests */ 2204 sev_enc_bit = ebx & 0x3f; 2205 2206 /* Maximum number of encrypted guests supported simultaneously */ 2207 max_sev_asid = ecx; 2208 if (!max_sev_asid) 2209 goto out; 2210 2211 /* Minimum ASID value that should be used for SEV guest */ 2212 min_sev_asid = edx; 2213 sev_me_mask = 1UL << (ebx & 0x3f); 2214 2215 /* 2216 * Initialize SEV ASID bitmaps. Allocate space for ASID 0 in the bitmap, 2217 * even though it's never used, so that the bitmap is indexed by the 2218 * actual ASID. 2219 */ 2220 nr_asids = max_sev_asid + 1; 2221 sev_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL); 2222 if (!sev_asid_bitmap) 2223 goto out; 2224 2225 sev_reclaim_asid_bitmap = bitmap_zalloc(nr_asids, GFP_KERNEL); 2226 if (!sev_reclaim_asid_bitmap) { 2227 bitmap_free(sev_asid_bitmap); 2228 sev_asid_bitmap = NULL; 2229 goto out; 2230 } 2231 2232 sev_asid_count = max_sev_asid - min_sev_asid + 1; 2233 WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV, sev_asid_count)); 2234 sev_supported = true; 2235 2236 /* SEV-ES support requested? */ 2237 if (!sev_es_enabled) 2238 goto out; 2239 2240 /* 2241 * SEV-ES requires MMIO caching as KVM doesn't have access to the guest 2242 * instruction stream, i.e. can't emulate in response to a #NPF and 2243 * instead relies on #NPF(RSVD) being reflected into the guest as #VC 2244 * (the guest can then do a #VMGEXIT to request MMIO emulation). 2245 */ 2246 if (!enable_mmio_caching) 2247 goto out; 2248 2249 /* Does the CPU support SEV-ES? */ 2250 if (!boot_cpu_has(X86_FEATURE_SEV_ES)) 2251 goto out; 2252 2253 /* Has the system been allocated ASIDs for SEV-ES? */ 2254 if (min_sev_asid == 1) 2255 goto out; 2256 2257 sev_es_asid_count = min_sev_asid - 1; 2258 WARN_ON_ONCE(misc_cg_set_capacity(MISC_CG_RES_SEV_ES, sev_es_asid_count)); 2259 sev_es_supported = true; 2260 2261out: 2262 if (boot_cpu_has(X86_FEATURE_SEV)) 2263 pr_info("SEV %s (ASIDs %u - %u)\n", 2264 sev_supported ? "enabled" : "disabled", 2265 min_sev_asid, max_sev_asid); 2266 if (boot_cpu_has(X86_FEATURE_SEV_ES)) 2267 pr_info("SEV-ES %s (ASIDs %u - %u)\n", 2268 sev_es_supported ? "enabled" : "disabled", 2269 min_sev_asid > 1 ? 1 : 0, min_sev_asid - 1); 2270 2271 sev_enabled = sev_supported; 2272 sev_es_enabled = sev_es_supported; 2273 if (!sev_es_enabled || !cpu_feature_enabled(X86_FEATURE_DEBUG_SWAP) || 2274 !cpu_feature_enabled(X86_FEATURE_NO_NESTED_DATA_BP)) 2275 sev_es_debug_swap_enabled = false; 2276#endif 2277} 2278 2279void sev_hardware_unsetup(void) 2280{ 2281 if (!sev_enabled) 2282 return; 2283 2284 /* No need to take sev_bitmap_lock, all VMs have been destroyed. */ 2285 sev_flush_asids(1, max_sev_asid); 2286 2287 bitmap_free(sev_asid_bitmap); 2288 bitmap_free(sev_reclaim_asid_bitmap); 2289 2290 misc_cg_set_capacity(MISC_CG_RES_SEV, 0); 2291 misc_cg_set_capacity(MISC_CG_RES_SEV_ES, 0); 2292} 2293 2294int sev_cpu_init(struct svm_cpu_data *sd) 2295{ 2296 if (!sev_enabled) 2297 return 0; 2298 2299 sd->sev_vmcbs = kcalloc(nr_asids, sizeof(void *), GFP_KERNEL); 2300 if (!sd->sev_vmcbs) 2301 return -ENOMEM; 2302 2303 return 0; 2304} 2305 2306/* 2307 * Pages used by hardware to hold guest encrypted state must be flushed before 2308 * returning them to the system. 2309 */ 2310static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va) 2311{ 2312 int asid = to_kvm_svm(vcpu->kvm)->sev_info.asid; 2313 2314 /* 2315 * Note! The address must be a kernel address, as regular page walk 2316 * checks are performed by VM_PAGE_FLUSH, i.e. operating on a user 2317 * address is non-deterministic and unsafe. This function deliberately 2318 * takes a pointer to deter passing in a user address. 2319 */ 2320 unsigned long addr = (unsigned long)va; 2321 2322 /* 2323 * If CPU enforced cache coherency for encrypted mappings of the 2324 * same physical page is supported, use CLFLUSHOPT instead. NOTE: cache 2325 * flush is still needed in order to work properly with DMA devices. 2326 */ 2327 if (boot_cpu_has(X86_FEATURE_SME_COHERENT)) { 2328 clflush_cache_range(va, PAGE_SIZE); 2329 return; 2330 } 2331 2332 /* 2333 * VM Page Flush takes a host virtual address and a guest ASID. Fall 2334 * back to WBINVD if this faults so as not to make any problems worse 2335 * by leaving stale encrypted data in the cache. 2336 */ 2337 if (WARN_ON_ONCE(wrmsrl_safe(MSR_AMD64_VM_PAGE_FLUSH, addr | asid))) 2338 goto do_wbinvd; 2339 2340 return; 2341 2342do_wbinvd: 2343 wbinvd_on_all_cpus(); 2344} 2345 2346void sev_guest_memory_reclaimed(struct kvm *kvm) 2347{ 2348 if (!sev_guest(kvm)) 2349 return; 2350 2351 wbinvd_on_all_cpus(); 2352} 2353 2354void sev_free_vcpu(struct kvm_vcpu *vcpu) 2355{ 2356 struct vcpu_svm *svm; 2357 2358 if (!sev_es_guest(vcpu->kvm)) 2359 return; 2360 2361 svm = to_svm(vcpu); 2362 2363 if (vcpu->arch.guest_state_protected) 2364 sev_flush_encrypted_page(vcpu, svm->sev_es.vmsa); 2365 2366 __free_page(virt_to_page(svm->sev_es.vmsa)); 2367 2368 if (svm->sev_es.ghcb_sa_free) 2369 kvfree(svm->sev_es.ghcb_sa); 2370} 2371 2372static void dump_ghcb(struct vcpu_svm *svm) 2373{ 2374 struct ghcb *ghcb = svm->sev_es.ghcb; 2375 unsigned int nbits; 2376 2377 /* Re-use the dump_invalid_vmcb module parameter */ 2378 if (!dump_invalid_vmcb) { 2379 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n"); 2380 return; 2381 } 2382 2383 nbits = sizeof(ghcb->save.valid_bitmap) * 8; 2384 2385 pr_err("GHCB (GPA=%016llx):\n", svm->vmcb->control.ghcb_gpa); 2386 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_code", 2387 ghcb->save.sw_exit_code, ghcb_sw_exit_code_is_valid(ghcb)); 2388 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_1", 2389 ghcb->save.sw_exit_info_1, ghcb_sw_exit_info_1_is_valid(ghcb)); 2390 pr_err("%-20s%016llx is_valid: %u\n", "sw_exit_info_2", 2391 ghcb->save.sw_exit_info_2, ghcb_sw_exit_info_2_is_valid(ghcb)); 2392 pr_err("%-20s%016llx is_valid: %u\n", "sw_scratch", 2393 ghcb->save.sw_scratch, ghcb_sw_scratch_is_valid(ghcb)); 2394 pr_err("%-20s%*pb\n", "valid_bitmap", nbits, ghcb->save.valid_bitmap); 2395} 2396 2397static void sev_es_sync_to_ghcb(struct vcpu_svm *svm) 2398{ 2399 struct kvm_vcpu *vcpu = &svm->vcpu; 2400 struct ghcb *ghcb = svm->sev_es.ghcb; 2401 2402 /* 2403 * The GHCB protocol so far allows for the following data 2404 * to be returned: 2405 * GPRs RAX, RBX, RCX, RDX 2406 * 2407 * Copy their values, even if they may not have been written during the 2408 * VM-Exit. It's the guest's responsibility to not consume random data. 2409 */ 2410 ghcb_set_rax(ghcb, vcpu->arch.regs[VCPU_REGS_RAX]); 2411 ghcb_set_rbx(ghcb, vcpu->arch.regs[VCPU_REGS_RBX]); 2412 ghcb_set_rcx(ghcb, vcpu->arch.regs[VCPU_REGS_RCX]); 2413 ghcb_set_rdx(ghcb, vcpu->arch.regs[VCPU_REGS_RDX]); 2414} 2415 2416static void sev_es_sync_from_ghcb(struct vcpu_svm *svm) 2417{ 2418 struct vmcb_control_area *control = &svm->vmcb->control; 2419 struct kvm_vcpu *vcpu = &svm->vcpu; 2420 struct ghcb *ghcb = svm->sev_es.ghcb; 2421 u64 exit_code; 2422 2423 /* 2424 * The GHCB protocol so far allows for the following data 2425 * to be supplied: 2426 * GPRs RAX, RBX, RCX, RDX 2427 * XCR0 2428 * CPL 2429 * 2430 * VMMCALL allows the guest to provide extra registers. KVM also 2431 * expects RSI for hypercalls, so include that, too. 2432 * 2433 * Copy their values to the appropriate location if supplied. 2434 */ 2435 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); 2436 2437 BUILD_BUG_ON(sizeof(svm->sev_es.valid_bitmap) != sizeof(ghcb->save.valid_bitmap)); 2438 memcpy(&svm->sev_es.valid_bitmap, &ghcb->save.valid_bitmap, sizeof(ghcb->save.valid_bitmap)); 2439 2440 vcpu->arch.regs[VCPU_REGS_RAX] = kvm_ghcb_get_rax_if_valid(svm, ghcb); 2441 vcpu->arch.regs[VCPU_REGS_RBX] = kvm_ghcb_get_rbx_if_valid(svm, ghcb); 2442 vcpu->arch.regs[VCPU_REGS_RCX] = kvm_ghcb_get_rcx_if_valid(svm, ghcb); 2443 vcpu->arch.regs[VCPU_REGS_RDX] = kvm_ghcb_get_rdx_if_valid(svm, ghcb); 2444 vcpu->arch.regs[VCPU_REGS_RSI] = kvm_ghcb_get_rsi_if_valid(svm, ghcb); 2445 2446 svm->vmcb->save.cpl = kvm_ghcb_get_cpl_if_valid(svm, ghcb); 2447 2448 if (kvm_ghcb_xcr0_is_valid(svm)) { 2449 vcpu->arch.xcr0 = ghcb_get_xcr0(ghcb); 2450 kvm_update_cpuid_runtime(vcpu); 2451 } 2452 2453 /* Copy the GHCB exit information into the VMCB fields */ 2454 exit_code = ghcb_get_sw_exit_code(ghcb); 2455 control->exit_code = lower_32_bits(exit_code); 2456 control->exit_code_hi = upper_32_bits(exit_code); 2457 control->exit_info_1 = ghcb_get_sw_exit_info_1(ghcb); 2458 control->exit_info_2 = ghcb_get_sw_exit_info_2(ghcb); 2459 svm->sev_es.sw_scratch = kvm_ghcb_get_sw_scratch_if_valid(svm, ghcb); 2460 2461 /* Clear the valid entries fields */ 2462 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap)); 2463} 2464 2465static u64 kvm_ghcb_get_sw_exit_code(struct vmcb_control_area *control) 2466{ 2467 return (((u64)control->exit_code_hi) << 32) | control->exit_code; 2468} 2469 2470static int sev_es_validate_vmgexit(struct vcpu_svm *svm) 2471{ 2472 struct vmcb_control_area *control = &svm->vmcb->control; 2473 struct kvm_vcpu *vcpu = &svm->vcpu; 2474 u64 exit_code; 2475 u64 reason; 2476 2477 /* 2478 * Retrieve the exit code now even though it may not be marked valid 2479 * as it could help with debugging. 2480 */ 2481 exit_code = kvm_ghcb_get_sw_exit_code(control); 2482 2483 /* Only GHCB Usage code 0 is supported */ 2484 if (svm->sev_es.ghcb->ghcb_usage) { 2485 reason = GHCB_ERR_INVALID_USAGE; 2486 goto vmgexit_err; 2487 } 2488 2489 reason = GHCB_ERR_MISSING_INPUT; 2490 2491 if (!kvm_ghcb_sw_exit_code_is_valid(svm) || 2492 !kvm_ghcb_sw_exit_info_1_is_valid(svm) || 2493 !kvm_ghcb_sw_exit_info_2_is_valid(svm)) 2494 goto vmgexit_err; 2495 2496 switch (exit_code) { 2497 case SVM_EXIT_READ_DR7: 2498 break; 2499 case SVM_EXIT_WRITE_DR7: 2500 if (!kvm_ghcb_rax_is_valid(svm)) 2501 goto vmgexit_err; 2502 break; 2503 case SVM_EXIT_RDTSC: 2504 break; 2505 case SVM_EXIT_RDPMC: 2506 if (!kvm_ghcb_rcx_is_valid(svm)) 2507 goto vmgexit_err; 2508 break; 2509 case SVM_EXIT_CPUID: 2510 if (!kvm_ghcb_rax_is_valid(svm) || 2511 !kvm_ghcb_rcx_is_valid(svm)) 2512 goto vmgexit_err; 2513 if (vcpu->arch.regs[VCPU_REGS_RAX] == 0xd) 2514 if (!kvm_ghcb_xcr0_is_valid(svm)) 2515 goto vmgexit_err; 2516 break; 2517 case SVM_EXIT_INVD: 2518 break; 2519 case SVM_EXIT_IOIO: 2520 if (control->exit_info_1 & SVM_IOIO_STR_MASK) { 2521 if (!kvm_ghcb_sw_scratch_is_valid(svm)) 2522 goto vmgexit_err; 2523 } else { 2524 if (!(control->exit_info_1 & SVM_IOIO_TYPE_MASK)) 2525 if (!kvm_ghcb_rax_is_valid(svm)) 2526 goto vmgexit_err; 2527 } 2528 break; 2529 case SVM_EXIT_MSR: 2530 if (!kvm_ghcb_rcx_is_valid(svm)) 2531 goto vmgexit_err; 2532 if (control->exit_info_1) { 2533 if (!kvm_ghcb_rax_is_valid(svm) || 2534 !kvm_ghcb_rdx_is_valid(svm)) 2535 goto vmgexit_err; 2536 } 2537 break; 2538 case SVM_EXIT_VMMCALL: 2539 if (!kvm_ghcb_rax_is_valid(svm) || 2540 !kvm_ghcb_cpl_is_valid(svm)) 2541 goto vmgexit_err; 2542 break; 2543 case SVM_EXIT_RDTSCP: 2544 break; 2545 case SVM_EXIT_WBINVD: 2546 break; 2547 case SVM_EXIT_MONITOR: 2548 if (!kvm_ghcb_rax_is_valid(svm) || 2549 !kvm_ghcb_rcx_is_valid(svm) || 2550 !kvm_ghcb_rdx_is_valid(svm)) 2551 goto vmgexit_err; 2552 break; 2553 case SVM_EXIT_MWAIT: 2554 if (!kvm_ghcb_rax_is_valid(svm) || 2555 !kvm_ghcb_rcx_is_valid(svm)) 2556 goto vmgexit_err; 2557 break; 2558 case SVM_VMGEXIT_MMIO_READ: 2559 case SVM_VMGEXIT_MMIO_WRITE: 2560 if (!kvm_ghcb_sw_scratch_is_valid(svm)) 2561 goto vmgexit_err; 2562 break; 2563 case SVM_VMGEXIT_NMI_COMPLETE: 2564 case SVM_VMGEXIT_AP_HLT_LOOP: 2565 case SVM_VMGEXIT_AP_JUMP_TABLE: 2566 case SVM_VMGEXIT_UNSUPPORTED_EVENT: 2567 break; 2568 default: 2569 reason = GHCB_ERR_INVALID_EVENT; 2570 goto vmgexit_err; 2571 } 2572 2573 return 0; 2574 2575vmgexit_err: 2576 if (reason == GHCB_ERR_INVALID_USAGE) { 2577 vcpu_unimpl(vcpu, "vmgexit: ghcb usage %#x is not valid\n", 2578 svm->sev_es.ghcb->ghcb_usage); 2579 } else if (reason == GHCB_ERR_INVALID_EVENT) { 2580 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx is not valid\n", 2581 exit_code); 2582 } else { 2583 vcpu_unimpl(vcpu, "vmgexit: exit code %#llx input is not valid\n", 2584 exit_code); 2585 dump_ghcb(svm); 2586 } 2587 2588 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2); 2589 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, reason); 2590 2591 /* Resume the guest to "return" the error code. */ 2592 return 1; 2593} 2594 2595void sev_es_unmap_ghcb(struct vcpu_svm *svm) 2596{ 2597 if (!svm->sev_es.ghcb) 2598 return; 2599 2600 if (svm->sev_es.ghcb_sa_free) { 2601 /* 2602 * The scratch area lives outside the GHCB, so there is a 2603 * buffer that, depending on the operation performed, may 2604 * need to be synced, then freed. 2605 */ 2606 if (svm->sev_es.ghcb_sa_sync) { 2607 kvm_write_guest(svm->vcpu.kvm, 2608 svm->sev_es.sw_scratch, 2609 svm->sev_es.ghcb_sa, 2610 svm->sev_es.ghcb_sa_len); 2611 svm->sev_es.ghcb_sa_sync = false; 2612 } 2613 2614 kvfree(svm->sev_es.ghcb_sa); 2615 svm->sev_es.ghcb_sa = NULL; 2616 svm->sev_es.ghcb_sa_free = false; 2617 } 2618 2619 trace_kvm_vmgexit_exit(svm->vcpu.vcpu_id, svm->sev_es.ghcb); 2620 2621 sev_es_sync_to_ghcb(svm); 2622 2623 kvm_vcpu_unmap(&svm->vcpu, &svm->sev_es.ghcb_map, true); 2624 svm->sev_es.ghcb = NULL; 2625} 2626 2627void pre_sev_run(struct vcpu_svm *svm, int cpu) 2628{ 2629 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); 2630 int asid = sev_get_asid(svm->vcpu.kvm); 2631 2632 /* Assign the asid allocated with this SEV guest */ 2633 svm->asid = asid; 2634 2635 /* 2636 * Flush guest TLB: 2637 * 2638 * 1) when different VMCB for the same ASID is to be run on the same host CPU. 2639 * 2) or this VMCB was executed on different host CPU in previous VMRUNs. 2640 */ 2641 if (sd->sev_vmcbs[asid] == svm->vmcb && 2642 svm->vcpu.arch.last_vmentry_cpu == cpu) 2643 return; 2644 2645 sd->sev_vmcbs[asid] = svm->vmcb; 2646 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; 2647 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); 2648} 2649 2650#define GHCB_SCRATCH_AREA_LIMIT (16ULL * PAGE_SIZE) 2651static int setup_vmgexit_scratch(struct vcpu_svm *svm, bool sync, u64 len) 2652{ 2653 struct vmcb_control_area *control = &svm->vmcb->control; 2654 u64 ghcb_scratch_beg, ghcb_scratch_end; 2655 u64 scratch_gpa_beg, scratch_gpa_end; 2656 void *scratch_va; 2657 2658 scratch_gpa_beg = svm->sev_es.sw_scratch; 2659 if (!scratch_gpa_beg) { 2660 pr_err("vmgexit: scratch gpa not provided\n"); 2661 goto e_scratch; 2662 } 2663 2664 scratch_gpa_end = scratch_gpa_beg + len; 2665 if (scratch_gpa_end < scratch_gpa_beg) { 2666 pr_err("vmgexit: scratch length (%#llx) not valid for scratch address (%#llx)\n", 2667 len, scratch_gpa_beg); 2668 goto e_scratch; 2669 } 2670 2671 if ((scratch_gpa_beg & PAGE_MASK) == control->ghcb_gpa) { 2672 /* Scratch area begins within GHCB */ 2673 ghcb_scratch_beg = control->ghcb_gpa + 2674 offsetof(struct ghcb, shared_buffer); 2675 ghcb_scratch_end = control->ghcb_gpa + 2676 offsetof(struct ghcb, reserved_0xff0); 2677 2678 /* 2679 * If the scratch area begins within the GHCB, it must be 2680 * completely contained in the GHCB shared buffer area. 2681 */ 2682 if (scratch_gpa_beg < ghcb_scratch_beg || 2683 scratch_gpa_end > ghcb_scratch_end) { 2684 pr_err("vmgexit: scratch area is outside of GHCB shared buffer area (%#llx - %#llx)\n", 2685 scratch_gpa_beg, scratch_gpa_end); 2686 goto e_scratch; 2687 } 2688 2689 scratch_va = (void *)svm->sev_es.ghcb; 2690 scratch_va += (scratch_gpa_beg - control->ghcb_gpa); 2691 } else { 2692 /* 2693 * The guest memory must be read into a kernel buffer, so 2694 * limit the size 2695 */ 2696 if (len > GHCB_SCRATCH_AREA_LIMIT) { 2697 pr_err("vmgexit: scratch area exceeds KVM limits (%#llx requested, %#llx limit)\n", 2698 len, GHCB_SCRATCH_AREA_LIMIT); 2699 goto e_scratch; 2700 } 2701 scratch_va = kvzalloc(len, GFP_KERNEL_ACCOUNT); 2702 if (!scratch_va) 2703 return -ENOMEM; 2704 2705 if (kvm_read_guest(svm->vcpu.kvm, scratch_gpa_beg, scratch_va, len)) { 2706 /* Unable to copy scratch area from guest */ 2707 pr_err("vmgexit: kvm_read_guest for scratch area failed\n"); 2708 2709 kvfree(scratch_va); 2710 return -EFAULT; 2711 } 2712 2713 /* 2714 * The scratch area is outside the GHCB. The operation will 2715 * dictate whether the buffer needs to be synced before running 2716 * the vCPU next time (i.e. a read was requested so the data 2717 * must be written back to the guest memory). 2718 */ 2719 svm->sev_es.ghcb_sa_sync = sync; 2720 svm->sev_es.ghcb_sa_free = true; 2721 } 2722 2723 svm->sev_es.ghcb_sa = scratch_va; 2724 svm->sev_es.ghcb_sa_len = len; 2725 2726 return 0; 2727 2728e_scratch: 2729 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2); 2730 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, GHCB_ERR_INVALID_SCRATCH_AREA); 2731 2732 return 1; 2733} 2734 2735static void set_ghcb_msr_bits(struct vcpu_svm *svm, u64 value, u64 mask, 2736 unsigned int pos) 2737{ 2738 svm->vmcb->control.ghcb_gpa &= ~(mask << pos); 2739 svm->vmcb->control.ghcb_gpa |= (value & mask) << pos; 2740} 2741 2742static u64 get_ghcb_msr_bits(struct vcpu_svm *svm, u64 mask, unsigned int pos) 2743{ 2744 return (svm->vmcb->control.ghcb_gpa >> pos) & mask; 2745} 2746 2747static void set_ghcb_msr(struct vcpu_svm *svm, u64 value) 2748{ 2749 svm->vmcb->control.ghcb_gpa = value; 2750} 2751 2752static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm) 2753{ 2754 struct vmcb_control_area *control = &svm->vmcb->control; 2755 struct kvm_vcpu *vcpu = &svm->vcpu; 2756 u64 ghcb_info; 2757 int ret = 1; 2758 2759 ghcb_info = control->ghcb_gpa & GHCB_MSR_INFO_MASK; 2760 2761 trace_kvm_vmgexit_msr_protocol_enter(svm->vcpu.vcpu_id, 2762 control->ghcb_gpa); 2763 2764 switch (ghcb_info) { 2765 case GHCB_MSR_SEV_INFO_REQ: 2766 set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX, 2767 GHCB_VERSION_MIN, 2768 sev_enc_bit)); 2769 break; 2770 case GHCB_MSR_CPUID_REQ: { 2771 u64 cpuid_fn, cpuid_reg, cpuid_value; 2772 2773 cpuid_fn = get_ghcb_msr_bits(svm, 2774 GHCB_MSR_CPUID_FUNC_MASK, 2775 GHCB_MSR_CPUID_FUNC_POS); 2776 2777 /* Initialize the registers needed by the CPUID intercept */ 2778 vcpu->arch.regs[VCPU_REGS_RAX] = cpuid_fn; 2779 vcpu->arch.regs[VCPU_REGS_RCX] = 0; 2780 2781 ret = svm_invoke_exit_handler(vcpu, SVM_EXIT_CPUID); 2782 if (!ret) { 2783 /* Error, keep GHCB MSR value as-is */ 2784 break; 2785 } 2786 2787 cpuid_reg = get_ghcb_msr_bits(svm, 2788 GHCB_MSR_CPUID_REG_MASK, 2789 GHCB_MSR_CPUID_REG_POS); 2790 if (cpuid_reg == 0) 2791 cpuid_value = vcpu->arch.regs[VCPU_REGS_RAX]; 2792 else if (cpuid_reg == 1) 2793 cpuid_value = vcpu->arch.regs[VCPU_REGS_RBX]; 2794 else if (cpuid_reg == 2) 2795 cpuid_value = vcpu->arch.regs[VCPU_REGS_RCX]; 2796 else 2797 cpuid_value = vcpu->arch.regs[VCPU_REGS_RDX]; 2798 2799 set_ghcb_msr_bits(svm, cpuid_value, 2800 GHCB_MSR_CPUID_VALUE_MASK, 2801 GHCB_MSR_CPUID_VALUE_POS); 2802 2803 set_ghcb_msr_bits(svm, GHCB_MSR_CPUID_RESP, 2804 GHCB_MSR_INFO_MASK, 2805 GHCB_MSR_INFO_POS); 2806 break; 2807 } 2808 case GHCB_MSR_TERM_REQ: { 2809 u64 reason_set, reason_code; 2810 2811 reason_set = get_ghcb_msr_bits(svm, 2812 GHCB_MSR_TERM_REASON_SET_MASK, 2813 GHCB_MSR_TERM_REASON_SET_POS); 2814 reason_code = get_ghcb_msr_bits(svm, 2815 GHCB_MSR_TERM_REASON_MASK, 2816 GHCB_MSR_TERM_REASON_POS); 2817 pr_info("SEV-ES guest requested termination: %#llx:%#llx\n", 2818 reason_set, reason_code); 2819 2820 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 2821 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM; 2822 vcpu->run->system_event.ndata = 1; 2823 vcpu->run->system_event.data[0] = control->ghcb_gpa; 2824 2825 return 0; 2826 } 2827 default: 2828 /* Error, keep GHCB MSR value as-is */ 2829 break; 2830 } 2831 2832 trace_kvm_vmgexit_msr_protocol_exit(svm->vcpu.vcpu_id, 2833 control->ghcb_gpa, ret); 2834 2835 return ret; 2836} 2837 2838int sev_handle_vmgexit(struct kvm_vcpu *vcpu) 2839{ 2840 struct vcpu_svm *svm = to_svm(vcpu); 2841 struct vmcb_control_area *control = &svm->vmcb->control; 2842 u64 ghcb_gpa, exit_code; 2843 int ret; 2844 2845 /* Validate the GHCB */ 2846 ghcb_gpa = control->ghcb_gpa; 2847 if (ghcb_gpa & GHCB_MSR_INFO_MASK) 2848 return sev_handle_vmgexit_msr_protocol(svm); 2849 2850 if (!ghcb_gpa) { 2851 vcpu_unimpl(vcpu, "vmgexit: GHCB gpa is not set\n"); 2852 2853 /* Without a GHCB, just return right back to the guest */ 2854 return 1; 2855 } 2856 2857 if (kvm_vcpu_map(vcpu, ghcb_gpa >> PAGE_SHIFT, &svm->sev_es.ghcb_map)) { 2858 /* Unable to map GHCB from guest */ 2859 vcpu_unimpl(vcpu, "vmgexit: error mapping GHCB [%#llx] from guest\n", 2860 ghcb_gpa); 2861 2862 /* Without a GHCB, just return right back to the guest */ 2863 return 1; 2864 } 2865 2866 svm->sev_es.ghcb = svm->sev_es.ghcb_map.hva; 2867 2868 trace_kvm_vmgexit_enter(vcpu->vcpu_id, svm->sev_es.ghcb); 2869 2870 sev_es_sync_from_ghcb(svm); 2871 ret = sev_es_validate_vmgexit(svm); 2872 if (ret) 2873 return ret; 2874 2875 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 0); 2876 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 0); 2877 2878 exit_code = kvm_ghcb_get_sw_exit_code(control); 2879 switch (exit_code) { 2880 case SVM_VMGEXIT_MMIO_READ: 2881 ret = setup_vmgexit_scratch(svm, true, control->exit_info_2); 2882 if (ret) 2883 break; 2884 2885 ret = kvm_sev_es_mmio_read(vcpu, 2886 control->exit_info_1, 2887 control->exit_info_2, 2888 svm->sev_es.ghcb_sa); 2889 break; 2890 case SVM_VMGEXIT_MMIO_WRITE: 2891 ret = setup_vmgexit_scratch(svm, false, control->exit_info_2); 2892 if (ret) 2893 break; 2894 2895 ret = kvm_sev_es_mmio_write(vcpu, 2896 control->exit_info_1, 2897 control->exit_info_2, 2898 svm->sev_es.ghcb_sa); 2899 break; 2900 case SVM_VMGEXIT_NMI_COMPLETE: 2901 ++vcpu->stat.nmi_window_exits; 2902 svm->nmi_masked = false; 2903 kvm_make_request(KVM_REQ_EVENT, vcpu); 2904 ret = 1; 2905 break; 2906 case SVM_VMGEXIT_AP_HLT_LOOP: 2907 ret = kvm_emulate_ap_reset_hold(vcpu); 2908 break; 2909 case SVM_VMGEXIT_AP_JUMP_TABLE: { 2910 struct kvm_sev_info *sev = &to_kvm_svm(vcpu->kvm)->sev_info; 2911 2912 switch (control->exit_info_1) { 2913 case 0: 2914 /* Set AP jump table address */ 2915 sev->ap_jump_table = control->exit_info_2; 2916 break; 2917 case 1: 2918 /* Get AP jump table address */ 2919 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, sev->ap_jump_table); 2920 break; 2921 default: 2922 pr_err("svm: vmgexit: unsupported AP jump table request - exit_info_1=%#llx\n", 2923 control->exit_info_1); 2924 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 2); 2925 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, GHCB_ERR_INVALID_INPUT); 2926 } 2927 2928 ret = 1; 2929 break; 2930 } 2931 case SVM_VMGEXIT_UNSUPPORTED_EVENT: 2932 vcpu_unimpl(vcpu, 2933 "vmgexit: unsupported event - exit_info_1=%#llx, exit_info_2=%#llx\n", 2934 control->exit_info_1, control->exit_info_2); 2935 ret = -EINVAL; 2936 break; 2937 default: 2938 ret = svm_invoke_exit_handler(vcpu, exit_code); 2939 } 2940 2941 return ret; 2942} 2943 2944int sev_es_string_io(struct vcpu_svm *svm, int size, unsigned int port, int in) 2945{ 2946 int count; 2947 int bytes; 2948 int r; 2949 2950 if (svm->vmcb->control.exit_info_2 > INT_MAX) 2951 return -EINVAL; 2952 2953 count = svm->vmcb->control.exit_info_2; 2954 if (unlikely(check_mul_overflow(count, size, &bytes))) 2955 return -EINVAL; 2956 2957 r = setup_vmgexit_scratch(svm, in, bytes); 2958 if (r) 2959 return r; 2960 2961 return kvm_sev_es_string_io(&svm->vcpu, size, port, svm->sev_es.ghcb_sa, 2962 count, in); 2963} 2964 2965static void sev_es_vcpu_after_set_cpuid(struct vcpu_svm *svm) 2966{ 2967 struct kvm_vcpu *vcpu = &svm->vcpu; 2968 2969 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX)) { 2970 bool v_tsc_aux = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) || 2971 guest_cpuid_has(vcpu, X86_FEATURE_RDPID); 2972 2973 set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, v_tsc_aux, v_tsc_aux); 2974 } 2975} 2976 2977void sev_vcpu_after_set_cpuid(struct vcpu_svm *svm) 2978{ 2979 struct kvm_vcpu *vcpu = &svm->vcpu; 2980 struct kvm_cpuid_entry2 *best; 2981 2982 /* For sev guests, the memory encryption bit is not reserved in CR3. */ 2983 best = kvm_find_cpuid_entry(vcpu, 0x8000001F); 2984 if (best) 2985 vcpu->arch.reserved_gpa_bits &= ~(1UL << (best->ebx & 0x3f)); 2986 2987 if (sev_es_guest(svm->vcpu.kvm)) 2988 sev_es_vcpu_after_set_cpuid(svm); 2989} 2990 2991static void sev_es_init_vmcb(struct vcpu_svm *svm) 2992{ 2993 struct vmcb *vmcb = svm->vmcb01.ptr; 2994 struct kvm_vcpu *vcpu = &svm->vcpu; 2995 2996 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ES_ENABLE; 2997 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; 2998 2999 /* 3000 * An SEV-ES guest requires a VMSA area that is a separate from the 3001 * VMCB page. Do not include the encryption mask on the VMSA physical 3002 * address since hardware will access it using the guest key. Note, 3003 * the VMSA will be NULL if this vCPU is the destination for intrahost 3004 * migration, and will be copied later. 3005 */ 3006 if (svm->sev_es.vmsa) 3007 svm->vmcb->control.vmsa_pa = __pa(svm->sev_es.vmsa); 3008 3009 /* Can't intercept CR register access, HV can't modify CR registers */ 3010 svm_clr_intercept(svm, INTERCEPT_CR0_READ); 3011 svm_clr_intercept(svm, INTERCEPT_CR4_READ); 3012 svm_clr_intercept(svm, INTERCEPT_CR8_READ); 3013 svm_clr_intercept(svm, INTERCEPT_CR0_WRITE); 3014 svm_clr_intercept(svm, INTERCEPT_CR4_WRITE); 3015 svm_clr_intercept(svm, INTERCEPT_CR8_WRITE); 3016 3017 svm_clr_intercept(svm, INTERCEPT_SELECTIVE_CR0); 3018 3019 /* Track EFER/CR register changes */ 3020 svm_set_intercept(svm, TRAP_EFER_WRITE); 3021 svm_set_intercept(svm, TRAP_CR0_WRITE); 3022 svm_set_intercept(svm, TRAP_CR4_WRITE); 3023 svm_set_intercept(svm, TRAP_CR8_WRITE); 3024 3025 vmcb->control.intercepts[INTERCEPT_DR] = 0; 3026 if (!sev_es_debug_swap_enabled) { 3027 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ); 3028 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE); 3029 recalc_intercepts(svm); 3030 } else { 3031 /* 3032 * Disable #DB intercept iff DebugSwap is enabled. KVM doesn't 3033 * allow debugging SEV-ES guests, and enables DebugSwap iff 3034 * NO_NESTED_DATA_BP is supported, so there's no reason to 3035 * intercept #DB when DebugSwap is enabled. For simplicity 3036 * with respect to guest debug, intercept #DB for other VMs 3037 * even if NO_NESTED_DATA_BP is supported, i.e. even if the 3038 * guest can't DoS the CPU with infinite #DB vectoring. 3039 */ 3040 clr_exception_intercept(svm, DB_VECTOR); 3041 } 3042 3043 /* Can't intercept XSETBV, HV can't modify XCR0 directly */ 3044 svm_clr_intercept(svm, INTERCEPT_XSETBV); 3045 3046 /* Clear intercepts on selected MSRs */ 3047 set_msr_interception(vcpu, svm->msrpm, MSR_EFER, 1, 1); 3048 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_CR_PAT, 1, 1); 3049 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); 3050 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); 3051 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); 3052 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); 3053} 3054 3055void sev_init_vmcb(struct vcpu_svm *svm) 3056{ 3057 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE; 3058 clr_exception_intercept(svm, UD_VECTOR); 3059 3060 /* 3061 * Don't intercept #GP for SEV guests, e.g. for the VMware backdoor, as 3062 * KVM can't decrypt guest memory to decode the faulting instruction. 3063 */ 3064 clr_exception_intercept(svm, GP_VECTOR); 3065 3066 if (sev_es_guest(svm->vcpu.kvm)) 3067 sev_es_init_vmcb(svm); 3068} 3069 3070void sev_es_vcpu_reset(struct vcpu_svm *svm) 3071{ 3072 /* 3073 * Set the GHCB MSR value as per the GHCB specification when emulating 3074 * vCPU RESET for an SEV-ES guest. 3075 */ 3076 set_ghcb_msr(svm, GHCB_MSR_SEV_INFO(GHCB_VERSION_MAX, 3077 GHCB_VERSION_MIN, 3078 sev_enc_bit)); 3079} 3080 3081void sev_es_prepare_switch_to_guest(struct sev_es_save_area *hostsa) 3082{ 3083 /* 3084 * All host state for SEV-ES guests is categorized into three swap types 3085 * based on how it is handled by hardware during a world switch: 3086 * 3087 * A: VMRUN: Host state saved in host save area 3088 * VMEXIT: Host state loaded from host save area 3089 * 3090 * B: VMRUN: Host state _NOT_ saved in host save area 3091 * VMEXIT: Host state loaded from host save area 3092 * 3093 * C: VMRUN: Host state _NOT_ saved in host save area 3094 * VMEXIT: Host state initialized to default(reset) values 3095 * 3096 * Manually save type-B state, i.e. state that is loaded by VMEXIT but 3097 * isn't saved by VMRUN, that isn't already saved by VMSAVE (performed 3098 * by common SVM code). 3099 */ 3100 hostsa->xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 3101 hostsa->pkru = read_pkru(); 3102 hostsa->xss = host_xss; 3103 3104 /* 3105 * If DebugSwap is enabled, debug registers are loaded but NOT saved by 3106 * the CPU (Type-B). If DebugSwap is disabled/unsupported, the CPU both 3107 * saves and loads debug registers (Type-A). 3108 */ 3109 if (sev_es_debug_swap_enabled) { 3110 hostsa->dr0 = native_get_debugreg(0); 3111 hostsa->dr1 = native_get_debugreg(1); 3112 hostsa->dr2 = native_get_debugreg(2); 3113 hostsa->dr3 = native_get_debugreg(3); 3114 hostsa->dr0_addr_mask = amd_get_dr_addr_mask(0); 3115 hostsa->dr1_addr_mask = amd_get_dr_addr_mask(1); 3116 hostsa->dr2_addr_mask = amd_get_dr_addr_mask(2); 3117 hostsa->dr3_addr_mask = amd_get_dr_addr_mask(3); 3118 } 3119} 3120 3121void sev_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 3122{ 3123 struct vcpu_svm *svm = to_svm(vcpu); 3124 3125 /* First SIPI: Use the values as initially set by the VMM */ 3126 if (!svm->sev_es.received_first_sipi) { 3127 svm->sev_es.received_first_sipi = true; 3128 return; 3129 } 3130 3131 /* 3132 * Subsequent SIPI: Return from an AP Reset Hold VMGEXIT, where 3133 * the guest will set the CS and RIP. Set SW_EXIT_INFO_2 to a 3134 * non-zero value. 3135 */ 3136 if (!svm->sev_es.ghcb) 3137 return; 3138 3139 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, 1); 3140} 3141