1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15#define pr_fmt(fmt) "SVM: " fmt 16 17#include <linux/kvm_types.h> 18#include <linux/hashtable.h> 19#include <linux/amd-iommu.h> 20#include <linux/kvm_host.h> 21 22#include <asm/irq_remapping.h> 23 24#include "trace.h" 25#include "lapic.h" 26#include "x86.h" 27#include "irq.h" 28#include "svm.h" 29 30/* enable / disable AVIC */ 31int avic; 32#ifdef CONFIG_X86_LOCAL_APIC 33module_param(avic, int, S_IRUGO); 34#endif 35 36#define SVM_AVIC_DOORBELL 0xc001011b 37 38#define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) 39 40/* 41 * 0xff is broadcast, so the max index allowed for physical APIC ID 42 * table is 0xfe. APIC IDs above 0xff are reserved. 43 */ 44#define AVIC_MAX_PHYSICAL_ID_COUNT 255 45 46#define AVIC_UNACCEL_ACCESS_WRITE_MASK 1 47#define AVIC_UNACCEL_ACCESS_OFFSET_MASK 0xFF0 48#define AVIC_UNACCEL_ACCESS_VECTOR_MASK 0xFFFFFFFF 49 50/* AVIC GATAG is encoded using VM and VCPU IDs */ 51#define AVIC_VCPU_ID_BITS 8 52#define AVIC_VCPU_ID_MASK ((1 << AVIC_VCPU_ID_BITS) - 1) 53 54#define AVIC_VM_ID_BITS 24 55#define AVIC_VM_ID_NR (1 << AVIC_VM_ID_BITS) 56#define AVIC_VM_ID_MASK ((1 << AVIC_VM_ID_BITS) - 1) 57 58#define AVIC_GATAG(x, y) (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \ 59 (y & AVIC_VCPU_ID_MASK)) 60#define AVIC_GATAG_TO_VMID(x) ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK) 61#define AVIC_GATAG_TO_VCPUID(x) (x & AVIC_VCPU_ID_MASK) 62 63/* Note: 64 * This hash table is used to map VM_ID to a struct kvm_svm, 65 * when handling AMD IOMMU GALOG notification to schedule in 66 * a particular vCPU. 67 */ 68#define SVM_VM_DATA_HASH_BITS 8 69static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS); 70static u32 next_vm_id = 0; 71static bool next_vm_id_wrapped = 0; 72static DEFINE_SPINLOCK(svm_vm_data_hash_lock); 73 74/* 75 * This is a wrapper of struct amd_iommu_ir_data. 76 */ 77struct amd_svm_iommu_ir { 78 struct list_head node; /* Used by SVM for per-vcpu ir_list */ 79 void *data; /* Storing pointer to struct amd_ir_data */ 80}; 81 82enum avic_ipi_failure_cause { 83 AVIC_IPI_FAILURE_INVALID_INT_TYPE, 84 AVIC_IPI_FAILURE_TARGET_NOT_RUNNING, 85 AVIC_IPI_FAILURE_INVALID_TARGET, 86 AVIC_IPI_FAILURE_INVALID_BACKING_PAGE, 87}; 88 89/* Note: 90 * This function is called from IOMMU driver to notify 91 * SVM to schedule in a particular vCPU of a particular VM. 92 */ 93int avic_ga_log_notifier(u32 ga_tag) 94{ 95 unsigned long flags; 96 struct kvm_svm *kvm_svm; 97 struct kvm_vcpu *vcpu = NULL; 98 u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag); 99 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag); 100 101 pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id); 102 trace_kvm_avic_ga_log(vm_id, vcpu_id); 103 104 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 105 hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) { 106 if (kvm_svm->avic_vm_id != vm_id) 107 continue; 108 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id); 109 break; 110 } 111 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 112 113 /* Note: 114 * At this point, the IOMMU should have already set the pending 115 * bit in the vAPIC backing page. So, we just need to schedule 116 * in the vcpu. 117 */ 118 if (vcpu) 119 kvm_vcpu_wake_up(vcpu); 120 121 return 0; 122} 123 124void avic_vm_destroy(struct kvm *kvm) 125{ 126 unsigned long flags; 127 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 128 129 if (!avic) 130 return; 131 132 if (kvm_svm->avic_logical_id_table_page) 133 __free_page(kvm_svm->avic_logical_id_table_page); 134 if (kvm_svm->avic_physical_id_table_page) 135 __free_page(kvm_svm->avic_physical_id_table_page); 136 137 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 138 hash_del(&kvm_svm->hnode); 139 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 140} 141 142int avic_vm_init(struct kvm *kvm) 143{ 144 unsigned long flags; 145 int err = -ENOMEM; 146 struct kvm_svm *kvm_svm = to_kvm_svm(kvm); 147 struct kvm_svm *k2; 148 struct page *p_page; 149 struct page *l_page; 150 u32 vm_id; 151 152 if (!avic) 153 return 0; 154 155 /* Allocating physical APIC ID table (4KB) */ 156 p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 157 if (!p_page) 158 goto free_avic; 159 160 kvm_svm->avic_physical_id_table_page = p_page; 161 162 /* Allocating logical APIC ID table (4KB) */ 163 l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 164 if (!l_page) 165 goto free_avic; 166 167 kvm_svm->avic_logical_id_table_page = l_page; 168 169 spin_lock_irqsave(&svm_vm_data_hash_lock, flags); 170 again: 171 vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; 172 if (vm_id == 0) { /* id is 1-based, zero is not okay */ 173 next_vm_id_wrapped = 1; 174 goto again; 175 } 176 /* Is it still in use? Only possible if wrapped at least once */ 177 if (next_vm_id_wrapped) { 178 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) { 179 if (k2->avic_vm_id == vm_id) 180 goto again; 181 } 182 } 183 kvm_svm->avic_vm_id = vm_id; 184 hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); 185 spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); 186 187 return 0; 188 189free_avic: 190 avic_vm_destroy(kvm); 191 return err; 192} 193 194void avic_init_vmcb(struct vcpu_svm *svm) 195{ 196 struct vmcb *vmcb = svm->vmcb; 197 struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); 198 phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); 199 phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); 200 phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page)); 201 202 vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK; 203 vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK; 204 vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK; 205 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT; 206 if (kvm_apicv_activated(svm->vcpu.kvm)) 207 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 208 else 209 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 210} 211 212static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, 213 unsigned int index) 214{ 215 u64 *avic_physical_id_table; 216 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 217 218 if (index >= AVIC_MAX_PHYSICAL_ID_COUNT) 219 return NULL; 220 221 avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page); 222 223 return &avic_physical_id_table[index]; 224} 225 226/** 227 * Note: 228 * AVIC hardware walks the nested page table to check permissions, 229 * but does not use the SPA address specified in the leaf page 230 * table entry since it uses address in the AVIC_BACKING_PAGE pointer 231 * field of the VMCB. Therefore, we set up the 232 * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here. 233 */ 234static int avic_update_access_page(struct kvm *kvm, bool activate) 235{ 236 int ret = 0; 237 238 mutex_lock(&kvm->slots_lock); 239 /* 240 * During kvm_destroy_vm(), kvm_pit_set_reinject() could trigger 241 * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 242 * memory region. So, we need to ensure that kvm->mm == current->mm. 243 */ 244 if ((kvm->arch.apic_access_page_done == activate) || 245 (kvm->mm != current->mm)) 246 goto out; 247 248 ret = __x86_set_memory_region(kvm, 249 APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 250 APIC_DEFAULT_PHYS_BASE, 251 activate ? PAGE_SIZE : 0); 252 if (ret) 253 goto out; 254 255 kvm->arch.apic_access_page_done = activate; 256out: 257 mutex_unlock(&kvm->slots_lock); 258 return ret; 259} 260 261static int avic_init_backing_page(struct kvm_vcpu *vcpu) 262{ 263 u64 *entry, new_entry; 264 int id = vcpu->vcpu_id; 265 struct vcpu_svm *svm = to_svm(vcpu); 266 267 if (id >= AVIC_MAX_PHYSICAL_ID_COUNT) 268 return -EINVAL; 269 270 if (!svm->vcpu.arch.apic->regs) 271 return -EINVAL; 272 273 if (kvm_apicv_activated(vcpu->kvm)) { 274 int ret; 275 276 ret = avic_update_access_page(vcpu->kvm, true); 277 if (ret) 278 return ret; 279 } 280 281 svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs); 282 283 /* Setting AVIC backing page address in the phy APIC ID table */ 284 entry = avic_get_physical_id_entry(vcpu, id); 285 if (!entry) 286 return -EINVAL; 287 288 new_entry = __sme_set((page_to_phys(svm->avic_backing_page) & 289 AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) | 290 AVIC_PHYSICAL_ID_ENTRY_VALID_MASK); 291 WRITE_ONCE(*entry, new_entry); 292 293 svm->avic_physical_id_cache = entry; 294 295 return 0; 296} 297 298int avic_incomplete_ipi_interception(struct vcpu_svm *svm) 299{ 300 u32 icrh = svm->vmcb->control.exit_info_1 >> 32; 301 u32 icrl = svm->vmcb->control.exit_info_1; 302 u32 id = svm->vmcb->control.exit_info_2 >> 32; 303 u32 index = svm->vmcb->control.exit_info_2 & 0xFF; 304 struct kvm_lapic *apic = svm->vcpu.arch.apic; 305 306 trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index); 307 308 switch (id) { 309 case AVIC_IPI_FAILURE_INVALID_INT_TYPE: 310 /* 311 * AVIC hardware handles the generation of 312 * IPIs when the specified Message Type is Fixed 313 * (also known as fixed delivery mode) and 314 * the Trigger Mode is edge-triggered. The hardware 315 * also supports self and broadcast delivery modes 316 * specified via the Destination Shorthand(DSH) 317 * field of the ICRL. Logical and physical APIC ID 318 * formats are supported. All other IPI types cause 319 * a #VMEXIT, which needs to emulated. 320 */ 321 kvm_lapic_reg_write(apic, APIC_ICR2, icrh); 322 kvm_lapic_reg_write(apic, APIC_ICR, icrl); 323 break; 324 case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: { 325 int i; 326 struct kvm_vcpu *vcpu; 327 struct kvm *kvm = svm->vcpu.kvm; 328 struct kvm_lapic *apic = svm->vcpu.arch.apic; 329 330 /* 331 * At this point, we expect that the AVIC HW has already 332 * set the appropriate IRR bits on the valid target 333 * vcpus. So, we just need to kick the appropriate vcpu. 334 */ 335 kvm_for_each_vcpu(i, vcpu, kvm) { 336 bool m = kvm_apic_match_dest(vcpu, apic, 337 icrl & APIC_SHORT_MASK, 338 GET_APIC_DEST_FIELD(icrh), 339 icrl & APIC_DEST_MASK); 340 341 if (m && !avic_vcpu_is_running(vcpu)) 342 kvm_vcpu_wake_up(vcpu); 343 } 344 break; 345 } 346 case AVIC_IPI_FAILURE_INVALID_TARGET: 347 break; 348 case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE: 349 WARN_ONCE(1, "Invalid backing page\n"); 350 break; 351 default: 352 pr_err("Unknown IPI interception\n"); 353 } 354 355 return 1; 356} 357 358static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) 359{ 360 struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); 361 int index; 362 u32 *logical_apic_id_table; 363 int dlid = GET_APIC_LOGICAL_ID(ldr); 364 365 if (!dlid) 366 return NULL; 367 368 if (flat) { /* flat */ 369 index = ffs(dlid) - 1; 370 if (index > 7) 371 return NULL; 372 } else { /* cluster */ 373 int cluster = (dlid & 0xf0) >> 4; 374 int apic = ffs(dlid & 0x0f) - 1; 375 376 if ((apic < 0) || (apic > 7) || 377 (cluster >= 0xf)) 378 return NULL; 379 index = (cluster << 2) + apic; 380 } 381 382 logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page); 383 384 return &logical_apic_id_table[index]; 385} 386 387static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr) 388{ 389 bool flat; 390 u32 *entry, new_entry; 391 392 flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT; 393 entry = avic_get_logical_id_entry(vcpu, ldr, flat); 394 if (!entry) 395 return -EINVAL; 396 397 new_entry = READ_ONCE(*entry); 398 new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK; 399 new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK); 400 new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK; 401 WRITE_ONCE(*entry, new_entry); 402 403 return 0; 404} 405 406static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu) 407{ 408 struct vcpu_svm *svm = to_svm(vcpu); 409 bool flat = svm->dfr_reg == APIC_DFR_FLAT; 410 u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat); 411 412 if (entry) 413 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry); 414} 415 416static int avic_handle_ldr_update(struct kvm_vcpu *vcpu) 417{ 418 int ret = 0; 419 struct vcpu_svm *svm = to_svm(vcpu); 420 u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR); 421 u32 id = kvm_xapic_id(vcpu->arch.apic); 422 423 if (ldr == svm->ldr_reg) 424 return 0; 425 426 avic_invalidate_logical_id_entry(vcpu); 427 428 if (ldr) 429 ret = avic_ldr_write(vcpu, id, ldr); 430 431 if (!ret) 432 svm->ldr_reg = ldr; 433 434 return ret; 435} 436 437static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu) 438{ 439 u64 *old, *new; 440 struct vcpu_svm *svm = to_svm(vcpu); 441 u32 id = kvm_xapic_id(vcpu->arch.apic); 442 443 if (vcpu->vcpu_id == id) 444 return 0; 445 446 old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id); 447 new = avic_get_physical_id_entry(vcpu, id); 448 if (!new || !old) 449 return 1; 450 451 /* We need to move physical_id_entry to new offset */ 452 *new = *old; 453 *old = 0ULL; 454 to_svm(vcpu)->avic_physical_id_cache = new; 455 456 /* 457 * Also update the guest physical APIC ID in the logical 458 * APIC ID table entry if already setup the LDR. 459 */ 460 if (svm->ldr_reg) 461 avic_handle_ldr_update(vcpu); 462 463 return 0; 464} 465 466static void avic_handle_dfr_update(struct kvm_vcpu *vcpu) 467{ 468 struct vcpu_svm *svm = to_svm(vcpu); 469 u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR); 470 471 if (svm->dfr_reg == dfr) 472 return; 473 474 avic_invalidate_logical_id_entry(vcpu); 475 svm->dfr_reg = dfr; 476} 477 478static int avic_unaccel_trap_write(struct vcpu_svm *svm) 479{ 480 struct kvm_lapic *apic = svm->vcpu.arch.apic; 481 u32 offset = svm->vmcb->control.exit_info_1 & 482 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 483 484 switch (offset) { 485 case APIC_ID: 486 if (avic_handle_apic_id_update(&svm->vcpu)) 487 return 0; 488 break; 489 case APIC_LDR: 490 if (avic_handle_ldr_update(&svm->vcpu)) 491 return 0; 492 break; 493 case APIC_DFR: 494 avic_handle_dfr_update(&svm->vcpu); 495 break; 496 default: 497 break; 498 } 499 500 kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset)); 501 502 return 1; 503} 504 505static bool is_avic_unaccelerated_access_trap(u32 offset) 506{ 507 bool ret = false; 508 509 switch (offset) { 510 case APIC_ID: 511 case APIC_EOI: 512 case APIC_RRR: 513 case APIC_LDR: 514 case APIC_DFR: 515 case APIC_SPIV: 516 case APIC_ESR: 517 case APIC_ICR: 518 case APIC_LVTT: 519 case APIC_LVTTHMR: 520 case APIC_LVTPC: 521 case APIC_LVT0: 522 case APIC_LVT1: 523 case APIC_LVTERR: 524 case APIC_TMICT: 525 case APIC_TDCR: 526 ret = true; 527 break; 528 default: 529 break; 530 } 531 return ret; 532} 533 534int avic_unaccelerated_access_interception(struct vcpu_svm *svm) 535{ 536 int ret = 0; 537 u32 offset = svm->vmcb->control.exit_info_1 & 538 AVIC_UNACCEL_ACCESS_OFFSET_MASK; 539 u32 vector = svm->vmcb->control.exit_info_2 & 540 AVIC_UNACCEL_ACCESS_VECTOR_MASK; 541 bool write = (svm->vmcb->control.exit_info_1 >> 32) & 542 AVIC_UNACCEL_ACCESS_WRITE_MASK; 543 bool trap = is_avic_unaccelerated_access_trap(offset); 544 545 trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset, 546 trap, write, vector); 547 if (trap) { 548 /* Handling Trap */ 549 WARN_ONCE(!write, "svm: Handling trap read.\n"); 550 ret = avic_unaccel_trap_write(svm); 551 } else { 552 /* Handling Fault */ 553 ret = kvm_emulate_instruction(&svm->vcpu, 0); 554 } 555 556 return ret; 557} 558 559int avic_init_vcpu(struct vcpu_svm *svm) 560{ 561 int ret; 562 struct kvm_vcpu *vcpu = &svm->vcpu; 563 564 if (!avic || !irqchip_in_kernel(vcpu->kvm)) 565 return 0; 566 567 ret = avic_init_backing_page(&svm->vcpu); 568 if (ret) 569 return ret; 570 571 INIT_LIST_HEAD(&svm->ir_list); 572 spin_lock_init(&svm->ir_list_lock); 573 svm->dfr_reg = APIC_DFR_FLAT; 574 575 return ret; 576} 577 578void avic_post_state_restore(struct kvm_vcpu *vcpu) 579{ 580 if (avic_handle_apic_id_update(vcpu) != 0) 581 return; 582 avic_handle_dfr_update(vcpu); 583 avic_handle_ldr_update(vcpu); 584} 585 586void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) 587{ 588 if (!avic || !lapic_in_kernel(vcpu)) 589 return; 590 591 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 592 kvm_request_apicv_update(vcpu->kvm, activate, 593 APICV_INHIBIT_REASON_IRQWIN); 594 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 595} 596 597void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu) 598{ 599 return; 600} 601 602void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr) 603{ 604} 605 606void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) 607{ 608} 609 610static int svm_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate) 611{ 612 int ret = 0; 613 unsigned long flags; 614 struct amd_svm_iommu_ir *ir; 615 struct vcpu_svm *svm = to_svm(vcpu); 616 617 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 618 return 0; 619 620 /* 621 * Here, we go through the per-vcpu ir_list to update all existing 622 * interrupt remapping table entry targeting this vcpu. 623 */ 624 spin_lock_irqsave(&svm->ir_list_lock, flags); 625 626 if (list_empty(&svm->ir_list)) 627 goto out; 628 629 list_for_each_entry(ir, &svm->ir_list, node) { 630 if (activate) 631 ret = amd_iommu_activate_guest_mode(ir->data); 632 else 633 ret = amd_iommu_deactivate_guest_mode(ir->data); 634 if (ret) 635 break; 636 } 637out: 638 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 639 return ret; 640} 641 642void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) 643{ 644 struct vcpu_svm *svm = to_svm(vcpu); 645 struct vmcb *vmcb = svm->vmcb; 646 bool activated = kvm_vcpu_apicv_active(vcpu); 647 648 if (!avic) 649 return; 650 651 if (activated) { 652 /** 653 * During AVIC temporary deactivation, guest could update 654 * APIC ID, DFR and LDR registers, which would not be trapped 655 * by avic_unaccelerated_access_interception(). In this case, 656 * we need to check and update the AVIC logical APIC ID table 657 * accordingly before re-activating. 658 */ 659 avic_post_state_restore(vcpu); 660 vmcb->control.int_ctl |= AVIC_ENABLE_MASK; 661 } else { 662 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK; 663 } 664 vmcb_mark_dirty(vmcb, VMCB_AVIC); 665 666 svm_set_pi_irte_mode(vcpu, activated); 667} 668 669void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) 670{ 671 return; 672} 673 674int svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec) 675{ 676 if (!vcpu->arch.apicv_active) 677 return -1; 678 679 kvm_lapic_set_irr(vec, vcpu->arch.apic); 680 smp_mb__after_atomic(); 681 682 if (avic_vcpu_is_running(vcpu)) { 683 int cpuid = vcpu->cpu; 684 685 if (cpuid != get_cpu()) 686 wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid)); 687 put_cpu(); 688 } else 689 kvm_vcpu_wake_up(vcpu); 690 691 return 0; 692} 693 694bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu) 695{ 696 return false; 697} 698 699static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 700{ 701 unsigned long flags; 702 struct amd_svm_iommu_ir *cur; 703 704 spin_lock_irqsave(&svm->ir_list_lock, flags); 705 list_for_each_entry(cur, &svm->ir_list, node) { 706 if (cur->data != pi->ir_data) 707 continue; 708 list_del(&cur->node); 709 kfree(cur); 710 break; 711 } 712 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 713} 714 715static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi) 716{ 717 int ret = 0; 718 unsigned long flags; 719 struct amd_svm_iommu_ir *ir; 720 721 /** 722 * In some cases, the existing irte is updaed and re-set, 723 * so we need to check here if it's already been * added 724 * to the ir_list. 725 */ 726 if (pi->ir_data && (pi->prev_ga_tag != 0)) { 727 struct kvm *kvm = svm->vcpu.kvm; 728 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag); 729 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id); 730 struct vcpu_svm *prev_svm; 731 732 if (!prev_vcpu) { 733 ret = -EINVAL; 734 goto out; 735 } 736 737 prev_svm = to_svm(prev_vcpu); 738 svm_ir_list_del(prev_svm, pi); 739 } 740 741 /** 742 * Allocating new amd_iommu_pi_data, which will get 743 * add to the per-vcpu ir_list. 744 */ 745 ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT); 746 if (!ir) { 747 ret = -ENOMEM; 748 goto out; 749 } 750 ir->data = pi->ir_data; 751 752 spin_lock_irqsave(&svm->ir_list_lock, flags); 753 list_add(&ir->node, &svm->ir_list); 754 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 755out: 756 return ret; 757} 758 759/** 760 * Note: 761 * The HW cannot support posting multicast/broadcast 762 * interrupts to a vCPU. So, we still use legacy interrupt 763 * remapping for these kind of interrupts. 764 * 765 * For lowest-priority interrupts, we only support 766 * those with single CPU as the destination, e.g. user 767 * configures the interrupts via /proc/irq or uses 768 * irqbalance to make the interrupts single-CPU. 769 */ 770static int 771get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, 772 struct vcpu_data *vcpu_info, struct vcpu_svm **svm) 773{ 774 struct kvm_lapic_irq irq; 775 struct kvm_vcpu *vcpu = NULL; 776 777 kvm_set_msi_irq(kvm, e, &irq); 778 779 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) || 780 !kvm_irq_is_postable(&irq)) { 781 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n", 782 __func__, irq.vector); 783 return -1; 784 } 785 786 pr_debug("SVM: %s: use GA mode for irq %u\n", __func__, 787 irq.vector); 788 *svm = to_svm(vcpu); 789 vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page)); 790 vcpu_info->vector = irq.vector; 791 792 return 0; 793} 794 795/* 796 * svm_update_pi_irte - set IRTE for Posted-Interrupts 797 * 798 * @kvm: kvm 799 * @host_irq: host irq of the interrupt 800 * @guest_irq: gsi of the interrupt 801 * @set: set or unset PI 802 * returns 0 on success, < 0 on failure 803 */ 804int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq, 805 uint32_t guest_irq, bool set) 806{ 807 struct kvm_kernel_irq_routing_entry *e; 808 struct kvm_irq_routing_table *irq_rt; 809 int idx, ret = 0; 810 811 if (!kvm_arch_has_assigned_device(kvm) || 812 !irq_remapping_cap(IRQ_POSTING_CAP)) 813 return 0; 814 815 pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n", 816 __func__, host_irq, guest_irq, set); 817 818 idx = srcu_read_lock(&kvm->irq_srcu); 819 irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); 820 821 if (guest_irq >= irq_rt->nr_rt_entries || 822 hlist_empty(&irq_rt->map[guest_irq])) { 823 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n", 824 guest_irq, irq_rt->nr_rt_entries); 825 goto out; 826 } 827 828 hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) { 829 struct vcpu_data vcpu_info; 830 struct vcpu_svm *svm = NULL; 831 832 if (e->type != KVM_IRQ_ROUTING_MSI) 833 continue; 834 835 /** 836 * Here, we setup with legacy mode in the following cases: 837 * 1. When cannot target interrupt to a specific vcpu. 838 * 2. Unsetting posted interrupt. 839 * 3. APIC virtialization is disabled for the vcpu. 840 * 4. IRQ has incompatible delivery mode (SMI, INIT, etc) 841 */ 842 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set && 843 kvm_vcpu_apicv_active(&svm->vcpu)) { 844 struct amd_iommu_pi_data pi; 845 846 /* Try to enable guest_mode in IRTE */ 847 pi.base = __sme_set(page_to_phys(svm->avic_backing_page) & 848 AVIC_HPA_MASK); 849 pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id, 850 svm->vcpu.vcpu_id); 851 pi.is_guest_mode = true; 852 pi.vcpu_data = &vcpu_info; 853 ret = irq_set_vcpu_affinity(host_irq, &pi); 854 855 /** 856 * Here, we successfully setting up vcpu affinity in 857 * IOMMU guest mode. Now, we need to store the posted 858 * interrupt information in a per-vcpu ir_list so that 859 * we can reference to them directly when we update vcpu 860 * scheduling information in IOMMU irte. 861 */ 862 if (!ret && pi.is_guest_mode) 863 svm_ir_list_add(svm, &pi); 864 } else { 865 /* Use legacy mode in IRTE */ 866 struct amd_iommu_pi_data pi; 867 868 /** 869 * Here, pi is used to: 870 * - Tell IOMMU to use legacy mode for this interrupt. 871 * - Retrieve ga_tag of prior interrupt remapping data. 872 */ 873 pi.prev_ga_tag = 0; 874 pi.is_guest_mode = false; 875 ret = irq_set_vcpu_affinity(host_irq, &pi); 876 877 /** 878 * Check if the posted interrupt was previously 879 * setup with the guest_mode by checking if the ga_tag 880 * was cached. If so, we need to clean up the per-vcpu 881 * ir_list. 882 */ 883 if (!ret && pi.prev_ga_tag) { 884 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag); 885 struct kvm_vcpu *vcpu; 886 887 vcpu = kvm_get_vcpu_by_id(kvm, id); 888 if (vcpu) 889 svm_ir_list_del(to_svm(vcpu), &pi); 890 } 891 } 892 893 if (!ret && svm) { 894 trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id, 895 e->gsi, vcpu_info.vector, 896 vcpu_info.pi_desc_addr, set); 897 } 898 899 if (ret < 0) { 900 pr_err("%s: failed to update PI IRTE\n", __func__); 901 goto out; 902 } 903 } 904 905 ret = 0; 906out: 907 srcu_read_unlock(&kvm->irq_srcu, idx); 908 return ret; 909} 910 911bool svm_check_apicv_inhibit_reasons(ulong bit) 912{ 913 ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) | 914 BIT(APICV_INHIBIT_REASON_HYPERV) | 915 BIT(APICV_INHIBIT_REASON_NESTED) | 916 BIT(APICV_INHIBIT_REASON_IRQWIN) | 917 BIT(APICV_INHIBIT_REASON_PIT_REINJ) | 918 BIT(APICV_INHIBIT_REASON_X2APIC); 919 920 return supported & BIT(bit); 921} 922 923void svm_pre_update_apicv_exec_ctrl(struct kvm *kvm, bool activate) 924{ 925 avic_update_access_page(kvm, activate); 926} 927 928static inline int 929avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r) 930{ 931 int ret = 0; 932 unsigned long flags; 933 struct amd_svm_iommu_ir *ir; 934 struct vcpu_svm *svm = to_svm(vcpu); 935 936 if (!kvm_arch_has_assigned_device(vcpu->kvm)) 937 return 0; 938 939 /* 940 * Here, we go through the per-vcpu ir_list to update all existing 941 * interrupt remapping table entry targeting this vcpu. 942 */ 943 spin_lock_irqsave(&svm->ir_list_lock, flags); 944 945 if (list_empty(&svm->ir_list)) 946 goto out; 947 948 list_for_each_entry(ir, &svm->ir_list, node) { 949 ret = amd_iommu_update_ga(cpu, r, ir->data); 950 if (ret) 951 break; 952 } 953out: 954 spin_unlock_irqrestore(&svm->ir_list_lock, flags); 955 return ret; 956} 957 958void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 959{ 960 u64 entry; 961 /* ID = 0xff (broadcast), ID > 0xff (reserved) */ 962 int h_physical_id = kvm_cpu_get_apicid(cpu); 963 struct vcpu_svm *svm = to_svm(vcpu); 964 965 if (!kvm_vcpu_apicv_active(vcpu)) 966 return; 967 968 /* 969 * Since the host physical APIC id is 8 bits, 970 * we can support host APIC ID upto 255. 971 */ 972 if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK)) 973 return; 974 975 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 976 WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK); 977 978 entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK; 979 entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK); 980 981 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 982 if (svm->avic_is_running) 983 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 984 985 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 986 avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, 987 svm->avic_is_running); 988} 989 990void avic_vcpu_put(struct kvm_vcpu *vcpu) 991{ 992 u64 entry; 993 struct vcpu_svm *svm = to_svm(vcpu); 994 995 if (!kvm_vcpu_apicv_active(vcpu)) 996 return; 997 998 entry = READ_ONCE(*(svm->avic_physical_id_cache)); 999 if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) 1000 avic_update_iommu_vcpu_affinity(vcpu, -1, 0); 1001 1002 entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK; 1003 WRITE_ONCE(*(svm->avic_physical_id_cache), entry); 1004} 1005 1006/** 1007 * This function is called during VCPU halt/unhalt. 1008 */ 1009static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run) 1010{ 1011 struct vcpu_svm *svm = to_svm(vcpu); 1012 1013 svm->avic_is_running = is_run; 1014 if (is_run) 1015 avic_vcpu_load(vcpu, vcpu->cpu); 1016 else 1017 avic_vcpu_put(vcpu); 1018} 1019 1020void svm_vcpu_blocking(struct kvm_vcpu *vcpu) 1021{ 1022 avic_set_running(vcpu, false); 1023} 1024 1025void svm_vcpu_unblocking(struct kvm_vcpu *vcpu) 1026{ 1027 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 1028 kvm_vcpu_update_apicv(vcpu); 1029 avic_set_running(vcpu, true); 1030} 1031