1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * VGIC MMIO handling functions 4 */ 5 6#include <linux/bitops.h> 7#include <linux/bsearch.h> 8#include <linux/interrupt.h> 9#include <linux/irq.h> 10#include <linux/kvm.h> 11#include <linux/kvm_host.h> 12#include <kvm/iodev.h> 13#include <kvm/arm_arch_timer.h> 14#include <kvm/arm_vgic.h> 15 16#include "vgic.h" 17#include "vgic-mmio.h" 18 19unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu, 20 gpa_t addr, unsigned int len) 21{ 22 return 0; 23} 24 25unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu, 26 gpa_t addr, unsigned int len) 27{ 28 return -1UL; 29} 30 31void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr, 32 unsigned int len, unsigned long val) 33{ 34 /* Ignore */ 35} 36 37int vgic_mmio_uaccess_write_wi(struct kvm_vcpu *vcpu, gpa_t addr, 38 unsigned int len, unsigned long val) 39{ 40 /* Ignore */ 41 return 0; 42} 43 44unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu, 45 gpa_t addr, unsigned int len) 46{ 47 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 48 u32 value = 0; 49 int i; 50 51 /* Loop over all IRQs affected by this read */ 52 for (i = 0; i < len * 8; i++) { 53 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 54 55 if (irq->group) 56 value |= BIT(i); 57 58 vgic_put_irq(vcpu->kvm, irq); 59 } 60 61 return value; 62} 63 64static void vgic_update_vsgi(struct vgic_irq *irq) 65{ 66 WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group)); 67} 68 69void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, 70 unsigned int len, unsigned long val) 71{ 72 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 73 int i; 74 unsigned long flags; 75 76 for (i = 0; i < len * 8; i++) { 77 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 78 79 raw_spin_lock_irqsave(&irq->irq_lock, flags); 80 irq->group = !!(val & BIT(i)); 81 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 82 vgic_update_vsgi(irq); 83 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 84 } else { 85 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 86 } 87 88 vgic_put_irq(vcpu->kvm, irq); 89 } 90} 91 92/* 93 * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value 94 * of the enabled bit, so there is only one function for both here. 95 */ 96unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu, 97 gpa_t addr, unsigned int len) 98{ 99 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 100 u32 value = 0; 101 int i; 102 103 /* Loop over all IRQs affected by this read */ 104 for (i = 0; i < len * 8; i++) { 105 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 106 107 if (irq->enabled) 108 value |= (1U << i); 109 110 vgic_put_irq(vcpu->kvm, irq); 111 } 112 113 return value; 114} 115 116void vgic_mmio_write_senable(struct kvm_vcpu *vcpu, 117 gpa_t addr, unsigned int len, 118 unsigned long val) 119{ 120 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 121 int i; 122 unsigned long flags; 123 124 for_each_set_bit(i, &val, len * 8) { 125 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 126 127 raw_spin_lock_irqsave(&irq->irq_lock, flags); 128 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 129 if (!irq->enabled) { 130 struct irq_data *data; 131 132 irq->enabled = true; 133 data = &irq_to_desc(irq->host_irq)->irq_data; 134 while (irqd_irq_disabled(data)) 135 enable_irq(irq->host_irq); 136 } 137 138 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 139 vgic_put_irq(vcpu->kvm, irq); 140 141 continue; 142 } else if (vgic_irq_is_mapped_level(irq)) { 143 bool was_high = irq->line_level; 144 145 /* 146 * We need to update the state of the interrupt because 147 * the guest might have changed the state of the device 148 * while the interrupt was disabled at the VGIC level. 149 */ 150 irq->line_level = vgic_get_phys_line_level(irq); 151 /* 152 * Deactivate the physical interrupt so the GIC will let 153 * us know when it is asserted again. 154 */ 155 if (!irq->active && was_high && !irq->line_level) 156 vgic_irq_set_phys_active(irq, false); 157 } 158 irq->enabled = true; 159 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 160 161 vgic_put_irq(vcpu->kvm, irq); 162 } 163} 164 165void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu, 166 gpa_t addr, unsigned int len, 167 unsigned long val) 168{ 169 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 170 int i; 171 unsigned long flags; 172 173 for_each_set_bit(i, &val, len * 8) { 174 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 175 176 raw_spin_lock_irqsave(&irq->irq_lock, flags); 177 if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled) 178 disable_irq_nosync(irq->host_irq); 179 180 irq->enabled = false; 181 182 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 183 vgic_put_irq(vcpu->kvm, irq); 184 } 185} 186 187int vgic_uaccess_write_senable(struct kvm_vcpu *vcpu, 188 gpa_t addr, unsigned int len, 189 unsigned long val) 190{ 191 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 192 int i; 193 unsigned long flags; 194 195 for_each_set_bit(i, &val, len * 8) { 196 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 197 198 raw_spin_lock_irqsave(&irq->irq_lock, flags); 199 irq->enabled = true; 200 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 201 202 vgic_put_irq(vcpu->kvm, irq); 203 } 204 205 return 0; 206} 207 208int vgic_uaccess_write_cenable(struct kvm_vcpu *vcpu, 209 gpa_t addr, unsigned int len, 210 unsigned long val) 211{ 212 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 213 int i; 214 unsigned long flags; 215 216 for_each_set_bit(i, &val, len * 8) { 217 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 218 219 raw_spin_lock_irqsave(&irq->irq_lock, flags); 220 irq->enabled = false; 221 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 222 223 vgic_put_irq(vcpu->kvm, irq); 224 } 225 226 return 0; 227} 228 229static unsigned long __read_pending(struct kvm_vcpu *vcpu, 230 gpa_t addr, unsigned int len, 231 bool is_user) 232{ 233 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 234 u32 value = 0; 235 int i; 236 237 /* Loop over all IRQs affected by this read */ 238 for (i = 0; i < len * 8; i++) { 239 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 240 unsigned long flags; 241 bool val; 242 243 /* 244 * When used from userspace with a GICv3 model: 245 * 246 * Pending state of interrupt is latched in pending_latch 247 * variable. Userspace will save and restore pending state 248 * and line_level separately. 249 * Refer to Documentation/virt/kvm/devices/arm-vgic-v3.rst 250 * for handling of ISPENDR and ICPENDR. 251 */ 252 raw_spin_lock_irqsave(&irq->irq_lock, flags); 253 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 254 int err; 255 256 val = false; 257 err = irq_get_irqchip_state(irq->host_irq, 258 IRQCHIP_STATE_PENDING, 259 &val); 260 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 261 } else if (!is_user && vgic_irq_is_mapped_level(irq)) { 262 val = vgic_get_phys_line_level(irq); 263 } else { 264 switch (vcpu->kvm->arch.vgic.vgic_model) { 265 case KVM_DEV_TYPE_ARM_VGIC_V3: 266 if (is_user) { 267 val = irq->pending_latch; 268 break; 269 } 270 fallthrough; 271 default: 272 val = irq_is_pending(irq); 273 break; 274 } 275 } 276 277 value |= ((u32)val << i); 278 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 279 280 vgic_put_irq(vcpu->kvm, irq); 281 } 282 283 return value; 284} 285 286unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu, 287 gpa_t addr, unsigned int len) 288{ 289 return __read_pending(vcpu, addr, len, false); 290} 291 292unsigned long vgic_uaccess_read_pending(struct kvm_vcpu *vcpu, 293 gpa_t addr, unsigned int len) 294{ 295 return __read_pending(vcpu, addr, len, true); 296} 297 298static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq) 299{ 300 return (vgic_irq_is_sgi(irq->intid) && 301 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2); 302} 303 304void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, 305 gpa_t addr, unsigned int len, 306 unsigned long val) 307{ 308 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 309 int i; 310 unsigned long flags; 311 312 for_each_set_bit(i, &val, len * 8) { 313 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 314 315 /* GICD_ISPENDR0 SGI bits are WI */ 316 if (is_vgic_v2_sgi(vcpu, irq)) { 317 vgic_put_irq(vcpu->kvm, irq); 318 continue; 319 } 320 321 raw_spin_lock_irqsave(&irq->irq_lock, flags); 322 323 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 324 /* HW SGI? Ask the GIC to inject it */ 325 int err; 326 err = irq_set_irqchip_state(irq->host_irq, 327 IRQCHIP_STATE_PENDING, 328 true); 329 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 330 331 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 332 vgic_put_irq(vcpu->kvm, irq); 333 334 continue; 335 } 336 337 irq->pending_latch = true; 338 if (irq->hw) 339 vgic_irq_set_phys_active(irq, true); 340 341 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 342 vgic_put_irq(vcpu->kvm, irq); 343 } 344} 345 346int vgic_uaccess_write_spending(struct kvm_vcpu *vcpu, 347 gpa_t addr, unsigned int len, 348 unsigned long val) 349{ 350 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 351 int i; 352 unsigned long flags; 353 354 for_each_set_bit(i, &val, len * 8) { 355 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 356 357 raw_spin_lock_irqsave(&irq->irq_lock, flags); 358 irq->pending_latch = true; 359 360 /* 361 * GICv2 SGIs are terribly broken. We can't restore 362 * the source of the interrupt, so just pick the vcpu 363 * itself as the source... 364 */ 365 if (is_vgic_v2_sgi(vcpu, irq)) 366 irq->source |= BIT(vcpu->vcpu_id); 367 368 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 369 370 vgic_put_irq(vcpu->kvm, irq); 371 } 372 373 return 0; 374} 375 376/* Must be called with irq->irq_lock held */ 377static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq) 378{ 379 irq->pending_latch = false; 380 381 /* 382 * We don't want the guest to effectively mask the physical 383 * interrupt by doing a write to SPENDR followed by a write to 384 * CPENDR for HW interrupts, so we clear the active state on 385 * the physical side if the virtual interrupt is not active. 386 * This may lead to taking an additional interrupt on the 387 * host, but that should not be a problem as the worst that 388 * can happen is an additional vgic injection. We also clear 389 * the pending state to maintain proper semantics for edge HW 390 * interrupts. 391 */ 392 vgic_irq_set_phys_pending(irq, false); 393 if (!irq->active) 394 vgic_irq_set_phys_active(irq, false); 395} 396 397void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, 398 gpa_t addr, unsigned int len, 399 unsigned long val) 400{ 401 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 402 int i; 403 unsigned long flags; 404 405 for_each_set_bit(i, &val, len * 8) { 406 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 407 408 /* GICD_ICPENDR0 SGI bits are WI */ 409 if (is_vgic_v2_sgi(vcpu, irq)) { 410 vgic_put_irq(vcpu->kvm, irq); 411 continue; 412 } 413 414 raw_spin_lock_irqsave(&irq->irq_lock, flags); 415 416 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 417 /* HW SGI? Ask the GIC to clear its pending bit */ 418 int err; 419 err = irq_set_irqchip_state(irq->host_irq, 420 IRQCHIP_STATE_PENDING, 421 false); 422 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 423 424 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 425 vgic_put_irq(vcpu->kvm, irq); 426 427 continue; 428 } 429 430 if (irq->hw) 431 vgic_hw_irq_cpending(vcpu, irq); 432 else 433 irq->pending_latch = false; 434 435 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 436 vgic_put_irq(vcpu->kvm, irq); 437 } 438} 439 440int vgic_uaccess_write_cpending(struct kvm_vcpu *vcpu, 441 gpa_t addr, unsigned int len, 442 unsigned long val) 443{ 444 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 445 int i; 446 unsigned long flags; 447 448 for_each_set_bit(i, &val, len * 8) { 449 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 450 451 raw_spin_lock_irqsave(&irq->irq_lock, flags); 452 /* 453 * More fun with GICv2 SGIs! If we're clearing one of them 454 * from userspace, which source vcpu to clear? Let's not 455 * even think of it, and blow the whole set. 456 */ 457 if (is_vgic_v2_sgi(vcpu, irq)) 458 irq->source = 0; 459 460 irq->pending_latch = false; 461 462 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 463 464 vgic_put_irq(vcpu->kvm, irq); 465 } 466 467 return 0; 468} 469 470/* 471 * If we are fiddling with an IRQ's active state, we have to make sure the IRQ 472 * is not queued on some running VCPU's LRs, because then the change to the 473 * active state can be overwritten when the VCPU's state is synced coming back 474 * from the guest. 475 * 476 * For shared interrupts as well as GICv3 private interrupts accessed from the 477 * non-owning CPU, we have to stop all the VCPUs because interrupts can be 478 * migrated while we don't hold the IRQ locks and we don't want to be chasing 479 * moving targets. 480 * 481 * For GICv2 private interrupts we don't have to do anything because 482 * userspace accesses to the VGIC state already require all VCPUs to be 483 * stopped, and only the VCPU itself can modify its private interrupts 484 * active state, which guarantees that the VCPU is not running. 485 */ 486static void vgic_access_active_prepare(struct kvm_vcpu *vcpu, u32 intid) 487{ 488 if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 && 489 vcpu != kvm_get_running_vcpu()) || 490 intid >= VGIC_NR_PRIVATE_IRQS) 491 kvm_arm_halt_guest(vcpu->kvm); 492} 493 494/* See vgic_access_active_prepare */ 495static void vgic_access_active_finish(struct kvm_vcpu *vcpu, u32 intid) 496{ 497 if ((vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 && 498 vcpu != kvm_get_running_vcpu()) || 499 intid >= VGIC_NR_PRIVATE_IRQS) 500 kvm_arm_resume_guest(vcpu->kvm); 501} 502 503static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu, 504 gpa_t addr, unsigned int len) 505{ 506 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 507 u32 value = 0; 508 int i; 509 510 /* Loop over all IRQs affected by this read */ 511 for (i = 0; i < len * 8; i++) { 512 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 513 514 /* 515 * Even for HW interrupts, don't evaluate the HW state as 516 * all the guest is interested in is the virtual state. 517 */ 518 if (irq->active) 519 value |= (1U << i); 520 521 vgic_put_irq(vcpu->kvm, irq); 522 } 523 524 return value; 525} 526 527unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu, 528 gpa_t addr, unsigned int len) 529{ 530 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 531 u32 val; 532 533 mutex_lock(&vcpu->kvm->arch.config_lock); 534 vgic_access_active_prepare(vcpu, intid); 535 536 val = __vgic_mmio_read_active(vcpu, addr, len); 537 538 vgic_access_active_finish(vcpu, intid); 539 mutex_unlock(&vcpu->kvm->arch.config_lock); 540 541 return val; 542} 543 544unsigned long vgic_uaccess_read_active(struct kvm_vcpu *vcpu, 545 gpa_t addr, unsigned int len) 546{ 547 return __vgic_mmio_read_active(vcpu, addr, len); 548} 549 550/* Must be called with irq->irq_lock held */ 551static void vgic_hw_irq_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, 552 bool active, bool is_uaccess) 553{ 554 if (is_uaccess) 555 return; 556 557 irq->active = active; 558 vgic_irq_set_phys_active(irq, active); 559} 560 561static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, 562 bool active) 563{ 564 unsigned long flags; 565 struct kvm_vcpu *requester_vcpu = kvm_get_running_vcpu(); 566 567 raw_spin_lock_irqsave(&irq->irq_lock, flags); 568 569 if (irq->hw && !vgic_irq_is_sgi(irq->intid)) { 570 vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu); 571 } else if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 572 /* 573 * GICv4.1 VSGI feature doesn't track an active state, 574 * so let's not kid ourselves, there is nothing we can 575 * do here. 576 */ 577 irq->active = false; 578 } else { 579 u32 model = vcpu->kvm->arch.vgic.vgic_model; 580 u8 active_source; 581 582 irq->active = active; 583 584 /* 585 * The GICv2 architecture indicates that the source CPUID for 586 * an SGI should be provided during an EOI which implies that 587 * the active state is stored somewhere, but at the same time 588 * this state is not architecturally exposed anywhere and we 589 * have no way of knowing the right source. 590 * 591 * This may lead to a VCPU not being able to receive 592 * additional instances of a particular SGI after migration 593 * for a GICv2 VM on some GIC implementations. Oh well. 594 */ 595 active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0; 596 597 if (model == KVM_DEV_TYPE_ARM_VGIC_V2 && 598 active && vgic_irq_is_sgi(irq->intid)) 599 irq->active_source = active_source; 600 } 601 602 if (irq->active) 603 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 604 else 605 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 606} 607 608static void __vgic_mmio_write_cactive(struct kvm_vcpu *vcpu, 609 gpa_t addr, unsigned int len, 610 unsigned long val) 611{ 612 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 613 int i; 614 615 for_each_set_bit(i, &val, len * 8) { 616 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 617 vgic_mmio_change_active(vcpu, irq, false); 618 vgic_put_irq(vcpu->kvm, irq); 619 } 620} 621 622void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu, 623 gpa_t addr, unsigned int len, 624 unsigned long val) 625{ 626 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 627 628 mutex_lock(&vcpu->kvm->arch.config_lock); 629 vgic_access_active_prepare(vcpu, intid); 630 631 __vgic_mmio_write_cactive(vcpu, addr, len, val); 632 633 vgic_access_active_finish(vcpu, intid); 634 mutex_unlock(&vcpu->kvm->arch.config_lock); 635} 636 637int vgic_mmio_uaccess_write_cactive(struct kvm_vcpu *vcpu, 638 gpa_t addr, unsigned int len, 639 unsigned long val) 640{ 641 __vgic_mmio_write_cactive(vcpu, addr, len, val); 642 return 0; 643} 644 645static void __vgic_mmio_write_sactive(struct kvm_vcpu *vcpu, 646 gpa_t addr, unsigned int len, 647 unsigned long val) 648{ 649 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 650 int i; 651 652 for_each_set_bit(i, &val, len * 8) { 653 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 654 vgic_mmio_change_active(vcpu, irq, true); 655 vgic_put_irq(vcpu->kvm, irq); 656 } 657} 658 659void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu, 660 gpa_t addr, unsigned int len, 661 unsigned long val) 662{ 663 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 664 665 mutex_lock(&vcpu->kvm->arch.config_lock); 666 vgic_access_active_prepare(vcpu, intid); 667 668 __vgic_mmio_write_sactive(vcpu, addr, len, val); 669 670 vgic_access_active_finish(vcpu, intid); 671 mutex_unlock(&vcpu->kvm->arch.config_lock); 672} 673 674int vgic_mmio_uaccess_write_sactive(struct kvm_vcpu *vcpu, 675 gpa_t addr, unsigned int len, 676 unsigned long val) 677{ 678 __vgic_mmio_write_sactive(vcpu, addr, len, val); 679 return 0; 680} 681 682unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu, 683 gpa_t addr, unsigned int len) 684{ 685 u32 intid = VGIC_ADDR_TO_INTID(addr, 8); 686 int i; 687 u64 val = 0; 688 689 for (i = 0; i < len; i++) { 690 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 691 692 val |= (u64)irq->priority << (i * 8); 693 694 vgic_put_irq(vcpu->kvm, irq); 695 } 696 697 return val; 698} 699 700/* 701 * We currently don't handle changing the priority of an interrupt that 702 * is already pending on a VCPU. If there is a need for this, we would 703 * need to make this VCPU exit and re-evaluate the priorities, potentially 704 * leading to this interrupt getting presented now to the guest (if it has 705 * been masked by the priority mask before). 706 */ 707void vgic_mmio_write_priority(struct kvm_vcpu *vcpu, 708 gpa_t addr, unsigned int len, 709 unsigned long val) 710{ 711 u32 intid = VGIC_ADDR_TO_INTID(addr, 8); 712 int i; 713 unsigned long flags; 714 715 for (i = 0; i < len; i++) { 716 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 717 718 raw_spin_lock_irqsave(&irq->irq_lock, flags); 719 /* Narrow the priority range to what we actually support */ 720 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); 721 if (irq->hw && vgic_irq_is_sgi(irq->intid)) 722 vgic_update_vsgi(irq); 723 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 724 725 vgic_put_irq(vcpu->kvm, irq); 726 } 727} 728 729unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu, 730 gpa_t addr, unsigned int len) 731{ 732 u32 intid = VGIC_ADDR_TO_INTID(addr, 2); 733 u32 value = 0; 734 int i; 735 736 for (i = 0; i < len * 4; i++) { 737 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 738 739 if (irq->config == VGIC_CONFIG_EDGE) 740 value |= (2U << (i * 2)); 741 742 vgic_put_irq(vcpu->kvm, irq); 743 } 744 745 return value; 746} 747 748void vgic_mmio_write_config(struct kvm_vcpu *vcpu, 749 gpa_t addr, unsigned int len, 750 unsigned long val) 751{ 752 u32 intid = VGIC_ADDR_TO_INTID(addr, 2); 753 int i; 754 unsigned long flags; 755 756 for (i = 0; i < len * 4; i++) { 757 struct vgic_irq *irq; 758 759 /* 760 * The configuration cannot be changed for SGIs in general, 761 * for PPIs this is IMPLEMENTATION DEFINED. The arch timer 762 * code relies on PPIs being level triggered, so we also 763 * make them read-only here. 764 */ 765 if (intid + i < VGIC_NR_PRIVATE_IRQS) 766 continue; 767 768 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 769 raw_spin_lock_irqsave(&irq->irq_lock, flags); 770 771 if (test_bit(i * 2 + 1, &val)) 772 irq->config = VGIC_CONFIG_EDGE; 773 else 774 irq->config = VGIC_CONFIG_LEVEL; 775 776 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 777 vgic_put_irq(vcpu->kvm, irq); 778 } 779} 780 781u32 vgic_read_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid) 782{ 783 int i; 784 u32 val = 0; 785 int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 786 787 for (i = 0; i < 32; i++) { 788 struct vgic_irq *irq; 789 790 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs) 791 continue; 792 793 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 794 if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level) 795 val |= (1U << i); 796 797 vgic_put_irq(vcpu->kvm, irq); 798 } 799 800 return val; 801} 802 803void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid, 804 const u32 val) 805{ 806 int i; 807 int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 808 unsigned long flags; 809 810 for (i = 0; i < 32; i++) { 811 struct vgic_irq *irq; 812 bool new_level; 813 814 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs) 815 continue; 816 817 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 818 819 /* 820 * Line level is set irrespective of irq type 821 * (level or edge) to avoid dependency that VM should 822 * restore irq config before line level. 823 */ 824 new_level = !!(val & (1U << i)); 825 raw_spin_lock_irqsave(&irq->irq_lock, flags); 826 irq->line_level = new_level; 827 if (new_level) 828 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 829 else 830 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 831 832 vgic_put_irq(vcpu->kvm, irq); 833 } 834} 835 836static int match_region(const void *key, const void *elt) 837{ 838 const unsigned int offset = (unsigned long)key; 839 const struct vgic_register_region *region = elt; 840 841 if (offset < region->reg_offset) 842 return -1; 843 844 if (offset >= region->reg_offset + region->len) 845 return 1; 846 847 return 0; 848} 849 850const struct vgic_register_region * 851vgic_find_mmio_region(const struct vgic_register_region *regions, 852 int nr_regions, unsigned int offset) 853{ 854 return bsearch((void *)(uintptr_t)offset, regions, nr_regions, 855 sizeof(regions[0]), match_region); 856} 857 858void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) 859{ 860 if (kvm_vgic_global_state.type == VGIC_V2) 861 vgic_v2_set_vmcr(vcpu, vmcr); 862 else 863 vgic_v3_set_vmcr(vcpu, vmcr); 864} 865 866void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) 867{ 868 if (kvm_vgic_global_state.type == VGIC_V2) 869 vgic_v2_get_vmcr(vcpu, vmcr); 870 else 871 vgic_v3_get_vmcr(vcpu, vmcr); 872} 873 874/* 875 * kvm_mmio_read_buf() returns a value in a format where it can be converted 876 * to a byte array and be directly observed as the guest wanted it to appear 877 * in memory if it had done the store itself, which is LE for the GIC, as the 878 * guest knows the GIC is always LE. 879 * 880 * We convert this value to the CPUs native format to deal with it as a data 881 * value. 882 */ 883unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len) 884{ 885 unsigned long data = kvm_mmio_read_buf(val, len); 886 887 switch (len) { 888 case 1: 889 return data; 890 case 2: 891 return le16_to_cpu(data); 892 case 4: 893 return le32_to_cpu(data); 894 default: 895 return le64_to_cpu(data); 896 } 897} 898 899/* 900 * kvm_mmio_write_buf() expects a value in a format such that if converted to 901 * a byte array it is observed as the guest would see it if it could perform 902 * the load directly. Since the GIC is LE, and the guest knows this, the 903 * guest expects a value in little endian format. 904 * 905 * We convert the data value from the CPUs native format to LE so that the 906 * value is returned in the proper format. 907 */ 908void vgic_data_host_to_mmio_bus(void *buf, unsigned int len, 909 unsigned long data) 910{ 911 switch (len) { 912 case 1: 913 break; 914 case 2: 915 data = cpu_to_le16(data); 916 break; 917 case 4: 918 data = cpu_to_le32(data); 919 break; 920 default: 921 data = cpu_to_le64(data); 922 } 923 924 kvm_mmio_write_buf(buf, len, data); 925} 926 927static 928struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev) 929{ 930 return container_of(dev, struct vgic_io_device, dev); 931} 932 933static bool check_region(const struct kvm *kvm, 934 const struct vgic_register_region *region, 935 gpa_t addr, int len) 936{ 937 int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 938 939 switch (len) { 940 case sizeof(u8): 941 flags = VGIC_ACCESS_8bit; 942 break; 943 case sizeof(u32): 944 flags = VGIC_ACCESS_32bit; 945 break; 946 case sizeof(u64): 947 flags = VGIC_ACCESS_64bit; 948 break; 949 default: 950 return false; 951 } 952 953 if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) { 954 if (!region->bits_per_irq) 955 return true; 956 957 /* Do we access a non-allocated IRQ? */ 958 return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs; 959 } 960 961 return false; 962} 963 964const struct vgic_register_region * 965vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, 966 gpa_t addr, int len) 967{ 968 const struct vgic_register_region *region; 969 970 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions, 971 addr - iodev->base_addr); 972 if (!region || !check_region(vcpu->kvm, region, addr, len)) 973 return NULL; 974 975 return region; 976} 977 978static int vgic_uaccess_read(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, 979 gpa_t addr, u32 *val) 980{ 981 const struct vgic_register_region *region; 982 struct kvm_vcpu *r_vcpu; 983 984 region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32)); 985 if (!region) { 986 *val = 0; 987 return 0; 988 } 989 990 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu; 991 if (region->uaccess_read) 992 *val = region->uaccess_read(r_vcpu, addr, sizeof(u32)); 993 else 994 *val = region->read(r_vcpu, addr, sizeof(u32)); 995 996 return 0; 997} 998 999static int vgic_uaccess_write(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, 1000 gpa_t addr, const u32 *val) 1001{ 1002 const struct vgic_register_region *region; 1003 struct kvm_vcpu *r_vcpu; 1004 1005 region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32)); 1006 if (!region) 1007 return 0; 1008 1009 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu; 1010 if (region->uaccess_write) 1011 return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val); 1012 1013 region->write(r_vcpu, addr, sizeof(u32), *val); 1014 return 0; 1015} 1016 1017/* 1018 * Userland access to VGIC registers. 1019 */ 1020int vgic_uaccess(struct kvm_vcpu *vcpu, struct vgic_io_device *dev, 1021 bool is_write, int offset, u32 *val) 1022{ 1023 if (is_write) 1024 return vgic_uaccess_write(vcpu, dev, offset, val); 1025 else 1026 return vgic_uaccess_read(vcpu, dev, offset, val); 1027} 1028 1029static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 1030 gpa_t addr, int len, void *val) 1031{ 1032 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 1033 const struct vgic_register_region *region; 1034 unsigned long data = 0; 1035 1036 region = vgic_get_mmio_region(vcpu, iodev, addr, len); 1037 if (!region) { 1038 memset(val, 0, len); 1039 return 0; 1040 } 1041 1042 switch (iodev->iodev_type) { 1043 case IODEV_CPUIF: 1044 data = region->read(vcpu, addr, len); 1045 break; 1046 case IODEV_DIST: 1047 data = region->read(vcpu, addr, len); 1048 break; 1049 case IODEV_REDIST: 1050 data = region->read(iodev->redist_vcpu, addr, len); 1051 break; 1052 case IODEV_ITS: 1053 data = region->its_read(vcpu->kvm, iodev->its, addr, len); 1054 break; 1055 } 1056 1057 vgic_data_host_to_mmio_bus(val, len, data); 1058 return 0; 1059} 1060 1061static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 1062 gpa_t addr, int len, const void *val) 1063{ 1064 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 1065 const struct vgic_register_region *region; 1066 unsigned long data = vgic_data_mmio_bus_to_host(val, len); 1067 1068 region = vgic_get_mmio_region(vcpu, iodev, addr, len); 1069 if (!region) 1070 return 0; 1071 1072 switch (iodev->iodev_type) { 1073 case IODEV_CPUIF: 1074 region->write(vcpu, addr, len, data); 1075 break; 1076 case IODEV_DIST: 1077 region->write(vcpu, addr, len, data); 1078 break; 1079 case IODEV_REDIST: 1080 region->write(iodev->redist_vcpu, addr, len, data); 1081 break; 1082 case IODEV_ITS: 1083 region->its_write(vcpu->kvm, iodev->its, addr, len, data); 1084 break; 1085 } 1086 1087 return 0; 1088} 1089 1090const struct kvm_io_device_ops kvm_io_gic_ops = { 1091 .read = dispatch_mmio_read, 1092 .write = dispatch_mmio_write, 1093}; 1094 1095int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address, 1096 enum vgic_type type) 1097{ 1098 struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev; 1099 unsigned int len; 1100 1101 switch (type) { 1102 case VGIC_V2: 1103 len = vgic_v2_init_dist_iodev(io_device); 1104 break; 1105 case VGIC_V3: 1106 len = vgic_v3_init_dist_iodev(io_device); 1107 break; 1108 default: 1109 BUG_ON(1); 1110 } 1111 1112 io_device->base_addr = dist_base_address; 1113 io_device->iodev_type = IODEV_DIST; 1114 io_device->redist_vcpu = NULL; 1115 1116 return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address, 1117 len, &io_device->dev); 1118} 1119