1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * VGIC MMIO handling functions 4 */ 5 6#include <linux/bitops.h> 7#include <linux/bsearch.h> 8#include <linux/interrupt.h> 9#include <linux/irq.h> 10#include <linux/kvm.h> 11#include <linux/kvm_host.h> 12#include <kvm/iodev.h> 13#include <kvm/arm_arch_timer.h> 14#include <kvm/arm_vgic.h> 15 16#include "vgic.h" 17#include "vgic-mmio.h" 18 19unsigned long vgic_mmio_read_raz(struct kvm_vcpu *vcpu, 20 gpa_t addr, unsigned int len) 21{ 22 return 0; 23} 24 25unsigned long vgic_mmio_read_rao(struct kvm_vcpu *vcpu, 26 gpa_t addr, unsigned int len) 27{ 28 return -1UL; 29} 30 31void vgic_mmio_write_wi(struct kvm_vcpu *vcpu, gpa_t addr, 32 unsigned int len, unsigned long val) 33{ 34 /* Ignore */ 35} 36 37int vgic_mmio_uaccess_write_wi(struct kvm_vcpu *vcpu, gpa_t addr, 38 unsigned int len, unsigned long val) 39{ 40 /* Ignore */ 41 return 0; 42} 43 44unsigned long vgic_mmio_read_group(struct kvm_vcpu *vcpu, 45 gpa_t addr, unsigned int len) 46{ 47 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 48 u32 value = 0; 49 int i; 50 51 /* Loop over all IRQs affected by this read */ 52 for (i = 0; i < len * 8; i++) { 53 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 54 55 if (irq->group) 56 value |= BIT(i); 57 58 vgic_put_irq(vcpu->kvm, irq); 59 } 60 61 return value; 62} 63 64static void vgic_update_vsgi(struct vgic_irq *irq) 65{ 66 WARN_ON(its_prop_update_vsgi(irq->host_irq, irq->priority, irq->group)); 67} 68 69void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, 70 unsigned int len, unsigned long val) 71{ 72 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 73 int i; 74 unsigned long flags; 75 76 for (i = 0; i < len * 8; i++) { 77 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 78 79 raw_spin_lock_irqsave(&irq->irq_lock, flags); 80 irq->group = !!(val & BIT(i)); 81 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 82 vgic_update_vsgi(irq); 83 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 84 } else { 85 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 86 } 87 88 vgic_put_irq(vcpu->kvm, irq); 89 } 90} 91 92/* 93 * Read accesses to both GICD_ICENABLER and GICD_ISENABLER return the value 94 * of the enabled bit, so there is only one function for both here. 95 */ 96unsigned long vgic_mmio_read_enable(struct kvm_vcpu *vcpu, 97 gpa_t addr, unsigned int len) 98{ 99 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 100 u32 value = 0; 101 int i; 102 103 /* Loop over all IRQs affected by this read */ 104 for (i = 0; i < len * 8; i++) { 105 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 106 107 if (irq->enabled) 108 value |= (1U << i); 109 110 vgic_put_irq(vcpu->kvm, irq); 111 } 112 113 return value; 114} 115 116void vgic_mmio_write_senable(struct kvm_vcpu *vcpu, 117 gpa_t addr, unsigned int len, 118 unsigned long val) 119{ 120 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 121 int i; 122 unsigned long flags; 123 124 for_each_set_bit(i, &val, len * 8) { 125 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 126 127 raw_spin_lock_irqsave(&irq->irq_lock, flags); 128 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 129 if (!irq->enabled) { 130 struct irq_data *data; 131 132 irq->enabled = true; 133 data = &irq_to_desc(irq->host_irq)->irq_data; 134 while (irqd_irq_disabled(data)) 135 enable_irq(irq->host_irq); 136 } 137 138 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 139 vgic_put_irq(vcpu->kvm, irq); 140 141 continue; 142 } else if (vgic_irq_is_mapped_level(irq)) { 143 bool was_high = irq->line_level; 144 145 /* 146 * We need to update the state of the interrupt because 147 * the guest might have changed the state of the device 148 * while the interrupt was disabled at the VGIC level. 149 */ 150 irq->line_level = vgic_get_phys_line_level(irq); 151 /* 152 * Deactivate the physical interrupt so the GIC will let 153 * us know when it is asserted again. 154 */ 155 if (!irq->active && was_high && !irq->line_level) 156 vgic_irq_set_phys_active(irq, false); 157 } 158 irq->enabled = true; 159 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 160 161 vgic_put_irq(vcpu->kvm, irq); 162 } 163} 164 165void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu, 166 gpa_t addr, unsigned int len, 167 unsigned long val) 168{ 169 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 170 int i; 171 unsigned long flags; 172 173 for_each_set_bit(i, &val, len * 8) { 174 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 175 176 raw_spin_lock_irqsave(&irq->irq_lock, flags); 177 if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled) 178 disable_irq_nosync(irq->host_irq); 179 180 irq->enabled = false; 181 182 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 183 vgic_put_irq(vcpu->kvm, irq); 184 } 185} 186 187int vgic_uaccess_write_senable(struct kvm_vcpu *vcpu, 188 gpa_t addr, unsigned int len, 189 unsigned long val) 190{ 191 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 192 int i; 193 unsigned long flags; 194 195 for_each_set_bit(i, &val, len * 8) { 196 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 197 198 raw_spin_lock_irqsave(&irq->irq_lock, flags); 199 irq->enabled = true; 200 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 201 202 vgic_put_irq(vcpu->kvm, irq); 203 } 204 205 return 0; 206} 207 208int vgic_uaccess_write_cenable(struct kvm_vcpu *vcpu, 209 gpa_t addr, unsigned int len, 210 unsigned long val) 211{ 212 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 213 int i; 214 unsigned long flags; 215 216 for_each_set_bit(i, &val, len * 8) { 217 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 218 219 raw_spin_lock_irqsave(&irq->irq_lock, flags); 220 irq->enabled = false; 221 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 222 223 vgic_put_irq(vcpu->kvm, irq); 224 } 225 226 return 0; 227} 228 229static unsigned long __read_pending(struct kvm_vcpu *vcpu, 230 gpa_t addr, unsigned int len, 231 bool is_user) 232{ 233 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 234 u32 value = 0; 235 int i; 236 237 /* Loop over all IRQs affected by this read */ 238 for (i = 0; i < len * 8; i++) { 239 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 240 unsigned long flags; 241 bool val; 242 243 raw_spin_lock_irqsave(&irq->irq_lock, flags); 244 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 245 int err; 246 247 val = false; 248 err = irq_get_irqchip_state(irq->host_irq, 249 IRQCHIP_STATE_PENDING, 250 &val); 251 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 252 } else if (!is_user && vgic_irq_is_mapped_level(irq)) { 253 val = vgic_get_phys_line_level(irq); 254 } else { 255 val = irq_is_pending(irq); 256 } 257 258 value |= ((u32)val << i); 259 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 260 261 vgic_put_irq(vcpu->kvm, irq); 262 } 263 264 return value; 265} 266 267unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu, 268 gpa_t addr, unsigned int len) 269{ 270 return __read_pending(vcpu, addr, len, false); 271} 272 273unsigned long vgic_uaccess_read_pending(struct kvm_vcpu *vcpu, 274 gpa_t addr, unsigned int len) 275{ 276 return __read_pending(vcpu, addr, len, true); 277} 278 279static bool is_vgic_v2_sgi(struct kvm_vcpu *vcpu, struct vgic_irq *irq) 280{ 281 return (vgic_irq_is_sgi(irq->intid) && 282 vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2); 283} 284 285void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, 286 gpa_t addr, unsigned int len, 287 unsigned long val) 288{ 289 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 290 int i; 291 unsigned long flags; 292 293 for_each_set_bit(i, &val, len * 8) { 294 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 295 296 /* GICD_ISPENDR0 SGI bits are WI */ 297 if (is_vgic_v2_sgi(vcpu, irq)) { 298 vgic_put_irq(vcpu->kvm, irq); 299 continue; 300 } 301 302 raw_spin_lock_irqsave(&irq->irq_lock, flags); 303 304 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 305 /* HW SGI? Ask the GIC to inject it */ 306 int err; 307 err = irq_set_irqchip_state(irq->host_irq, 308 IRQCHIP_STATE_PENDING, 309 true); 310 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 311 312 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 313 vgic_put_irq(vcpu->kvm, irq); 314 315 continue; 316 } 317 318 irq->pending_latch = true; 319 if (irq->hw) 320 vgic_irq_set_phys_active(irq, true); 321 322 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 323 vgic_put_irq(vcpu->kvm, irq); 324 } 325} 326 327int vgic_uaccess_write_spending(struct kvm_vcpu *vcpu, 328 gpa_t addr, unsigned int len, 329 unsigned long val) 330{ 331 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 332 int i; 333 unsigned long flags; 334 335 for_each_set_bit(i, &val, len * 8) { 336 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 337 338 raw_spin_lock_irqsave(&irq->irq_lock, flags); 339 irq->pending_latch = true; 340 341 /* 342 * GICv2 SGIs are terribly broken. We can't restore 343 * the source of the interrupt, so just pick the vcpu 344 * itself as the source... 345 */ 346 if (is_vgic_v2_sgi(vcpu, irq)) 347 irq->source |= BIT(vcpu->vcpu_id); 348 349 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 350 351 vgic_put_irq(vcpu->kvm, irq); 352 } 353 354 return 0; 355} 356 357/* Must be called with irq->irq_lock held */ 358static void vgic_hw_irq_cpending(struct kvm_vcpu *vcpu, struct vgic_irq *irq) 359{ 360 irq->pending_latch = false; 361 362 /* 363 * We don't want the guest to effectively mask the physical 364 * interrupt by doing a write to SPENDR followed by a write to 365 * CPENDR for HW interrupts, so we clear the active state on 366 * the physical side if the virtual interrupt is not active. 367 * This may lead to taking an additional interrupt on the 368 * host, but that should not be a problem as the worst that 369 * can happen is an additional vgic injection. We also clear 370 * the pending state to maintain proper semantics for edge HW 371 * interrupts. 372 */ 373 vgic_irq_set_phys_pending(irq, false); 374 if (!irq->active) 375 vgic_irq_set_phys_active(irq, false); 376} 377 378void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, 379 gpa_t addr, unsigned int len, 380 unsigned long val) 381{ 382 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 383 int i; 384 unsigned long flags; 385 386 for_each_set_bit(i, &val, len * 8) { 387 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 388 389 /* GICD_ICPENDR0 SGI bits are WI */ 390 if (is_vgic_v2_sgi(vcpu, irq)) { 391 vgic_put_irq(vcpu->kvm, irq); 392 continue; 393 } 394 395 raw_spin_lock_irqsave(&irq->irq_lock, flags); 396 397 if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 398 /* HW SGI? Ask the GIC to clear its pending bit */ 399 int err; 400 err = irq_set_irqchip_state(irq->host_irq, 401 IRQCHIP_STATE_PENDING, 402 false); 403 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq); 404 405 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 406 vgic_put_irq(vcpu->kvm, irq); 407 408 continue; 409 } 410 411 if (irq->hw) 412 vgic_hw_irq_cpending(vcpu, irq); 413 else 414 irq->pending_latch = false; 415 416 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 417 vgic_put_irq(vcpu->kvm, irq); 418 } 419} 420 421int vgic_uaccess_write_cpending(struct kvm_vcpu *vcpu, 422 gpa_t addr, unsigned int len, 423 unsigned long val) 424{ 425 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 426 int i; 427 unsigned long flags; 428 429 for_each_set_bit(i, &val, len * 8) { 430 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 431 432 raw_spin_lock_irqsave(&irq->irq_lock, flags); 433 /* 434 * More fun with GICv2 SGIs! If we're clearing one of them 435 * from userspace, which source vcpu to clear? Let's not 436 * even think of it, and blow the whole set. 437 */ 438 if (is_vgic_v2_sgi(vcpu, irq)) 439 irq->source = 0; 440 441 irq->pending_latch = false; 442 443 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 444 445 vgic_put_irq(vcpu->kvm, irq); 446 } 447 448 return 0; 449} 450 451/* 452 * If we are fiddling with an IRQ's active state, we have to make sure the IRQ 453 * is not queued on some running VCPU's LRs, because then the change to the 454 * active state can be overwritten when the VCPU's state is synced coming back 455 * from the guest. 456 * 457 * For shared interrupts as well as GICv3 private interrupts, we have to 458 * stop all the VCPUs because interrupts can be migrated while we don't hold 459 * the IRQ locks and we don't want to be chasing moving targets. 460 * 461 * For GICv2 private interrupts we don't have to do anything because 462 * userspace accesses to the VGIC state already require all VCPUs to be 463 * stopped, and only the VCPU itself can modify its private interrupts 464 * active state, which guarantees that the VCPU is not running. 465 */ 466static void vgic_access_active_prepare(struct kvm_vcpu *vcpu, u32 intid) 467{ 468 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 || 469 intid >= VGIC_NR_PRIVATE_IRQS) 470 kvm_arm_halt_guest(vcpu->kvm); 471} 472 473/* See vgic_access_active_prepare */ 474static void vgic_access_active_finish(struct kvm_vcpu *vcpu, u32 intid) 475{ 476 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 || 477 intid >= VGIC_NR_PRIVATE_IRQS) 478 kvm_arm_resume_guest(vcpu->kvm); 479} 480 481static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu, 482 gpa_t addr, unsigned int len) 483{ 484 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 485 u32 value = 0; 486 int i; 487 488 /* Loop over all IRQs affected by this read */ 489 for (i = 0; i < len * 8; i++) { 490 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 491 492 /* 493 * Even for HW interrupts, don't evaluate the HW state as 494 * all the guest is interested in is the virtual state. 495 */ 496 if (irq->active) 497 value |= (1U << i); 498 499 vgic_put_irq(vcpu->kvm, irq); 500 } 501 502 return value; 503} 504 505unsigned long vgic_mmio_read_active(struct kvm_vcpu *vcpu, 506 gpa_t addr, unsigned int len) 507{ 508 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 509 u32 val; 510 511 mutex_lock(&vcpu->kvm->lock); 512 vgic_access_active_prepare(vcpu, intid); 513 514 val = __vgic_mmio_read_active(vcpu, addr, len); 515 516 vgic_access_active_finish(vcpu, intid); 517 mutex_unlock(&vcpu->kvm->lock); 518 519 return val; 520} 521 522unsigned long vgic_uaccess_read_active(struct kvm_vcpu *vcpu, 523 gpa_t addr, unsigned int len) 524{ 525 return __vgic_mmio_read_active(vcpu, addr, len); 526} 527 528/* Must be called with irq->irq_lock held */ 529static void vgic_hw_irq_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, 530 bool active, bool is_uaccess) 531{ 532 if (is_uaccess) 533 return; 534 535 irq->active = active; 536 vgic_irq_set_phys_active(irq, active); 537} 538 539static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, 540 bool active) 541{ 542 unsigned long flags; 543 struct kvm_vcpu *requester_vcpu = kvm_get_running_vcpu(); 544 545 raw_spin_lock_irqsave(&irq->irq_lock, flags); 546 547 if (irq->hw && !vgic_irq_is_sgi(irq->intid)) { 548 vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu); 549 } else if (irq->hw && vgic_irq_is_sgi(irq->intid)) { 550 /* 551 * GICv4.1 VSGI feature doesn't track an active state, 552 * so let's not kid ourselves, there is nothing we can 553 * do here. 554 */ 555 irq->active = false; 556 } else { 557 u32 model = vcpu->kvm->arch.vgic.vgic_model; 558 u8 active_source; 559 560 irq->active = active; 561 562 /* 563 * The GICv2 architecture indicates that the source CPUID for 564 * an SGI should be provided during an EOI which implies that 565 * the active state is stored somewhere, but at the same time 566 * this state is not architecturally exposed anywhere and we 567 * have no way of knowing the right source. 568 * 569 * This may lead to a VCPU not being able to receive 570 * additional instances of a particular SGI after migration 571 * for a GICv2 VM on some GIC implementations. Oh well. 572 */ 573 active_source = (requester_vcpu) ? requester_vcpu->vcpu_id : 0; 574 575 if (model == KVM_DEV_TYPE_ARM_VGIC_V2 && 576 active && vgic_irq_is_sgi(irq->intid)) 577 irq->active_source = active_source; 578 } 579 580 if (irq->active) 581 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 582 else 583 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 584} 585 586static void __vgic_mmio_write_cactive(struct kvm_vcpu *vcpu, 587 gpa_t addr, unsigned int len, 588 unsigned long val) 589{ 590 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 591 int i; 592 593 for_each_set_bit(i, &val, len * 8) { 594 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 595 vgic_mmio_change_active(vcpu, irq, false); 596 vgic_put_irq(vcpu->kvm, irq); 597 } 598} 599 600void vgic_mmio_write_cactive(struct kvm_vcpu *vcpu, 601 gpa_t addr, unsigned int len, 602 unsigned long val) 603{ 604 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 605 606 mutex_lock(&vcpu->kvm->lock); 607 vgic_access_active_prepare(vcpu, intid); 608 609 __vgic_mmio_write_cactive(vcpu, addr, len, val); 610 611 vgic_access_active_finish(vcpu, intid); 612 mutex_unlock(&vcpu->kvm->lock); 613} 614 615int vgic_mmio_uaccess_write_cactive(struct kvm_vcpu *vcpu, 616 gpa_t addr, unsigned int len, 617 unsigned long val) 618{ 619 __vgic_mmio_write_cactive(vcpu, addr, len, val); 620 return 0; 621} 622 623static void __vgic_mmio_write_sactive(struct kvm_vcpu *vcpu, 624 gpa_t addr, unsigned int len, 625 unsigned long val) 626{ 627 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 628 int i; 629 630 for_each_set_bit(i, &val, len * 8) { 631 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 632 vgic_mmio_change_active(vcpu, irq, true); 633 vgic_put_irq(vcpu->kvm, irq); 634 } 635} 636 637void vgic_mmio_write_sactive(struct kvm_vcpu *vcpu, 638 gpa_t addr, unsigned int len, 639 unsigned long val) 640{ 641 u32 intid = VGIC_ADDR_TO_INTID(addr, 1); 642 643 mutex_lock(&vcpu->kvm->lock); 644 vgic_access_active_prepare(vcpu, intid); 645 646 __vgic_mmio_write_sactive(vcpu, addr, len, val); 647 648 vgic_access_active_finish(vcpu, intid); 649 mutex_unlock(&vcpu->kvm->lock); 650} 651 652int vgic_mmio_uaccess_write_sactive(struct kvm_vcpu *vcpu, 653 gpa_t addr, unsigned int len, 654 unsigned long val) 655{ 656 __vgic_mmio_write_sactive(vcpu, addr, len, val); 657 return 0; 658} 659 660unsigned long vgic_mmio_read_priority(struct kvm_vcpu *vcpu, 661 gpa_t addr, unsigned int len) 662{ 663 u32 intid = VGIC_ADDR_TO_INTID(addr, 8); 664 int i; 665 u64 val = 0; 666 667 for (i = 0; i < len; i++) { 668 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 669 670 val |= (u64)irq->priority << (i * 8); 671 672 vgic_put_irq(vcpu->kvm, irq); 673 } 674 675 return val; 676} 677 678/* 679 * We currently don't handle changing the priority of an interrupt that 680 * is already pending on a VCPU. If there is a need for this, we would 681 * need to make this VCPU exit and re-evaluate the priorities, potentially 682 * leading to this interrupt getting presented now to the guest (if it has 683 * been masked by the priority mask before). 684 */ 685void vgic_mmio_write_priority(struct kvm_vcpu *vcpu, 686 gpa_t addr, unsigned int len, 687 unsigned long val) 688{ 689 u32 intid = VGIC_ADDR_TO_INTID(addr, 8); 690 int i; 691 unsigned long flags; 692 693 for (i = 0; i < len; i++) { 694 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 695 696 raw_spin_lock_irqsave(&irq->irq_lock, flags); 697 /* Narrow the priority range to what we actually support */ 698 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); 699 if (irq->hw && vgic_irq_is_sgi(irq->intid)) 700 vgic_update_vsgi(irq); 701 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 702 703 vgic_put_irq(vcpu->kvm, irq); 704 } 705} 706 707unsigned long vgic_mmio_read_config(struct kvm_vcpu *vcpu, 708 gpa_t addr, unsigned int len) 709{ 710 u32 intid = VGIC_ADDR_TO_INTID(addr, 2); 711 u32 value = 0; 712 int i; 713 714 for (i = 0; i < len * 4; i++) { 715 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 716 717 if (irq->config == VGIC_CONFIG_EDGE) 718 value |= (2U << (i * 2)); 719 720 vgic_put_irq(vcpu->kvm, irq); 721 } 722 723 return value; 724} 725 726void vgic_mmio_write_config(struct kvm_vcpu *vcpu, 727 gpa_t addr, unsigned int len, 728 unsigned long val) 729{ 730 u32 intid = VGIC_ADDR_TO_INTID(addr, 2); 731 int i; 732 unsigned long flags; 733 734 for (i = 0; i < len * 4; i++) { 735 struct vgic_irq *irq; 736 737 /* 738 * The configuration cannot be changed for SGIs in general, 739 * for PPIs this is IMPLEMENTATION DEFINED. The arch timer 740 * code relies on PPIs being level triggered, so we also 741 * make them read-only here. 742 */ 743 if (intid + i < VGIC_NR_PRIVATE_IRQS) 744 continue; 745 746 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 747 raw_spin_lock_irqsave(&irq->irq_lock, flags); 748 749 if (test_bit(i * 2 + 1, &val)) 750 irq->config = VGIC_CONFIG_EDGE; 751 else 752 irq->config = VGIC_CONFIG_LEVEL; 753 754 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 755 vgic_put_irq(vcpu->kvm, irq); 756 } 757} 758 759u64 vgic_read_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid) 760{ 761 int i; 762 u64 val = 0; 763 int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 764 765 for (i = 0; i < 32; i++) { 766 struct vgic_irq *irq; 767 768 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs) 769 continue; 770 771 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 772 if (irq->config == VGIC_CONFIG_LEVEL && irq->line_level) 773 val |= (1U << i); 774 775 vgic_put_irq(vcpu->kvm, irq); 776 } 777 778 return val; 779} 780 781void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid, 782 const u64 val) 783{ 784 int i; 785 int nr_irqs = vcpu->kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 786 unsigned long flags; 787 788 for (i = 0; i < 32; i++) { 789 struct vgic_irq *irq; 790 bool new_level; 791 792 if ((intid + i) < VGIC_NR_SGIS || (intid + i) >= nr_irqs) 793 continue; 794 795 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); 796 797 /* 798 * Line level is set irrespective of irq type 799 * (level or edge) to avoid dependency that VM should 800 * restore irq config before line level. 801 */ 802 new_level = !!(val & (1U << i)); 803 raw_spin_lock_irqsave(&irq->irq_lock, flags); 804 irq->line_level = new_level; 805 if (new_level) 806 vgic_queue_irq_unlock(vcpu->kvm, irq, flags); 807 else 808 raw_spin_unlock_irqrestore(&irq->irq_lock, flags); 809 810 vgic_put_irq(vcpu->kvm, irq); 811 } 812} 813 814static int match_region(const void *key, const void *elt) 815{ 816 const unsigned int offset = (unsigned long)key; 817 const struct vgic_register_region *region = elt; 818 819 if (offset < region->reg_offset) 820 return -1; 821 822 if (offset >= region->reg_offset + region->len) 823 return 1; 824 825 return 0; 826} 827 828const struct vgic_register_region * 829vgic_find_mmio_region(const struct vgic_register_region *regions, 830 int nr_regions, unsigned int offset) 831{ 832 return bsearch((void *)(uintptr_t)offset, regions, nr_regions, 833 sizeof(regions[0]), match_region); 834} 835 836void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) 837{ 838 if (kvm_vgic_global_state.type == VGIC_V2) 839 vgic_v2_set_vmcr(vcpu, vmcr); 840 else 841 vgic_v3_set_vmcr(vcpu, vmcr); 842} 843 844void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) 845{ 846 if (kvm_vgic_global_state.type == VGIC_V2) 847 vgic_v2_get_vmcr(vcpu, vmcr); 848 else 849 vgic_v3_get_vmcr(vcpu, vmcr); 850} 851 852/* 853 * kvm_mmio_read_buf() returns a value in a format where it can be converted 854 * to a byte array and be directly observed as the guest wanted it to appear 855 * in memory if it had done the store itself, which is LE for the GIC, as the 856 * guest knows the GIC is always LE. 857 * 858 * We convert this value to the CPUs native format to deal with it as a data 859 * value. 860 */ 861unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len) 862{ 863 unsigned long data = kvm_mmio_read_buf(val, len); 864 865 switch (len) { 866 case 1: 867 return data; 868 case 2: 869 return le16_to_cpu(data); 870 case 4: 871 return le32_to_cpu(data); 872 default: 873 return le64_to_cpu(data); 874 } 875} 876 877/* 878 * kvm_mmio_write_buf() expects a value in a format such that if converted to 879 * a byte array it is observed as the guest would see it if it could perform 880 * the load directly. Since the GIC is LE, and the guest knows this, the 881 * guest expects a value in little endian format. 882 * 883 * We convert the data value from the CPUs native format to LE so that the 884 * value is returned in the proper format. 885 */ 886void vgic_data_host_to_mmio_bus(void *buf, unsigned int len, 887 unsigned long data) 888{ 889 switch (len) { 890 case 1: 891 break; 892 case 2: 893 data = cpu_to_le16(data); 894 break; 895 case 4: 896 data = cpu_to_le32(data); 897 break; 898 default: 899 data = cpu_to_le64(data); 900 } 901 902 kvm_mmio_write_buf(buf, len, data); 903} 904 905static 906struct vgic_io_device *kvm_to_vgic_iodev(const struct kvm_io_device *dev) 907{ 908 return container_of(dev, struct vgic_io_device, dev); 909} 910 911static bool check_region(const struct kvm *kvm, 912 const struct vgic_register_region *region, 913 gpa_t addr, int len) 914{ 915 int flags, nr_irqs = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; 916 917 switch (len) { 918 case sizeof(u8): 919 flags = VGIC_ACCESS_8bit; 920 break; 921 case sizeof(u32): 922 flags = VGIC_ACCESS_32bit; 923 break; 924 case sizeof(u64): 925 flags = VGIC_ACCESS_64bit; 926 break; 927 default: 928 return false; 929 } 930 931 if ((region->access_flags & flags) && IS_ALIGNED(addr, len)) { 932 if (!region->bits_per_irq) 933 return true; 934 935 /* Do we access a non-allocated IRQ? */ 936 return VGIC_ADDR_TO_INTID(addr, region->bits_per_irq) < nr_irqs; 937 } 938 939 return false; 940} 941 942const struct vgic_register_region * 943vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, 944 gpa_t addr, int len) 945{ 946 const struct vgic_register_region *region; 947 948 region = vgic_find_mmio_region(iodev->regions, iodev->nr_regions, 949 addr - iodev->base_addr); 950 if (!region || !check_region(vcpu->kvm, region, addr, len)) 951 return NULL; 952 953 return region; 954} 955 956static int vgic_uaccess_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 957 gpa_t addr, u32 *val) 958{ 959 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 960 const struct vgic_register_region *region; 961 struct kvm_vcpu *r_vcpu; 962 963 region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32)); 964 if (!region) { 965 *val = 0; 966 return 0; 967 } 968 969 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu; 970 if (region->uaccess_read) 971 *val = region->uaccess_read(r_vcpu, addr, sizeof(u32)); 972 else 973 *val = region->read(r_vcpu, addr, sizeof(u32)); 974 975 return 0; 976} 977 978static int vgic_uaccess_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 979 gpa_t addr, const u32 *val) 980{ 981 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 982 const struct vgic_register_region *region; 983 struct kvm_vcpu *r_vcpu; 984 985 region = vgic_get_mmio_region(vcpu, iodev, addr, sizeof(u32)); 986 if (!region) 987 return 0; 988 989 r_vcpu = iodev->redist_vcpu ? iodev->redist_vcpu : vcpu; 990 if (region->uaccess_write) 991 return region->uaccess_write(r_vcpu, addr, sizeof(u32), *val); 992 993 region->write(r_vcpu, addr, sizeof(u32), *val); 994 return 0; 995} 996 997/* 998 * Userland access to VGIC registers. 999 */ 1000int vgic_uaccess(struct kvm_vcpu *vcpu, struct vgic_io_device *dev, 1001 bool is_write, int offset, u32 *val) 1002{ 1003 if (is_write) 1004 return vgic_uaccess_write(vcpu, &dev->dev, offset, val); 1005 else 1006 return vgic_uaccess_read(vcpu, &dev->dev, offset, val); 1007} 1008 1009static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 1010 gpa_t addr, int len, void *val) 1011{ 1012 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 1013 const struct vgic_register_region *region; 1014 unsigned long data = 0; 1015 1016 region = vgic_get_mmio_region(vcpu, iodev, addr, len); 1017 if (!region) { 1018 memset(val, 0, len); 1019 return 0; 1020 } 1021 1022 switch (iodev->iodev_type) { 1023 case IODEV_CPUIF: 1024 data = region->read(vcpu, addr, len); 1025 break; 1026 case IODEV_DIST: 1027 data = region->read(vcpu, addr, len); 1028 break; 1029 case IODEV_REDIST: 1030 data = region->read(iodev->redist_vcpu, addr, len); 1031 break; 1032 case IODEV_ITS: 1033 data = region->its_read(vcpu->kvm, iodev->its, addr, len); 1034 break; 1035 } 1036 1037 vgic_data_host_to_mmio_bus(val, len, data); 1038 return 0; 1039} 1040 1041static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, 1042 gpa_t addr, int len, const void *val) 1043{ 1044 struct vgic_io_device *iodev = kvm_to_vgic_iodev(dev); 1045 const struct vgic_register_region *region; 1046 unsigned long data = vgic_data_mmio_bus_to_host(val, len); 1047 1048 region = vgic_get_mmio_region(vcpu, iodev, addr, len); 1049 if (!region) 1050 return 0; 1051 1052 switch (iodev->iodev_type) { 1053 case IODEV_CPUIF: 1054 region->write(vcpu, addr, len, data); 1055 break; 1056 case IODEV_DIST: 1057 region->write(vcpu, addr, len, data); 1058 break; 1059 case IODEV_REDIST: 1060 region->write(iodev->redist_vcpu, addr, len, data); 1061 break; 1062 case IODEV_ITS: 1063 region->its_write(vcpu->kvm, iodev->its, addr, len, data); 1064 break; 1065 } 1066 1067 return 0; 1068} 1069 1070struct kvm_io_device_ops kvm_io_gic_ops = { 1071 .read = dispatch_mmio_read, 1072 .write = dispatch_mmio_write, 1073}; 1074 1075int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address, 1076 enum vgic_type type) 1077{ 1078 struct vgic_io_device *io_device = &kvm->arch.vgic.dist_iodev; 1079 int ret = 0; 1080 unsigned int len; 1081 1082 switch (type) { 1083 case VGIC_V2: 1084 len = vgic_v2_init_dist_iodev(io_device); 1085 break; 1086 case VGIC_V3: 1087 len = vgic_v3_init_dist_iodev(io_device); 1088 break; 1089 default: 1090 BUG_ON(1); 1091 } 1092 1093 io_device->base_addr = dist_base_address; 1094 io_device->iodev_type = IODEV_DIST; 1095 io_device->redist_vcpu = NULL; 1096 1097 mutex_lock(&kvm->slots_lock); 1098 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, dist_base_address, 1099 len, &io_device->dev); 1100 mutex_unlock(&kvm->slots_lock); 1101 1102 return ret; 1103} 1104