1// SPDX-License-Identifier: GPL-2.0 2/* 3 * handling kvm guest interrupts 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 */ 9 10#define KMSG_COMPONENT "kvm-s390" 11#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 12 13#include <linux/interrupt.h> 14#include <linux/kvm_host.h> 15#include <linux/hrtimer.h> 16#include <linux/mmu_context.h> 17#include <linux/nospec.h> 18#include <linux/signal.h> 19#include <linux/slab.h> 20#include <linux/bitmap.h> 21#include <linux/vmalloc.h> 22#include <asm/asm-offsets.h> 23#include <asm/dis.h> 24#include <linux/uaccess.h> 25#include <asm/sclp.h> 26#include <asm/isc.h> 27#include <asm/gmap.h> 28#include <asm/switch_to.h> 29#include <asm/nmi.h> 30#include <asm/airq.h> 31#include "kvm-s390.h" 32#include "gaccess.h" 33#include "trace-s390.h" 34 35#define PFAULT_INIT 0x0600 36#define PFAULT_DONE 0x0680 37#define VIRTIO_PARAM 0x0d00 38 39static struct kvm_s390_gib *gib; 40 41/* handle external calls via sigp interpretation facility */ 42static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) 43{ 44 int c, scn; 45 46 if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND)) 47 return 0; 48 49 BUG_ON(!kvm_s390_use_sca_entries()); 50 read_lock(&vcpu->kvm->arch.sca_lock); 51 if (vcpu->kvm->arch.use_esca) { 52 struct esca_block *sca = vcpu->kvm->arch.sca; 53 union esca_sigp_ctrl sigp_ctrl = 54 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 55 56 c = sigp_ctrl.c; 57 scn = sigp_ctrl.scn; 58 } else { 59 struct bsca_block *sca = vcpu->kvm->arch.sca; 60 union bsca_sigp_ctrl sigp_ctrl = 61 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 62 63 c = sigp_ctrl.c; 64 scn = sigp_ctrl.scn; 65 } 66 read_unlock(&vcpu->kvm->arch.sca_lock); 67 68 if (src_id) 69 *src_id = scn; 70 71 return c; 72} 73 74static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) 75{ 76 int expect, rc; 77 78 BUG_ON(!kvm_s390_use_sca_entries()); 79 read_lock(&vcpu->kvm->arch.sca_lock); 80 if (vcpu->kvm->arch.use_esca) { 81 struct esca_block *sca = vcpu->kvm->arch.sca; 82 union esca_sigp_ctrl *sigp_ctrl = 83 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 84 union esca_sigp_ctrl new_val = {0}, old_val; 85 86 old_val = READ_ONCE(*sigp_ctrl); 87 new_val.scn = src_id; 88 new_val.c = 1; 89 old_val.c = 0; 90 91 expect = old_val.value; 92 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 93 } else { 94 struct bsca_block *sca = vcpu->kvm->arch.sca; 95 union bsca_sigp_ctrl *sigp_ctrl = 96 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 97 union bsca_sigp_ctrl new_val = {0}, old_val; 98 99 old_val = READ_ONCE(*sigp_ctrl); 100 new_val.scn = src_id; 101 new_val.c = 1; 102 old_val.c = 0; 103 104 expect = old_val.value; 105 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 106 } 107 read_unlock(&vcpu->kvm->arch.sca_lock); 108 109 if (rc != expect) { 110 /* another external call is pending */ 111 return -EBUSY; 112 } 113 kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND); 114 return 0; 115} 116 117static void sca_clear_ext_call(struct kvm_vcpu *vcpu) 118{ 119 int rc, expect; 120 121 if (!kvm_s390_use_sca_entries()) 122 return; 123 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND); 124 read_lock(&vcpu->kvm->arch.sca_lock); 125 if (vcpu->kvm->arch.use_esca) { 126 struct esca_block *sca = vcpu->kvm->arch.sca; 127 union esca_sigp_ctrl *sigp_ctrl = 128 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 129 union esca_sigp_ctrl old; 130 131 old = READ_ONCE(*sigp_ctrl); 132 expect = old.value; 133 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 134 } else { 135 struct bsca_block *sca = vcpu->kvm->arch.sca; 136 union bsca_sigp_ctrl *sigp_ctrl = 137 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 138 union bsca_sigp_ctrl old; 139 140 old = READ_ONCE(*sigp_ctrl); 141 expect = old.value; 142 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 143 } 144 read_unlock(&vcpu->kvm->arch.sca_lock); 145 WARN_ON(rc != expect); /* cannot clear? */ 146} 147 148int psw_extint_disabled(struct kvm_vcpu *vcpu) 149{ 150 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 151} 152 153static int psw_ioint_disabled(struct kvm_vcpu *vcpu) 154{ 155 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO); 156} 157 158static int psw_mchk_disabled(struct kvm_vcpu *vcpu) 159{ 160 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK); 161} 162 163static int psw_interrupts_disabled(struct kvm_vcpu *vcpu) 164{ 165 return psw_extint_disabled(vcpu) && 166 psw_ioint_disabled(vcpu) && 167 psw_mchk_disabled(vcpu); 168} 169 170static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu) 171{ 172 if (psw_extint_disabled(vcpu) || 173 !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK)) 174 return 0; 175 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu)) 176 /* No timer interrupts when single stepping */ 177 return 0; 178 return 1; 179} 180 181static int ckc_irq_pending(struct kvm_vcpu *vcpu) 182{ 183 const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 184 const u64 ckc = vcpu->arch.sie_block->ckc; 185 186 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) { 187 if ((s64)ckc >= (s64)now) 188 return 0; 189 } else if (ckc >= now) { 190 return 0; 191 } 192 return ckc_interrupts_enabled(vcpu); 193} 194 195static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu) 196{ 197 return !psw_extint_disabled(vcpu) && 198 (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK); 199} 200 201static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu) 202{ 203 if (!cpu_timer_interrupts_enabled(vcpu)) 204 return 0; 205 return kvm_s390_get_cpu_timer(vcpu) >> 63; 206} 207 208static uint64_t isc_to_isc_bits(int isc) 209{ 210 return (0x80 >> isc) << 24; 211} 212 213static inline u32 isc_to_int_word(u8 isc) 214{ 215 return ((u32)isc << 27) | 0x80000000; 216} 217 218static inline u8 int_word_to_isc(u32 int_word) 219{ 220 return (int_word & 0x38000000) >> 27; 221} 222 223/* 224 * To use atomic bitmap functions, we have to provide a bitmap address 225 * that is u64 aligned. However, the ipm might be u32 aligned. 226 * Therefore, we logically start the bitmap at the very beginning of the 227 * struct and fixup the bit number. 228 */ 229#define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE) 230 231/** 232 * gisa_set_iam - change the GISA interruption alert mask 233 * 234 * @gisa: gisa to operate on 235 * @iam: new IAM value to use 236 * 237 * Change the IAM atomically with the next alert address and the IPM 238 * of the GISA if the GISA is not part of the GIB alert list. All three 239 * fields are located in the first long word of the GISA. 240 * 241 * Returns: 0 on success 242 * -EBUSY in case the gisa is part of the alert list 243 */ 244static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam) 245{ 246 u64 word, _word; 247 248 do { 249 word = READ_ONCE(gisa->u64.word[0]); 250 if ((u64)gisa != word >> 32) 251 return -EBUSY; 252 _word = (word & ~0xffUL) | iam; 253 } while (cmpxchg(&gisa->u64.word[0], word, _word) != word); 254 255 return 0; 256} 257 258/** 259 * gisa_clear_ipm - clear the GISA interruption pending mask 260 * 261 * @gisa: gisa to operate on 262 * 263 * Clear the IPM atomically with the next alert address and the IAM 264 * of the GISA unconditionally. All three fields are located in the 265 * first long word of the GISA. 266 */ 267static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa) 268{ 269 u64 word, _word; 270 271 do { 272 word = READ_ONCE(gisa->u64.word[0]); 273 _word = word & ~(0xffUL << 24); 274 } while (cmpxchg(&gisa->u64.word[0], word, _word) != word); 275} 276 277/** 278 * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM 279 * 280 * @gi: gisa interrupt struct to work on 281 * 282 * Atomically restores the interruption alert mask if none of the 283 * relevant ISCs are pending and return the IPM. 284 * 285 * Returns: the relevant pending ISCs 286 */ 287static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi) 288{ 289 u8 pending_mask, alert_mask; 290 u64 word, _word; 291 292 do { 293 word = READ_ONCE(gi->origin->u64.word[0]); 294 alert_mask = READ_ONCE(gi->alert.mask); 295 pending_mask = (u8)(word >> 24) & alert_mask; 296 if (pending_mask) 297 return pending_mask; 298 _word = (word & ~0xffUL) | alert_mask; 299 } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word); 300 301 return 0; 302} 303 304static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa) 305{ 306 return READ_ONCE(gisa->next_alert) != (u32)(u64)gisa; 307} 308 309static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) 310{ 311 set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); 312} 313 314static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa) 315{ 316 return READ_ONCE(gisa->ipm); 317} 318 319static inline void gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) 320{ 321 clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); 322} 323 324static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) 325{ 326 return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); 327} 328 329static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) 330{ 331 unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs | 332 vcpu->arch.local_int.pending_irqs; 333 334 pending &= ~vcpu->kvm->arch.float_int.masked_irqs; 335 return pending; 336} 337 338static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 339{ 340 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; 341 unsigned long pending_mask; 342 343 pending_mask = pending_irqs_no_gisa(vcpu); 344 if (gi->origin) 345 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7; 346 return pending_mask; 347} 348 349static inline int isc_to_irq_type(unsigned long isc) 350{ 351 return IRQ_PEND_IO_ISC_0 - isc; 352} 353 354static inline int irq_type_to_isc(unsigned long irq_type) 355{ 356 return IRQ_PEND_IO_ISC_0 - irq_type; 357} 358 359static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 360 unsigned long active_mask) 361{ 362 int i; 363 364 for (i = 0; i <= MAX_ISC; i++) 365 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i))) 366 active_mask &= ~(1UL << (isc_to_irq_type(i))); 367 368 return active_mask; 369} 370 371static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) 372{ 373 unsigned long active_mask; 374 375 active_mask = pending_irqs(vcpu); 376 if (!active_mask) 377 return 0; 378 379 if (psw_extint_disabled(vcpu)) 380 active_mask &= ~IRQ_PEND_EXT_MASK; 381 if (psw_ioint_disabled(vcpu)) 382 active_mask &= ~IRQ_PEND_IO_MASK; 383 else 384 active_mask = disable_iscs(vcpu, active_mask); 385 if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK)) 386 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 387 if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK)) 388 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask); 389 if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK)) 390 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 391 if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK)) 392 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 393 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) { 394 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); 395 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask); 396 } 397 if (psw_mchk_disabled(vcpu)) 398 active_mask &= ~IRQ_PEND_MCHK_MASK; 399 /* PV guest cpus can have a single interruption injected at a time. */ 400 if (kvm_s390_pv_cpu_get_handle(vcpu) && 401 vcpu->arch.sie_block->iictl != IICTL_CODE_NONE) 402 active_mask &= ~(IRQ_PEND_EXT_II_MASK | 403 IRQ_PEND_IO_MASK | 404 IRQ_PEND_MCHK_MASK); 405 /* 406 * Check both floating and local interrupt's cr14 because 407 * bit IRQ_PEND_MCHK_REP could be set in both cases. 408 */ 409 if (!(vcpu->arch.sie_block->gcr[14] & 410 (vcpu->kvm->arch.float_int.mchk.cr14 | 411 vcpu->arch.local_int.irq.mchk.cr14))) 412 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask); 413 414 /* 415 * STOP irqs will never be actively delivered. They are triggered via 416 * intercept requests and cleared when the stop intercept is performed. 417 */ 418 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask); 419 420 return active_mask; 421} 422 423static void __set_cpu_idle(struct kvm_vcpu *vcpu) 424{ 425 kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT); 426 set_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.idle_mask); 427} 428 429static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 430{ 431 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT); 432 clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.idle_mask); 433} 434 435static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 436{ 437 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT | 438 CPUSTAT_STOP_INT); 439 vcpu->arch.sie_block->lctl = 0x0000; 440 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 441 442 if (guestdbg_enabled(vcpu)) { 443 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 | 444 LCTL_CR10 | LCTL_CR11); 445 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT); 446 } 447} 448 449static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 450{ 451 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK)) 452 return; 453 if (psw_ioint_disabled(vcpu)) 454 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT); 455 else 456 vcpu->arch.sie_block->lctl |= LCTL_CR6; 457} 458 459static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 460{ 461 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK)) 462 return; 463 if (psw_extint_disabled(vcpu)) 464 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 465 else 466 vcpu->arch.sie_block->lctl |= LCTL_CR0; 467} 468 469static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) 470{ 471 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK)) 472 return; 473 if (psw_mchk_disabled(vcpu)) 474 vcpu->arch.sie_block->ictl |= ICTL_LPSW; 475 else 476 vcpu->arch.sie_block->lctl |= LCTL_CR14; 477} 478 479static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 480{ 481 if (kvm_s390_is_stop_irq_pending(vcpu)) 482 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); 483} 484 485/* Set interception request for non-deliverable interrupts */ 486static void set_intercept_indicators(struct kvm_vcpu *vcpu) 487{ 488 set_intercept_indicators_io(vcpu); 489 set_intercept_indicators_ext(vcpu); 490 set_intercept_indicators_mchk(vcpu); 491 set_intercept_indicators_stop(vcpu); 492} 493 494static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) 495{ 496 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 497 int rc = 0; 498 499 vcpu->stat.deliver_cputm++; 500 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 501 0, 0); 502 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 503 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 504 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER; 505 } else { 506 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, 507 (u16 *)__LC_EXT_INT_CODE); 508 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 509 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 510 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 511 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 512 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 513 } 514 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 515 return rc ? -EFAULT : 0; 516} 517 518static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) 519{ 520 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 521 int rc = 0; 522 523 vcpu->stat.deliver_ckc++; 524 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 525 0, 0); 526 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 527 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 528 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP; 529 } else { 530 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, 531 (u16 __user *)__LC_EXT_INT_CODE); 532 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 533 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 534 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 535 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 536 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 537 } 538 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 539 return rc ? -EFAULT : 0; 540} 541 542static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu) 543{ 544 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 545 struct kvm_s390_ext_info ext; 546 int rc; 547 548 spin_lock(&li->lock); 549 ext = li->irq.ext; 550 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 551 li->irq.ext.ext_params2 = 0; 552 spin_unlock(&li->lock); 553 554 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx", 555 ext.ext_params2); 556 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 557 KVM_S390_INT_PFAULT_INIT, 558 0, ext.ext_params2); 559 560 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE); 561 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR); 562 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 563 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 564 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 565 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 566 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2); 567 return rc ? -EFAULT : 0; 568} 569 570static int __write_machine_check(struct kvm_vcpu *vcpu, 571 struct kvm_s390_mchk_info *mchk) 572{ 573 unsigned long ext_sa_addr; 574 unsigned long lc; 575 freg_t fprs[NUM_FPRS]; 576 union mci mci; 577 int rc; 578 579 /* 580 * All other possible payload for a machine check (e.g. the register 581 * contents in the save area) will be handled by the ultravisor, as 582 * the hypervisor does not not have the needed information for 583 * protected guests. 584 */ 585 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 586 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK; 587 vcpu->arch.sie_block->mcic = mchk->mcic; 588 vcpu->arch.sie_block->faddr = mchk->failing_storage_address; 589 vcpu->arch.sie_block->edc = mchk->ext_damage_code; 590 return 0; 591 } 592 593 mci.val = mchk->mcic; 594 /* take care of lazy register loading */ 595 save_fpu_regs(); 596 save_access_regs(vcpu->run->s.regs.acrs); 597 if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) 598 save_gs_cb(current->thread.gs_cb); 599 600 /* Extended save area */ 601 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr, 602 sizeof(unsigned long)); 603 /* Only bits 0 through 63-LC are used for address formation */ 604 lc = ext_sa_addr & MCESA_LC_MASK; 605 if (test_kvm_facility(vcpu->kvm, 133)) { 606 switch (lc) { 607 case 0: 608 case 10: 609 ext_sa_addr &= ~0x3ffUL; 610 break; 611 case 11: 612 ext_sa_addr &= ~0x7ffUL; 613 break; 614 case 12: 615 ext_sa_addr &= ~0xfffUL; 616 break; 617 default: 618 ext_sa_addr = 0; 619 break; 620 } 621 } else { 622 ext_sa_addr &= ~0x3ffUL; 623 } 624 625 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) { 626 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs, 627 512)) 628 mci.vr = 0; 629 } else { 630 mci.vr = 0; 631 } 632 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133) 633 && (lc == 11 || lc == 12)) { 634 if (write_guest_abs(vcpu, ext_sa_addr + 1024, 635 &vcpu->run->s.regs.gscb, 32)) 636 mci.gs = 0; 637 } else { 638 mci.gs = 0; 639 } 640 641 /* General interruption information */ 642 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID); 643 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 644 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 645 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 646 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 647 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE); 648 649 /* Register-save areas */ 650 if (MACHINE_HAS_VX) { 651 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 652 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128); 653 } else { 654 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, 655 vcpu->run->s.regs.fprs, 128); 656 } 657 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, 658 vcpu->run->s.regs.gprs, 128); 659 rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, 660 (u32 __user *) __LC_FP_CREG_SAVE_AREA); 661 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, 662 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); 663 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu), 664 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA); 665 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8, 666 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA); 667 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA, 668 &vcpu->run->s.regs.acrs, 64); 669 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA, 670 &vcpu->arch.sie_block->gcr, 128); 671 672 /* Extended interruption information */ 673 rc |= put_guest_lc(vcpu, mchk->ext_damage_code, 674 (u32 __user *) __LC_EXT_DAMAGE_CODE); 675 rc |= put_guest_lc(vcpu, mchk->failing_storage_address, 676 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 677 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout, 678 sizeof(mchk->fixed_logout)); 679 return rc ? -EFAULT : 0; 680} 681 682static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 683{ 684 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 685 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 686 struct kvm_s390_mchk_info mchk = {}; 687 int deliver = 0; 688 int rc = 0; 689 690 spin_lock(&fi->lock); 691 spin_lock(&li->lock); 692 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || 693 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { 694 /* 695 * If there was an exigent machine check pending, then any 696 * repressible machine checks that might have been pending 697 * are indicated along with it, so always clear bits for 698 * repressible and exigent interrupts 699 */ 700 mchk = li->irq.mchk; 701 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 702 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 703 memset(&li->irq.mchk, 0, sizeof(mchk)); 704 deliver = 1; 705 } 706 /* 707 * We indicate floating repressible conditions along with 708 * other pending conditions. Channel Report Pending and Channel 709 * Subsystem damage are the only two and and are indicated by 710 * bits in mcic and masked in cr14. 711 */ 712 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 713 mchk.mcic |= fi->mchk.mcic; 714 mchk.cr14 |= fi->mchk.cr14; 715 memset(&fi->mchk, 0, sizeof(mchk)); 716 deliver = 1; 717 } 718 spin_unlock(&li->lock); 719 spin_unlock(&fi->lock); 720 721 if (deliver) { 722 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", 723 mchk.mcic); 724 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 725 KVM_S390_MCHK, 726 mchk.cr14, mchk.mcic); 727 vcpu->stat.deliver_machine_check++; 728 rc = __write_machine_check(vcpu, &mchk); 729 } 730 return rc; 731} 732 733static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) 734{ 735 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 736 int rc = 0; 737 738 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); 739 vcpu->stat.deliver_restart_signal++; 740 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 741 742 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 743 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART; 744 } else { 745 rc = write_guest_lc(vcpu, 746 offsetof(struct lowcore, restart_old_psw), 747 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 748 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), 749 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 750 } 751 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); 752 return rc ? -EFAULT : 0; 753} 754 755static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu) 756{ 757 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 758 struct kvm_s390_prefix_info prefix; 759 760 spin_lock(&li->lock); 761 prefix = li->irq.prefix; 762 li->irq.prefix.address = 0; 763 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 764 spin_unlock(&li->lock); 765 766 vcpu->stat.deliver_prefix_signal++; 767 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 768 KVM_S390_SIGP_SET_PREFIX, 769 prefix.address, 0); 770 771 kvm_s390_set_prefix(vcpu, prefix.address); 772 return 0; 773} 774 775static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) 776{ 777 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 778 int rc; 779 int cpu_addr; 780 781 spin_lock(&li->lock); 782 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS); 783 clear_bit(cpu_addr, li->sigp_emerg_pending); 784 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS)) 785 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 786 spin_unlock(&li->lock); 787 788 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg"); 789 vcpu->stat.deliver_emergency_signal++; 790 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 791 cpu_addr, 0); 792 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 793 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 794 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG; 795 vcpu->arch.sie_block->extcpuaddr = cpu_addr; 796 return 0; 797 } 798 799 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, 800 (u16 *)__LC_EXT_INT_CODE); 801 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR); 802 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 803 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 804 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 805 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 806 return rc ? -EFAULT : 0; 807} 808 809static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) 810{ 811 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 812 struct kvm_s390_extcall_info extcall; 813 int rc; 814 815 spin_lock(&li->lock); 816 extcall = li->irq.extcall; 817 li->irq.extcall.code = 0; 818 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 819 spin_unlock(&li->lock); 820 821 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call"); 822 vcpu->stat.deliver_external_call++; 823 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 824 KVM_S390_INT_EXTERNAL_CALL, 825 extcall.code, 0); 826 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 827 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 828 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL; 829 vcpu->arch.sie_block->extcpuaddr = extcall.code; 830 return 0; 831 } 832 833 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, 834 (u16 *)__LC_EXT_INT_CODE); 835 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR); 836 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 837 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 838 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw, 839 sizeof(psw_t)); 840 return rc ? -EFAULT : 0; 841} 842 843static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code) 844{ 845 switch (code) { 846 case PGM_SPECIFICATION: 847 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION; 848 break; 849 case PGM_OPERAND: 850 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND; 851 break; 852 default: 853 return -EINVAL; 854 } 855 return 0; 856} 857 858static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) 859{ 860 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 861 struct kvm_s390_pgm_info pgm_info; 862 int rc = 0, nullifying = false; 863 u16 ilen; 864 865 spin_lock(&li->lock); 866 pgm_info = li->irq.pgm; 867 clear_bit(IRQ_PEND_PROG, &li->pending_irqs); 868 memset(&li->irq.pgm, 0, sizeof(pgm_info)); 869 spin_unlock(&li->lock); 870 871 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK; 872 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d", 873 pgm_info.code, ilen); 874 vcpu->stat.deliver_program++; 875 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 876 pgm_info.code, 0); 877 878 /* PER is handled by the ultravisor */ 879 if (kvm_s390_pv_cpu_is_protected(vcpu)) 880 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER); 881 882 switch (pgm_info.code & ~PGM_PER) { 883 case PGM_AFX_TRANSLATION: 884 case PGM_ASX_TRANSLATION: 885 case PGM_EX_TRANSLATION: 886 case PGM_LFX_TRANSLATION: 887 case PGM_LSTE_SEQUENCE: 888 case PGM_LSX_TRANSLATION: 889 case PGM_LX_TRANSLATION: 890 case PGM_PRIMARY_AUTHORITY: 891 case PGM_SECONDARY_AUTHORITY: 892 nullifying = true; 893 fallthrough; 894 case PGM_SPACE_SWITCH: 895 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 896 (u64 *)__LC_TRANS_EXC_CODE); 897 break; 898 case PGM_ALEN_TRANSLATION: 899 case PGM_ALE_SEQUENCE: 900 case PGM_ASTE_INSTANCE: 901 case PGM_ASTE_SEQUENCE: 902 case PGM_ASTE_VALIDITY: 903 case PGM_EXTENDED_AUTHORITY: 904 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 905 (u8 *)__LC_EXC_ACCESS_ID); 906 nullifying = true; 907 break; 908 case PGM_ASCE_TYPE: 909 case PGM_PAGE_TRANSLATION: 910 case PGM_REGION_FIRST_TRANS: 911 case PGM_REGION_SECOND_TRANS: 912 case PGM_REGION_THIRD_TRANS: 913 case PGM_SEGMENT_TRANSLATION: 914 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 915 (u64 *)__LC_TRANS_EXC_CODE); 916 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 917 (u8 *)__LC_EXC_ACCESS_ID); 918 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 919 (u8 *)__LC_OP_ACCESS_ID); 920 nullifying = true; 921 break; 922 case PGM_MONITOR: 923 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 924 (u16 *)__LC_MON_CLASS_NR); 925 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 926 (u64 *)__LC_MON_CODE); 927 break; 928 case PGM_VECTOR_PROCESSING: 929 case PGM_DATA: 930 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 931 (u32 *)__LC_DATA_EXC_CODE); 932 break; 933 case PGM_PROTECTION: 934 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 935 (u64 *)__LC_TRANS_EXC_CODE); 936 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 937 (u8 *)__LC_EXC_ACCESS_ID); 938 break; 939 case PGM_STACK_FULL: 940 case PGM_STACK_EMPTY: 941 case PGM_STACK_SPECIFICATION: 942 case PGM_STACK_TYPE: 943 case PGM_STACK_OPERATION: 944 case PGM_TRACE_TABEL: 945 case PGM_CRYPTO_OPERATION: 946 nullifying = true; 947 break; 948 } 949 950 if (pgm_info.code & PGM_PER) { 951 rc |= put_guest_lc(vcpu, pgm_info.per_code, 952 (u8 *) __LC_PER_CODE); 953 rc |= put_guest_lc(vcpu, pgm_info.per_atmid, 954 (u8 *)__LC_PER_ATMID); 955 rc |= put_guest_lc(vcpu, pgm_info.per_address, 956 (u64 *) __LC_PER_ADDRESS); 957 rc |= put_guest_lc(vcpu, pgm_info.per_access_id, 958 (u8 *) __LC_PER_ACCESS_ID); 959 } 960 961 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND)) 962 kvm_s390_rewind_psw(vcpu, ilen); 963 964 /* bit 1+2 of the target are the ilc, so we can directly use ilen */ 965 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC); 966 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea, 967 (u64 *) __LC_LAST_BREAK); 968 rc |= put_guest_lc(vcpu, pgm_info.code, 969 (u16 *)__LC_PGM_INT_CODE); 970 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 971 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 972 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW, 973 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 974 return rc ? -EFAULT : 0; 975} 976 977#define SCCB_MASK 0xFFFFFFF8 978#define SCCB_EVENT_PENDING 0x3 979 980static int write_sclp(struct kvm_vcpu *vcpu, u32 parm) 981{ 982 int rc; 983 984 if (kvm_s390_pv_cpu_get_handle(vcpu)) { 985 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 986 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG; 987 vcpu->arch.sie_block->eiparams = parm; 988 return 0; 989 } 990 991 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 992 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 993 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 994 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 995 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 996 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 997 rc |= put_guest_lc(vcpu, parm, 998 (u32 *)__LC_EXT_PARAMS); 999 1000 return rc ? -EFAULT : 0; 1001} 1002 1003static int __must_check __deliver_service(struct kvm_vcpu *vcpu) 1004{ 1005 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 1006 struct kvm_s390_ext_info ext; 1007 1008 spin_lock(&fi->lock); 1009 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || 1010 !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { 1011 spin_unlock(&fi->lock); 1012 return 0; 1013 } 1014 ext = fi->srv_signal; 1015 memset(&fi->srv_signal, 0, sizeof(ext)); 1016 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1017 clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); 1018 if (kvm_s390_pv_cpu_is_protected(vcpu)) 1019 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); 1020 spin_unlock(&fi->lock); 1021 1022 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", 1023 ext.ext_params); 1024 vcpu->stat.deliver_service_signal++; 1025 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 1026 ext.ext_params, 0); 1027 1028 return write_sclp(vcpu, ext.ext_params); 1029} 1030 1031static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) 1032{ 1033 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 1034 struct kvm_s390_ext_info ext; 1035 1036 spin_lock(&fi->lock); 1037 if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { 1038 spin_unlock(&fi->lock); 1039 return 0; 1040 } 1041 ext = fi->srv_signal; 1042 /* only clear the event bit */ 1043 fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; 1044 clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); 1045 spin_unlock(&fi->lock); 1046 1047 VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); 1048 vcpu->stat.deliver_service_signal++; 1049 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 1050 ext.ext_params, 0); 1051 1052 return write_sclp(vcpu, SCCB_EVENT_PENDING); 1053} 1054 1055static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) 1056{ 1057 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 1058 struct kvm_s390_interrupt_info *inti; 1059 int rc = 0; 1060 1061 spin_lock(&fi->lock); 1062 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], 1063 struct kvm_s390_interrupt_info, 1064 list); 1065 if (inti) { 1066 list_del(&inti->list); 1067 fi->counters[FIRQ_CNTR_PFAULT] -= 1; 1068 } 1069 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) 1070 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1071 spin_unlock(&fi->lock); 1072 1073 if (inti) { 1074 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1075 KVM_S390_INT_PFAULT_DONE, 0, 1076 inti->ext.ext_params2); 1077 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx", 1078 inti->ext.ext_params2); 1079 1080 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 1081 (u16 *)__LC_EXT_INT_CODE); 1082 rc |= put_guest_lc(vcpu, PFAULT_DONE, 1083 (u16 *)__LC_EXT_CPU_ADDR); 1084 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 1085 &vcpu->arch.sie_block->gpsw, 1086 sizeof(psw_t)); 1087 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 1088 &vcpu->arch.sie_block->gpsw, 1089 sizeof(psw_t)); 1090 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 1091 (u64 *)__LC_EXT_PARAMS2); 1092 kfree(inti); 1093 } 1094 return rc ? -EFAULT : 0; 1095} 1096 1097static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) 1098{ 1099 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 1100 struct kvm_s390_interrupt_info *inti; 1101 int rc = 0; 1102 1103 spin_lock(&fi->lock); 1104 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], 1105 struct kvm_s390_interrupt_info, 1106 list); 1107 if (inti) { 1108 VCPU_EVENT(vcpu, 4, 1109 "deliver: virtio parm: 0x%x,parm64: 0x%llx", 1110 inti->ext.ext_params, inti->ext.ext_params2); 1111 vcpu->stat.deliver_virtio++; 1112 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1113 inti->type, 1114 inti->ext.ext_params, 1115 inti->ext.ext_params2); 1116 list_del(&inti->list); 1117 fi->counters[FIRQ_CNTR_VIRTIO] -= 1; 1118 } 1119 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) 1120 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1121 spin_unlock(&fi->lock); 1122 1123 if (inti) { 1124 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 1125 (u16 *)__LC_EXT_INT_CODE); 1126 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, 1127 (u16 *)__LC_EXT_CPU_ADDR); 1128 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 1129 &vcpu->arch.sie_block->gpsw, 1130 sizeof(psw_t)); 1131 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 1132 &vcpu->arch.sie_block->gpsw, 1133 sizeof(psw_t)); 1134 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 1135 (u32 *)__LC_EXT_PARAMS); 1136 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 1137 (u64 *)__LC_EXT_PARAMS2); 1138 kfree(inti); 1139 } 1140 return rc ? -EFAULT : 0; 1141} 1142 1143static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io) 1144{ 1145 int rc; 1146 1147 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 1148 vcpu->arch.sie_block->iictl = IICTL_CODE_IO; 1149 vcpu->arch.sie_block->subchannel_id = io->subchannel_id; 1150 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr; 1151 vcpu->arch.sie_block->io_int_parm = io->io_int_parm; 1152 vcpu->arch.sie_block->io_int_word = io->io_int_word; 1153 return 0; 1154 } 1155 1156 rc = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID); 1157 rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR); 1158 rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM); 1159 rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD); 1160 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW, 1161 &vcpu->arch.sie_block->gpsw, 1162 sizeof(psw_t)); 1163 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW, 1164 &vcpu->arch.sie_block->gpsw, 1165 sizeof(psw_t)); 1166 return rc ? -EFAULT : 0; 1167} 1168 1169static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 1170 unsigned long irq_type) 1171{ 1172 struct list_head *isc_list; 1173 struct kvm_s390_float_interrupt *fi; 1174 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; 1175 struct kvm_s390_interrupt_info *inti = NULL; 1176 struct kvm_s390_io_info io; 1177 u32 isc; 1178 int rc = 0; 1179 1180 fi = &vcpu->kvm->arch.float_int; 1181 1182 spin_lock(&fi->lock); 1183 isc = irq_type_to_isc(irq_type); 1184 isc_list = &fi->lists[isc]; 1185 inti = list_first_entry_or_null(isc_list, 1186 struct kvm_s390_interrupt_info, 1187 list); 1188 if (inti) { 1189 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1190 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)"); 1191 else 1192 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x", 1193 inti->io.subchannel_id >> 8, 1194 inti->io.subchannel_id >> 1 & 0x3, 1195 inti->io.subchannel_nr); 1196 1197 vcpu->stat.deliver_io++; 1198 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1199 inti->type, 1200 ((__u32)inti->io.subchannel_id << 16) | 1201 inti->io.subchannel_nr, 1202 ((__u64)inti->io.io_int_parm << 32) | 1203 inti->io.io_int_word); 1204 list_del(&inti->list); 1205 fi->counters[FIRQ_CNTR_IO] -= 1; 1206 } 1207 if (list_empty(isc_list)) 1208 clear_bit(irq_type, &fi->pending_irqs); 1209 spin_unlock(&fi->lock); 1210 1211 if (inti) { 1212 rc = __do_deliver_io(vcpu, &(inti->io)); 1213 kfree(inti); 1214 goto out; 1215 } 1216 1217 if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) { 1218 /* 1219 * in case an adapter interrupt was not delivered 1220 * in SIE context KVM will handle the delivery 1221 */ 1222 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc); 1223 memset(&io, 0, sizeof(io)); 1224 io.io_int_word = isc_to_int_word(isc); 1225 vcpu->stat.deliver_io++; 1226 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1227 KVM_S390_INT_IO(1, 0, 0, 0), 1228 ((__u32)io.subchannel_id << 16) | 1229 io.subchannel_nr, 1230 ((__u64)io.io_int_parm << 32) | 1231 io.io_int_word); 1232 rc = __do_deliver_io(vcpu, &io); 1233 } 1234out: 1235 return rc; 1236} 1237 1238/* Check whether an external call is pending (deliverable or not) */ 1239int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 1240{ 1241 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1242 1243 if (!sclp.has_sigpif) 1244 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 1245 1246 return sca_ext_call_pending(vcpu, NULL); 1247} 1248 1249int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 1250{ 1251 if (deliverable_irqs(vcpu)) 1252 return 1; 1253 1254 if (kvm_cpu_has_pending_timer(vcpu)) 1255 return 1; 1256 1257 /* external call pending and deliverable */ 1258 if (kvm_s390_ext_call_pending(vcpu) && 1259 !psw_extint_disabled(vcpu) && 1260 (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK)) 1261 return 1; 1262 1263 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu)) 1264 return 1; 1265 return 0; 1266} 1267 1268int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 1269{ 1270 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu); 1271} 1272 1273static u64 __calculate_sltime(struct kvm_vcpu *vcpu) 1274{ 1275 const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 1276 const u64 ckc = vcpu->arch.sie_block->ckc; 1277 u64 cputm, sltime = 0; 1278 1279 if (ckc_interrupts_enabled(vcpu)) { 1280 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) { 1281 if ((s64)now < (s64)ckc) 1282 sltime = tod_to_ns((s64)ckc - (s64)now); 1283 } else if (now < ckc) { 1284 sltime = tod_to_ns(ckc - now); 1285 } 1286 /* already expired */ 1287 if (!sltime) 1288 return 0; 1289 if (cpu_timer_interrupts_enabled(vcpu)) { 1290 cputm = kvm_s390_get_cpu_timer(vcpu); 1291 /* already expired? */ 1292 if (cputm >> 63) 1293 return 0; 1294 return min(sltime, tod_to_ns(cputm)); 1295 } 1296 } else if (cpu_timer_interrupts_enabled(vcpu)) { 1297 sltime = kvm_s390_get_cpu_timer(vcpu); 1298 /* already expired? */ 1299 if (sltime >> 63) 1300 return 0; 1301 } 1302 return sltime; 1303} 1304 1305int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 1306{ 1307 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; 1308 u64 sltime; 1309 1310 vcpu->stat.exit_wait_state++; 1311 1312 /* fast path */ 1313 if (kvm_arch_vcpu_runnable(vcpu)) 1314 return 0; 1315 1316 if (psw_interrupts_disabled(vcpu)) { 1317 VCPU_EVENT(vcpu, 3, "%s", "disabled wait"); 1318 return -EOPNOTSUPP; /* disabled wait */ 1319 } 1320 1321 if (gi->origin && 1322 (gisa_get_ipm_or_restore_iam(gi) & 1323 vcpu->arch.sie_block->gcr[6] >> 24)) 1324 return 0; 1325 1326 if (!ckc_interrupts_enabled(vcpu) && 1327 !cpu_timer_interrupts_enabled(vcpu)) { 1328 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); 1329 __set_cpu_idle(vcpu); 1330 goto no_timer; 1331 } 1332 1333 sltime = __calculate_sltime(vcpu); 1334 if (!sltime) 1335 return 0; 1336 1337 __set_cpu_idle(vcpu); 1338 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL); 1339 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1340no_timer: 1341 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1342 kvm_vcpu_block(vcpu); 1343 __unset_cpu_idle(vcpu); 1344 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1345 1346 hrtimer_cancel(&vcpu->arch.ckc_timer); 1347 return 0; 1348} 1349 1350void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) 1351{ 1352 vcpu->valid_wakeup = true; 1353 kvm_vcpu_wake_up(vcpu); 1354 1355 /* 1356 * The VCPU might not be sleeping but rather executing VSIE. Let's 1357 * kick it, so it leaves the SIE to process the request. 1358 */ 1359 kvm_s390_vsie_kick(vcpu); 1360} 1361 1362enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer) 1363{ 1364 struct kvm_vcpu *vcpu; 1365 u64 sltime; 1366 1367 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer); 1368 sltime = __calculate_sltime(vcpu); 1369 1370 /* 1371 * If the monotonic clock runs faster than the tod clock we might be 1372 * woken up too early and have to go back to sleep to avoid deadlocks. 1373 */ 1374 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime))) 1375 return HRTIMER_RESTART; 1376 kvm_s390_vcpu_wakeup(vcpu); 1377 return HRTIMER_NORESTART; 1378} 1379 1380void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) 1381{ 1382 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1383 1384 spin_lock(&li->lock); 1385 li->pending_irqs = 0; 1386 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS); 1387 memset(&li->irq, 0, sizeof(li->irq)); 1388 spin_unlock(&li->lock); 1389 1390 sca_clear_ext_call(vcpu); 1391} 1392 1393int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 1394{ 1395 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1396 int rc = 0; 1397 unsigned long irq_type; 1398 unsigned long irqs; 1399 1400 __reset_intercept_indicators(vcpu); 1401 1402 /* pending ckc conditions might have been invalidated */ 1403 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1404 if (ckc_irq_pending(vcpu)) 1405 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1406 1407 /* pending cpu timer conditions might have been invalidated */ 1408 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1409 if (cpu_timer_irq_pending(vcpu)) 1410 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1411 1412 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1413 /* bits are in the reverse order of interrupt priority */ 1414 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT); 1415 switch (irq_type) { 1416 case IRQ_PEND_IO_ISC_0: 1417 case IRQ_PEND_IO_ISC_1: 1418 case IRQ_PEND_IO_ISC_2: 1419 case IRQ_PEND_IO_ISC_3: 1420 case IRQ_PEND_IO_ISC_4: 1421 case IRQ_PEND_IO_ISC_5: 1422 case IRQ_PEND_IO_ISC_6: 1423 case IRQ_PEND_IO_ISC_7: 1424 rc = __deliver_io(vcpu, irq_type); 1425 break; 1426 case IRQ_PEND_MCHK_EX: 1427 case IRQ_PEND_MCHK_REP: 1428 rc = __deliver_machine_check(vcpu); 1429 break; 1430 case IRQ_PEND_PROG: 1431 rc = __deliver_prog(vcpu); 1432 break; 1433 case IRQ_PEND_EXT_EMERGENCY: 1434 rc = __deliver_emergency_signal(vcpu); 1435 break; 1436 case IRQ_PEND_EXT_EXTERNAL: 1437 rc = __deliver_external_call(vcpu); 1438 break; 1439 case IRQ_PEND_EXT_CLOCK_COMP: 1440 rc = __deliver_ckc(vcpu); 1441 break; 1442 case IRQ_PEND_EXT_CPU_TIMER: 1443 rc = __deliver_cpu_timer(vcpu); 1444 break; 1445 case IRQ_PEND_RESTART: 1446 rc = __deliver_restart(vcpu); 1447 break; 1448 case IRQ_PEND_SET_PREFIX: 1449 rc = __deliver_set_prefix(vcpu); 1450 break; 1451 case IRQ_PEND_PFAULT_INIT: 1452 rc = __deliver_pfault_init(vcpu); 1453 break; 1454 case IRQ_PEND_EXT_SERVICE: 1455 rc = __deliver_service(vcpu); 1456 break; 1457 case IRQ_PEND_EXT_SERVICE_EV: 1458 rc = __deliver_service_ev(vcpu); 1459 break; 1460 case IRQ_PEND_PFAULT_DONE: 1461 rc = __deliver_pfault_done(vcpu); 1462 break; 1463 case IRQ_PEND_VIRTIO: 1464 rc = __deliver_virtio(vcpu); 1465 break; 1466 default: 1467 WARN_ONCE(1, "Unknown pending irq type %ld", irq_type); 1468 clear_bit(irq_type, &li->pending_irqs); 1469 } 1470 } 1471 1472 set_intercept_indicators(vcpu); 1473 1474 return rc; 1475} 1476 1477static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1478{ 1479 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1480 1481 vcpu->stat.inject_program++; 1482 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code); 1483 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 1484 irq->u.pgm.code, 0); 1485 1486 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) { 1487 /* auto detection if no valid ILC was given */ 1488 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK; 1489 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu); 1490 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID; 1491 } 1492 1493 if (irq->u.pgm.code == PGM_PER) { 1494 li->irq.pgm.code |= PGM_PER; 1495 li->irq.pgm.flags = irq->u.pgm.flags; 1496 /* only modify PER related information */ 1497 li->irq.pgm.per_address = irq->u.pgm.per_address; 1498 li->irq.pgm.per_code = irq->u.pgm.per_code; 1499 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid; 1500 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id; 1501 } else if (!(irq->u.pgm.code & PGM_PER)) { 1502 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) | 1503 irq->u.pgm.code; 1504 li->irq.pgm.flags = irq->u.pgm.flags; 1505 /* only modify non-PER information */ 1506 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code; 1507 li->irq.pgm.mon_code = irq->u.pgm.mon_code; 1508 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code; 1509 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr; 1510 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id; 1511 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id; 1512 } else { 1513 li->irq.pgm = irq->u.pgm; 1514 } 1515 set_bit(IRQ_PEND_PROG, &li->pending_irqs); 1516 return 0; 1517} 1518 1519static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1520{ 1521 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1522 1523 vcpu->stat.inject_pfault_init++; 1524 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx", 1525 irq->u.ext.ext_params2); 1526 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT, 1527 irq->u.ext.ext_params, 1528 irq->u.ext.ext_params2); 1529 1530 li->irq.ext = irq->u.ext; 1531 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1532 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1533 return 0; 1534} 1535 1536static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1537{ 1538 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1539 struct kvm_s390_extcall_info *extcall = &li->irq.extcall; 1540 uint16_t src_id = irq->u.extcall.code; 1541 1542 vcpu->stat.inject_external_call++; 1543 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u", 1544 src_id); 1545 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, 1546 src_id, 0); 1547 1548 /* sending vcpu invalid */ 1549 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) 1550 return -EINVAL; 1551 1552 if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu)) 1553 return sca_inject_ext_call(vcpu, src_id); 1554 1555 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1556 return -EBUSY; 1557 *extcall = irq->u.extcall; 1558 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1559 return 0; 1560} 1561 1562static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1563{ 1564 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1565 struct kvm_s390_prefix_info *prefix = &li->irq.prefix; 1566 1567 vcpu->stat.inject_set_prefix++; 1568 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x", 1569 irq->u.prefix.address); 1570 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX, 1571 irq->u.prefix.address, 0); 1572 1573 if (!is_vcpu_stopped(vcpu)) 1574 return -EBUSY; 1575 1576 *prefix = irq->u.prefix; 1577 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 1578 return 0; 1579} 1580 1581#define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) 1582static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1583{ 1584 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1585 struct kvm_s390_stop_info *stop = &li->irq.stop; 1586 int rc = 0; 1587 1588 vcpu->stat.inject_stop_signal++; 1589 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); 1590 1591 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) 1592 return -EINVAL; 1593 1594 if (is_vcpu_stopped(vcpu)) { 1595 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) 1596 rc = kvm_s390_store_status_unloaded(vcpu, 1597 KVM_S390_STORE_STATUS_NOADDR); 1598 return rc; 1599 } 1600 1601 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1602 return -EBUSY; 1603 stop->flags = irq->u.stop.flags; 1604 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); 1605 return 0; 1606} 1607 1608static int __inject_sigp_restart(struct kvm_vcpu *vcpu) 1609{ 1610 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1611 1612 vcpu->stat.inject_restart++; 1613 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int"); 1614 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 1615 1616 set_bit(IRQ_PEND_RESTART, &li->pending_irqs); 1617 return 0; 1618} 1619 1620static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, 1621 struct kvm_s390_irq *irq) 1622{ 1623 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1624 1625 vcpu->stat.inject_emergency_signal++; 1626 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u", 1627 irq->u.emerg.code); 1628 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 1629 irq->u.emerg.code, 0); 1630 1631 /* sending vcpu invalid */ 1632 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL) 1633 return -EINVAL; 1634 1635 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1636 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1637 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1638 return 0; 1639} 1640 1641static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1642{ 1643 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1644 struct kvm_s390_mchk_info *mchk = &li->irq.mchk; 1645 1646 vcpu->stat.inject_mchk++; 1647 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx", 1648 irq->u.mchk.mcic); 1649 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0, 1650 irq->u.mchk.mcic); 1651 1652 /* 1653 * Because repressible machine checks can be indicated along with 1654 * exigent machine checks (PoP, Chapter 11, Interruption action) 1655 * we need to combine cr14, mcic and external damage code. 1656 * Failing storage address and the logout area should not be or'ed 1657 * together, we just indicate the last occurrence of the corresponding 1658 * machine check 1659 */ 1660 mchk->cr14 |= irq->u.mchk.cr14; 1661 mchk->mcic |= irq->u.mchk.mcic; 1662 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code; 1663 mchk->failing_storage_address = irq->u.mchk.failing_storage_address; 1664 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout, 1665 sizeof(mchk->fixed_logout)); 1666 if (mchk->mcic & MCHK_EX_MASK) 1667 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 1668 else if (mchk->mcic & MCHK_REP_MASK) 1669 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 1670 return 0; 1671} 1672 1673static int __inject_ckc(struct kvm_vcpu *vcpu) 1674{ 1675 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1676 1677 vcpu->stat.inject_ckc++; 1678 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external"); 1679 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 1680 0, 0); 1681 1682 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1683 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1684 return 0; 1685} 1686 1687static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 1688{ 1689 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1690 1691 vcpu->stat.inject_cputm++; 1692 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external"); 1693 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 1694 0, 0); 1695 1696 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1697 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1698 return 0; 1699} 1700 1701static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, 1702 int isc, u32 schid) 1703{ 1704 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1705 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1706 struct kvm_s390_interrupt_info *iter; 1707 u16 id = (schid & 0xffff0000U) >> 16; 1708 u16 nr = schid & 0x0000ffffU; 1709 1710 spin_lock(&fi->lock); 1711 list_for_each_entry(iter, isc_list, list) { 1712 if (schid && (id != iter->io.subchannel_id || 1713 nr != iter->io.subchannel_nr)) 1714 continue; 1715 /* found an appropriate entry */ 1716 list_del_init(&iter->list); 1717 fi->counters[FIRQ_CNTR_IO] -= 1; 1718 if (list_empty(isc_list)) 1719 clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1720 spin_unlock(&fi->lock); 1721 return iter; 1722 } 1723 spin_unlock(&fi->lock); 1724 return NULL; 1725} 1726 1727static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm, 1728 u64 isc_mask, u32 schid) 1729{ 1730 struct kvm_s390_interrupt_info *inti = NULL; 1731 int isc; 1732 1733 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1734 if (isc_mask & isc_to_isc_bits(isc)) 1735 inti = get_io_int(kvm, isc, schid); 1736 } 1737 return inti; 1738} 1739 1740static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid) 1741{ 1742 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 1743 unsigned long active_mask; 1744 int isc; 1745 1746 if (schid) 1747 goto out; 1748 if (!gi->origin) 1749 goto out; 1750 1751 active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32; 1752 while (active_mask) { 1753 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1); 1754 if (gisa_tac_ipm_gisc(gi->origin, isc)) 1755 return isc; 1756 clear_bit_inv(isc, &active_mask); 1757 } 1758out: 1759 return -EINVAL; 1760} 1761 1762/* 1763 * Dequeue and return an I/O interrupt matching any of the interruption 1764 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1765 * Take into account the interrupts pending in the interrupt list and in GISA. 1766 * 1767 * Note that for a guest that does not enable I/O interrupts 1768 * but relies on TPI, a flood of classic interrupts may starve 1769 * out adapter interrupts on the same isc. Linux does not do 1770 * that, and it is possible to work around the issue by configuring 1771 * different iscs for classic and adapter interrupts in the guest, 1772 * but we may want to revisit this in the future. 1773 */ 1774struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1775 u64 isc_mask, u32 schid) 1776{ 1777 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 1778 struct kvm_s390_interrupt_info *inti, *tmp_inti; 1779 int isc; 1780 1781 inti = get_top_io_int(kvm, isc_mask, schid); 1782 1783 isc = get_top_gisa_isc(kvm, isc_mask, schid); 1784 if (isc < 0) 1785 /* no AI in GISA */ 1786 goto out; 1787 1788 if (!inti) 1789 /* AI in GISA but no classical IO int */ 1790 goto gisa_out; 1791 1792 /* both types of interrupts present */ 1793 if (int_word_to_isc(inti->io.io_int_word) <= isc) { 1794 /* classical IO int with higher priority */ 1795 gisa_set_ipm_gisc(gi->origin, isc); 1796 goto out; 1797 } 1798gisa_out: 1799 tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1800 if (tmp_inti) { 1801 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0); 1802 tmp_inti->io.io_int_word = isc_to_int_word(isc); 1803 if (inti) 1804 kvm_s390_reinject_io_int(kvm, inti); 1805 inti = tmp_inti; 1806 } else 1807 gisa_set_ipm_gisc(gi->origin, isc); 1808out: 1809 return inti; 1810} 1811 1812static int __inject_service(struct kvm *kvm, 1813 struct kvm_s390_interrupt_info *inti) 1814{ 1815 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1816 1817 kvm->stat.inject_service_signal++; 1818 spin_lock(&fi->lock); 1819 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; 1820 1821 /* We always allow events, track them separately from the sccb ints */ 1822 if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING) 1823 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); 1824 1825 /* 1826 * Early versions of the QEMU s390 bios will inject several 1827 * service interrupts after another without handling a 1828 * condition code indicating busy. 1829 * We will silently ignore those superfluous sccb values. 1830 * A future version of QEMU will take care of serialization 1831 * of servc requests 1832 */ 1833 if (fi->srv_signal.ext_params & SCCB_MASK) 1834 goto out; 1835 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; 1836 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1837out: 1838 spin_unlock(&fi->lock); 1839 kfree(inti); 1840 return 0; 1841} 1842 1843static int __inject_virtio(struct kvm *kvm, 1844 struct kvm_s390_interrupt_info *inti) 1845{ 1846 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1847 1848 kvm->stat.inject_virtio++; 1849 spin_lock(&fi->lock); 1850 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { 1851 spin_unlock(&fi->lock); 1852 return -EBUSY; 1853 } 1854 fi->counters[FIRQ_CNTR_VIRTIO] += 1; 1855 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); 1856 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1857 spin_unlock(&fi->lock); 1858 return 0; 1859} 1860 1861static int __inject_pfault_done(struct kvm *kvm, 1862 struct kvm_s390_interrupt_info *inti) 1863{ 1864 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1865 1866 kvm->stat.inject_pfault_done++; 1867 spin_lock(&fi->lock); 1868 if (fi->counters[FIRQ_CNTR_PFAULT] >= 1869 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { 1870 spin_unlock(&fi->lock); 1871 return -EBUSY; 1872 } 1873 fi->counters[FIRQ_CNTR_PFAULT] += 1; 1874 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); 1875 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1876 spin_unlock(&fi->lock); 1877 return 0; 1878} 1879 1880#define CR_PENDING_SUBCLASS 28 1881static int __inject_float_mchk(struct kvm *kvm, 1882 struct kvm_s390_interrupt_info *inti) 1883{ 1884 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1885 1886 kvm->stat.inject_float_mchk++; 1887 spin_lock(&fi->lock); 1888 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); 1889 fi->mchk.mcic |= inti->mchk.mcic; 1890 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); 1891 spin_unlock(&fi->lock); 1892 kfree(inti); 1893 return 0; 1894} 1895 1896static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1897{ 1898 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 1899 struct kvm_s390_float_interrupt *fi; 1900 struct list_head *list; 1901 int isc; 1902 1903 kvm->stat.inject_io++; 1904 isc = int_word_to_isc(inti->io.io_int_word); 1905 1906 /* 1907 * Do not make use of gisa in protected mode. We do not use the lock 1908 * checking variant as this is just a performance optimization and we 1909 * do not hold the lock here. This is ok as the code will pick 1910 * interrupts from both "lists" for delivery. 1911 */ 1912 if (!kvm_s390_pv_get_handle(kvm) && 1913 gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) { 1914 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc); 1915 gisa_set_ipm_gisc(gi->origin, isc); 1916 kfree(inti); 1917 return 0; 1918 } 1919 1920 fi = &kvm->arch.float_int; 1921 spin_lock(&fi->lock); 1922 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1923 spin_unlock(&fi->lock); 1924 return -EBUSY; 1925 } 1926 fi->counters[FIRQ_CNTR_IO] += 1; 1927 1928 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1929 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)"); 1930 else 1931 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x", 1932 inti->io.subchannel_id >> 8, 1933 inti->io.subchannel_id >> 1 & 0x3, 1934 inti->io.subchannel_nr); 1935 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1936 list_add_tail(&inti->list, list); 1937 set_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1938 spin_unlock(&fi->lock); 1939 return 0; 1940} 1941 1942/* 1943 * Find a destination VCPU for a floating irq and kick it. 1944 */ 1945static void __floating_irq_kick(struct kvm *kvm, u64 type) 1946{ 1947 struct kvm_vcpu *dst_vcpu; 1948 int sigcpu, online_vcpus, nr_tries = 0; 1949 1950 online_vcpus = atomic_read(&kvm->online_vcpus); 1951 if (!online_vcpus) 1952 return; 1953 1954 /* find idle VCPUs first, then round robin */ 1955 sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus); 1956 if (sigcpu == online_vcpus) { 1957 do { 1958 sigcpu = kvm->arch.float_int.next_rr_cpu++; 1959 kvm->arch.float_int.next_rr_cpu %= online_vcpus; 1960 /* avoid endless loops if all vcpus are stopped */ 1961 if (nr_tries++ >= online_vcpus) 1962 return; 1963 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu))); 1964 } 1965 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1966 1967 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1968 switch (type) { 1969 case KVM_S390_MCHK: 1970 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT); 1971 break; 1972 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1973 if (!(type & KVM_S390_INT_IO_AI_MASK && 1974 kvm->arch.gisa_int.origin) || 1975 kvm_s390_pv_cpu_get_handle(dst_vcpu)) 1976 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); 1977 break; 1978 default: 1979 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT); 1980 break; 1981 } 1982 kvm_s390_vcpu_wakeup(dst_vcpu); 1983} 1984 1985static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1986{ 1987 u64 type = READ_ONCE(inti->type); 1988 int rc; 1989 1990 switch (type) { 1991 case KVM_S390_MCHK: 1992 rc = __inject_float_mchk(kvm, inti); 1993 break; 1994 case KVM_S390_INT_VIRTIO: 1995 rc = __inject_virtio(kvm, inti); 1996 break; 1997 case KVM_S390_INT_SERVICE: 1998 rc = __inject_service(kvm, inti); 1999 break; 2000 case KVM_S390_INT_PFAULT_DONE: 2001 rc = __inject_pfault_done(kvm, inti); 2002 break; 2003 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2004 rc = __inject_io(kvm, inti); 2005 break; 2006 default: 2007 rc = -EINVAL; 2008 } 2009 if (rc) 2010 return rc; 2011 2012 __floating_irq_kick(kvm, type); 2013 return 0; 2014} 2015 2016int kvm_s390_inject_vm(struct kvm *kvm, 2017 struct kvm_s390_interrupt *s390int) 2018{ 2019 struct kvm_s390_interrupt_info *inti; 2020 int rc; 2021 2022 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 2023 if (!inti) 2024 return -ENOMEM; 2025 2026 inti->type = s390int->type; 2027 switch (inti->type) { 2028 case KVM_S390_INT_VIRTIO: 2029 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx", 2030 s390int->parm, s390int->parm64); 2031 inti->ext.ext_params = s390int->parm; 2032 inti->ext.ext_params2 = s390int->parm64; 2033 break; 2034 case KVM_S390_INT_SERVICE: 2035 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm); 2036 inti->ext.ext_params = s390int->parm; 2037 break; 2038 case KVM_S390_INT_PFAULT_DONE: 2039 inti->ext.ext_params2 = s390int->parm64; 2040 break; 2041 case KVM_S390_MCHK: 2042 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx", 2043 s390int->parm64); 2044 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */ 2045 inti->mchk.mcic = s390int->parm64; 2046 break; 2047 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2048 inti->io.subchannel_id = s390int->parm >> 16; 2049 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu; 2050 inti->io.io_int_parm = s390int->parm64 >> 32; 2051 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; 2052 break; 2053 default: 2054 kfree(inti); 2055 return -EINVAL; 2056 } 2057 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 2058 2); 2059 2060 rc = __inject_vm(kvm, inti); 2061 if (rc) 2062 kfree(inti); 2063 return rc; 2064} 2065 2066int kvm_s390_reinject_io_int(struct kvm *kvm, 2067 struct kvm_s390_interrupt_info *inti) 2068{ 2069 return __inject_vm(kvm, inti); 2070} 2071 2072int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 2073 struct kvm_s390_irq *irq) 2074{ 2075 irq->type = s390int->type; 2076 switch (irq->type) { 2077 case KVM_S390_PROGRAM_INT: 2078 if (s390int->parm & 0xffff0000) 2079 return -EINVAL; 2080 irq->u.pgm.code = s390int->parm; 2081 break; 2082 case KVM_S390_SIGP_SET_PREFIX: 2083 irq->u.prefix.address = s390int->parm; 2084 break; 2085 case KVM_S390_SIGP_STOP: 2086 irq->u.stop.flags = s390int->parm; 2087 break; 2088 case KVM_S390_INT_EXTERNAL_CALL: 2089 if (s390int->parm & 0xffff0000) 2090 return -EINVAL; 2091 irq->u.extcall.code = s390int->parm; 2092 break; 2093 case KVM_S390_INT_EMERGENCY: 2094 if (s390int->parm & 0xffff0000) 2095 return -EINVAL; 2096 irq->u.emerg.code = s390int->parm; 2097 break; 2098 case KVM_S390_MCHK: 2099 irq->u.mchk.mcic = s390int->parm64; 2100 break; 2101 case KVM_S390_INT_PFAULT_INIT: 2102 irq->u.ext.ext_params = s390int->parm; 2103 irq->u.ext.ext_params2 = s390int->parm64; 2104 break; 2105 case KVM_S390_RESTART: 2106 case KVM_S390_INT_CLOCK_COMP: 2107 case KVM_S390_INT_CPU_TIMER: 2108 break; 2109 default: 2110 return -EINVAL; 2111 } 2112 return 0; 2113} 2114 2115int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu) 2116{ 2117 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2118 2119 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 2120} 2121 2122int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu) 2123{ 2124 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2125 2126 return test_bit(IRQ_PEND_RESTART, &li->pending_irqs); 2127} 2128 2129void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) 2130{ 2131 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2132 2133 spin_lock(&li->lock); 2134 li->irq.stop.flags = 0; 2135 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 2136 spin_unlock(&li->lock); 2137} 2138 2139static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 2140{ 2141 int rc; 2142 2143 switch (irq->type) { 2144 case KVM_S390_PROGRAM_INT: 2145 rc = __inject_prog(vcpu, irq); 2146 break; 2147 case KVM_S390_SIGP_SET_PREFIX: 2148 rc = __inject_set_prefix(vcpu, irq); 2149 break; 2150 case KVM_S390_SIGP_STOP: 2151 rc = __inject_sigp_stop(vcpu, irq); 2152 break; 2153 case KVM_S390_RESTART: 2154 rc = __inject_sigp_restart(vcpu); 2155 break; 2156 case KVM_S390_INT_CLOCK_COMP: 2157 rc = __inject_ckc(vcpu); 2158 break; 2159 case KVM_S390_INT_CPU_TIMER: 2160 rc = __inject_cpu_timer(vcpu); 2161 break; 2162 case KVM_S390_INT_EXTERNAL_CALL: 2163 rc = __inject_extcall(vcpu, irq); 2164 break; 2165 case KVM_S390_INT_EMERGENCY: 2166 rc = __inject_sigp_emergency(vcpu, irq); 2167 break; 2168 case KVM_S390_MCHK: 2169 rc = __inject_mchk(vcpu, irq); 2170 break; 2171 case KVM_S390_INT_PFAULT_INIT: 2172 rc = __inject_pfault_init(vcpu, irq); 2173 break; 2174 case KVM_S390_INT_VIRTIO: 2175 case KVM_S390_INT_SERVICE: 2176 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2177 default: 2178 rc = -EINVAL; 2179 } 2180 2181 return rc; 2182} 2183 2184int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 2185{ 2186 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2187 int rc; 2188 2189 spin_lock(&li->lock); 2190 rc = do_inject_vcpu(vcpu, irq); 2191 spin_unlock(&li->lock); 2192 if (!rc) 2193 kvm_s390_vcpu_wakeup(vcpu); 2194 return rc; 2195} 2196 2197static inline void clear_irq_list(struct list_head *_list) 2198{ 2199 struct kvm_s390_interrupt_info *inti, *n; 2200 2201 list_for_each_entry_safe(inti, n, _list, list) { 2202 list_del(&inti->list); 2203 kfree(inti); 2204 } 2205} 2206 2207static void inti_to_irq(struct kvm_s390_interrupt_info *inti, 2208 struct kvm_s390_irq *irq) 2209{ 2210 irq->type = inti->type; 2211 switch (inti->type) { 2212 case KVM_S390_INT_PFAULT_INIT: 2213 case KVM_S390_INT_PFAULT_DONE: 2214 case KVM_S390_INT_VIRTIO: 2215 irq->u.ext = inti->ext; 2216 break; 2217 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2218 irq->u.io = inti->io; 2219 break; 2220 } 2221} 2222 2223void kvm_s390_clear_float_irqs(struct kvm *kvm) 2224{ 2225 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2226 int i; 2227 2228 mutex_lock(&kvm->lock); 2229 if (!kvm_s390_pv_is_protected(kvm)) 2230 fi->masked_irqs = 0; 2231 mutex_unlock(&kvm->lock); 2232 spin_lock(&fi->lock); 2233 fi->pending_irqs = 0; 2234 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); 2235 memset(&fi->mchk, 0, sizeof(fi->mchk)); 2236 for (i = 0; i < FIRQ_LIST_COUNT; i++) 2237 clear_irq_list(&fi->lists[i]); 2238 for (i = 0; i < FIRQ_MAX_COUNT; i++) 2239 fi->counters[i] = 0; 2240 spin_unlock(&fi->lock); 2241 kvm_s390_gisa_clear(kvm); 2242}; 2243 2244static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 2245{ 2246 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 2247 struct kvm_s390_interrupt_info *inti; 2248 struct kvm_s390_float_interrupt *fi; 2249 struct kvm_s390_irq *buf; 2250 struct kvm_s390_irq *irq; 2251 int max_irqs; 2252 int ret = 0; 2253 int n = 0; 2254 int i; 2255 2256 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) 2257 return -EINVAL; 2258 2259 /* 2260 * We are already using -ENOMEM to signal 2261 * userspace it may retry with a bigger buffer, 2262 * so we need to use something else for this case 2263 */ 2264 buf = vzalloc(len); 2265 if (!buf) 2266 return -ENOBUFS; 2267 2268 max_irqs = len / sizeof(struct kvm_s390_irq); 2269 2270 if (gi->origin && gisa_get_ipm(gi->origin)) { 2271 for (i = 0; i <= MAX_ISC; i++) { 2272 if (n == max_irqs) { 2273 /* signal userspace to try again */ 2274 ret = -ENOMEM; 2275 goto out_nolock; 2276 } 2277 if (gisa_tac_ipm_gisc(gi->origin, i)) { 2278 irq = (struct kvm_s390_irq *) &buf[n]; 2279 irq->type = KVM_S390_INT_IO(1, 0, 0, 0); 2280 irq->u.io.io_int_word = isc_to_int_word(i); 2281 n++; 2282 } 2283 } 2284 } 2285 fi = &kvm->arch.float_int; 2286 spin_lock(&fi->lock); 2287 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 2288 list_for_each_entry(inti, &fi->lists[i], list) { 2289 if (n == max_irqs) { 2290 /* signal userspace to try again */ 2291 ret = -ENOMEM; 2292 goto out; 2293 } 2294 inti_to_irq(inti, &buf[n]); 2295 n++; 2296 } 2297 } 2298 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) || 2299 test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) { 2300 if (n == max_irqs) { 2301 /* signal userspace to try again */ 2302 ret = -ENOMEM; 2303 goto out; 2304 } 2305 irq = (struct kvm_s390_irq *) &buf[n]; 2306 irq->type = KVM_S390_INT_SERVICE; 2307 irq->u.ext = fi->srv_signal; 2308 n++; 2309 } 2310 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 2311 if (n == max_irqs) { 2312 /* signal userspace to try again */ 2313 ret = -ENOMEM; 2314 goto out; 2315 } 2316 irq = (struct kvm_s390_irq *) &buf[n]; 2317 irq->type = KVM_S390_MCHK; 2318 irq->u.mchk = fi->mchk; 2319 n++; 2320} 2321 2322out: 2323 spin_unlock(&fi->lock); 2324out_nolock: 2325 if (!ret && n > 0) { 2326 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 2327 ret = -EFAULT; 2328 } 2329 vfree(buf); 2330 2331 return ret < 0 ? ret : n; 2332} 2333 2334static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) 2335{ 2336 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2337 struct kvm_s390_ais_all ais; 2338 2339 if (attr->attr < sizeof(ais)) 2340 return -EINVAL; 2341 2342 if (!test_kvm_facility(kvm, 72)) 2343 return -EOPNOTSUPP; 2344 2345 mutex_lock(&fi->ais_lock); 2346 ais.simm = fi->simm; 2347 ais.nimm = fi->nimm; 2348 mutex_unlock(&fi->ais_lock); 2349 2350 if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) 2351 return -EFAULT; 2352 2353 return 0; 2354} 2355 2356static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2357{ 2358 int r; 2359 2360 switch (attr->group) { 2361 case KVM_DEV_FLIC_GET_ALL_IRQS: 2362 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr, 2363 attr->attr); 2364 break; 2365 case KVM_DEV_FLIC_AISM_ALL: 2366 r = flic_ais_mode_get_all(dev->kvm, attr); 2367 break; 2368 default: 2369 r = -EINVAL; 2370 } 2371 2372 return r; 2373} 2374 2375static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti, 2376 u64 addr) 2377{ 2378 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 2379 void *target = NULL; 2380 void __user *source; 2381 u64 size; 2382 2383 if (get_user(inti->type, (u64 __user *)addr)) 2384 return -EFAULT; 2385 2386 switch (inti->type) { 2387 case KVM_S390_INT_PFAULT_INIT: 2388 case KVM_S390_INT_PFAULT_DONE: 2389 case KVM_S390_INT_VIRTIO: 2390 case KVM_S390_INT_SERVICE: 2391 target = (void *) &inti->ext; 2392 source = &uptr->u.ext; 2393 size = sizeof(inti->ext); 2394 break; 2395 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2396 target = (void *) &inti->io; 2397 source = &uptr->u.io; 2398 size = sizeof(inti->io); 2399 break; 2400 case KVM_S390_MCHK: 2401 target = (void *) &inti->mchk; 2402 source = &uptr->u.mchk; 2403 size = sizeof(inti->mchk); 2404 break; 2405 default: 2406 return -EINVAL; 2407 } 2408 2409 if (copy_from_user(target, source, size)) 2410 return -EFAULT; 2411 2412 return 0; 2413} 2414 2415static int enqueue_floating_irq(struct kvm_device *dev, 2416 struct kvm_device_attr *attr) 2417{ 2418 struct kvm_s390_interrupt_info *inti = NULL; 2419 int r = 0; 2420 int len = attr->attr; 2421 2422 if (len % sizeof(struct kvm_s390_irq) != 0) 2423 return -EINVAL; 2424 else if (len > KVM_S390_FLIC_MAX_BUFFER) 2425 return -EINVAL; 2426 2427 while (len >= sizeof(struct kvm_s390_irq)) { 2428 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 2429 if (!inti) 2430 return -ENOMEM; 2431 2432 r = copy_irq_from_user(inti, attr->addr); 2433 if (r) { 2434 kfree(inti); 2435 return r; 2436 } 2437 r = __inject_vm(dev->kvm, inti); 2438 if (r) { 2439 kfree(inti); 2440 return r; 2441 } 2442 len -= sizeof(struct kvm_s390_irq); 2443 attr->addr += sizeof(struct kvm_s390_irq); 2444 } 2445 2446 return r; 2447} 2448 2449static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id) 2450{ 2451 if (id >= MAX_S390_IO_ADAPTERS) 2452 return NULL; 2453 id = array_index_nospec(id, MAX_S390_IO_ADAPTERS); 2454 return kvm->arch.adapters[id]; 2455} 2456 2457static int register_io_adapter(struct kvm_device *dev, 2458 struct kvm_device_attr *attr) 2459{ 2460 struct s390_io_adapter *adapter; 2461 struct kvm_s390_io_adapter adapter_info; 2462 2463 if (copy_from_user(&adapter_info, 2464 (void __user *)attr->addr, sizeof(adapter_info))) 2465 return -EFAULT; 2466 2467 if (adapter_info.id >= MAX_S390_IO_ADAPTERS) 2468 return -EINVAL; 2469 2470 adapter_info.id = array_index_nospec(adapter_info.id, 2471 MAX_S390_IO_ADAPTERS); 2472 2473 if (dev->kvm->arch.adapters[adapter_info.id] != NULL) 2474 return -EINVAL; 2475 2476 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 2477 if (!adapter) 2478 return -ENOMEM; 2479 2480 adapter->id = adapter_info.id; 2481 adapter->isc = adapter_info.isc; 2482 adapter->maskable = adapter_info.maskable; 2483 adapter->masked = false; 2484 adapter->swap = adapter_info.swap; 2485 adapter->suppressible = (adapter_info.flags) & 2486 KVM_S390_ADAPTER_SUPPRESSIBLE; 2487 dev->kvm->arch.adapters[adapter->id] = adapter; 2488 2489 return 0; 2490} 2491 2492int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) 2493{ 2494 int ret; 2495 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2496 2497 if (!adapter || !adapter->maskable) 2498 return -EINVAL; 2499 ret = adapter->masked; 2500 adapter->masked = masked; 2501 return ret; 2502} 2503 2504void kvm_s390_destroy_adapters(struct kvm *kvm) 2505{ 2506 int i; 2507 2508 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) 2509 kfree(kvm->arch.adapters[i]); 2510} 2511 2512static int modify_io_adapter(struct kvm_device *dev, 2513 struct kvm_device_attr *attr) 2514{ 2515 struct kvm_s390_io_adapter_req req; 2516 struct s390_io_adapter *adapter; 2517 int ret; 2518 2519 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2520 return -EFAULT; 2521 2522 adapter = get_io_adapter(dev->kvm, req.id); 2523 if (!adapter) 2524 return -EINVAL; 2525 switch (req.type) { 2526 case KVM_S390_IO_ADAPTER_MASK: 2527 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask); 2528 if (ret > 0) 2529 ret = 0; 2530 break; 2531 /* 2532 * The following operations are no longer needed and therefore no-ops. 2533 * The gpa to hva translation is done when an IRQ route is set up. The 2534 * set_irq code uses get_user_pages_remote() to do the actual write. 2535 */ 2536 case KVM_S390_IO_ADAPTER_MAP: 2537 case KVM_S390_IO_ADAPTER_UNMAP: 2538 ret = 0; 2539 break; 2540 default: 2541 ret = -EINVAL; 2542 } 2543 2544 return ret; 2545} 2546 2547static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) 2548 2549{ 2550 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 2551 u32 schid; 2552 2553 if (attr->flags) 2554 return -EINVAL; 2555 if (attr->attr != sizeof(schid)) 2556 return -EINVAL; 2557 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) 2558 return -EFAULT; 2559 if (!schid) 2560 return -EINVAL; 2561 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); 2562 /* 2563 * If userspace is conforming to the architecture, we can have at most 2564 * one pending I/O interrupt per subchannel, so this is effectively a 2565 * clear all. 2566 */ 2567 return 0; 2568} 2569 2570static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2571{ 2572 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2573 struct kvm_s390_ais_req req; 2574 int ret = 0; 2575 2576 if (!test_kvm_facility(kvm, 72)) 2577 return -EOPNOTSUPP; 2578 2579 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2580 return -EFAULT; 2581 2582 if (req.isc > MAX_ISC) 2583 return -EINVAL; 2584 2585 trace_kvm_s390_modify_ais_mode(req.isc, 2586 (fi->simm & AIS_MODE_MASK(req.isc)) ? 2587 (fi->nimm & AIS_MODE_MASK(req.isc)) ? 2588 2 : KVM_S390_AIS_MODE_SINGLE : 2589 KVM_S390_AIS_MODE_ALL, req.mode); 2590 2591 mutex_lock(&fi->ais_lock); 2592 switch (req.mode) { 2593 case KVM_S390_AIS_MODE_ALL: 2594 fi->simm &= ~AIS_MODE_MASK(req.isc); 2595 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2596 break; 2597 case KVM_S390_AIS_MODE_SINGLE: 2598 fi->simm |= AIS_MODE_MASK(req.isc); 2599 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2600 break; 2601 default: 2602 ret = -EINVAL; 2603 } 2604 mutex_unlock(&fi->ais_lock); 2605 2606 return ret; 2607} 2608 2609static int kvm_s390_inject_airq(struct kvm *kvm, 2610 struct s390_io_adapter *adapter) 2611{ 2612 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2613 struct kvm_s390_interrupt s390int = { 2614 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2615 .parm = 0, 2616 .parm64 = isc_to_int_word(adapter->isc), 2617 }; 2618 int ret = 0; 2619 2620 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) 2621 return kvm_s390_inject_vm(kvm, &s390int); 2622 2623 mutex_lock(&fi->ais_lock); 2624 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 2625 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 2626 goto out; 2627 } 2628 2629 ret = kvm_s390_inject_vm(kvm, &s390int); 2630 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 2631 fi->nimm |= AIS_MODE_MASK(adapter->isc); 2632 trace_kvm_s390_modify_ais_mode(adapter->isc, 2633 KVM_S390_AIS_MODE_SINGLE, 2); 2634 } 2635out: 2636 mutex_unlock(&fi->ais_lock); 2637 return ret; 2638} 2639 2640static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) 2641{ 2642 unsigned int id = attr->attr; 2643 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2644 2645 if (!adapter) 2646 return -EINVAL; 2647 2648 return kvm_s390_inject_airq(kvm, adapter); 2649} 2650 2651static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) 2652{ 2653 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2654 struct kvm_s390_ais_all ais; 2655 2656 if (!test_kvm_facility(kvm, 72)) 2657 return -EOPNOTSUPP; 2658 2659 if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) 2660 return -EFAULT; 2661 2662 mutex_lock(&fi->ais_lock); 2663 fi->simm = ais.simm; 2664 fi->nimm = ais.nimm; 2665 mutex_unlock(&fi->ais_lock); 2666 2667 return 0; 2668} 2669 2670static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2671{ 2672 int r = 0; 2673 unsigned int i; 2674 struct kvm_vcpu *vcpu; 2675 2676 switch (attr->group) { 2677 case KVM_DEV_FLIC_ENQUEUE: 2678 r = enqueue_floating_irq(dev, attr); 2679 break; 2680 case KVM_DEV_FLIC_CLEAR_IRQS: 2681 kvm_s390_clear_float_irqs(dev->kvm); 2682 break; 2683 case KVM_DEV_FLIC_APF_ENABLE: 2684 dev->kvm->arch.gmap->pfault_enabled = 1; 2685 break; 2686 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2687 dev->kvm->arch.gmap->pfault_enabled = 0; 2688 /* 2689 * Make sure no async faults are in transition when 2690 * clearing the queues. So we don't need to worry 2691 * about late coming workers. 2692 */ 2693 synchronize_srcu(&dev->kvm->srcu); 2694 kvm_for_each_vcpu(i, vcpu, dev->kvm) 2695 kvm_clear_async_pf_completion_queue(vcpu); 2696 break; 2697 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2698 r = register_io_adapter(dev, attr); 2699 break; 2700 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2701 r = modify_io_adapter(dev, attr); 2702 break; 2703 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2704 r = clear_io_irq(dev->kvm, attr); 2705 break; 2706 case KVM_DEV_FLIC_AISM: 2707 r = modify_ais_mode(dev->kvm, attr); 2708 break; 2709 case KVM_DEV_FLIC_AIRQ_INJECT: 2710 r = flic_inject_airq(dev->kvm, attr); 2711 break; 2712 case KVM_DEV_FLIC_AISM_ALL: 2713 r = flic_ais_mode_set_all(dev->kvm, attr); 2714 break; 2715 default: 2716 r = -EINVAL; 2717 } 2718 2719 return r; 2720} 2721 2722static int flic_has_attr(struct kvm_device *dev, 2723 struct kvm_device_attr *attr) 2724{ 2725 switch (attr->group) { 2726 case KVM_DEV_FLIC_GET_ALL_IRQS: 2727 case KVM_DEV_FLIC_ENQUEUE: 2728 case KVM_DEV_FLIC_CLEAR_IRQS: 2729 case KVM_DEV_FLIC_APF_ENABLE: 2730 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2731 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2732 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2733 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2734 case KVM_DEV_FLIC_AISM: 2735 case KVM_DEV_FLIC_AIRQ_INJECT: 2736 case KVM_DEV_FLIC_AISM_ALL: 2737 return 0; 2738 } 2739 return -ENXIO; 2740} 2741 2742static int flic_create(struct kvm_device *dev, u32 type) 2743{ 2744 if (!dev) 2745 return -EINVAL; 2746 if (dev->kvm->arch.flic) 2747 return -EINVAL; 2748 dev->kvm->arch.flic = dev; 2749 return 0; 2750} 2751 2752static void flic_destroy(struct kvm_device *dev) 2753{ 2754 dev->kvm->arch.flic = NULL; 2755 kfree(dev); 2756} 2757 2758/* s390 floating irq controller (flic) */ 2759struct kvm_device_ops kvm_flic_ops = { 2760 .name = "kvm-flic", 2761 .get_attr = flic_get_attr, 2762 .set_attr = flic_set_attr, 2763 .has_attr = flic_has_attr, 2764 .create = flic_create, 2765 .destroy = flic_destroy, 2766}; 2767 2768static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) 2769{ 2770 unsigned long bit; 2771 2772 bit = bit_nr + (addr % PAGE_SIZE) * 8; 2773 2774 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; 2775} 2776 2777static struct page *get_map_page(struct kvm *kvm, u64 uaddr) 2778{ 2779 struct page *page = NULL; 2780 2781 mmap_read_lock(kvm->mm); 2782 get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE, 2783 &page, NULL, NULL); 2784 mmap_read_unlock(kvm->mm); 2785 return page; 2786} 2787 2788static int adapter_indicators_set(struct kvm *kvm, 2789 struct s390_io_adapter *adapter, 2790 struct kvm_s390_adapter_int *adapter_int) 2791{ 2792 unsigned long bit; 2793 int summary_set, idx; 2794 struct page *ind_page, *summary_page; 2795 void *map; 2796 2797 ind_page = get_map_page(kvm, adapter_int->ind_addr); 2798 if (!ind_page) 2799 return -1; 2800 summary_page = get_map_page(kvm, adapter_int->summary_addr); 2801 if (!summary_page) { 2802 put_page(ind_page); 2803 return -1; 2804 } 2805 2806 idx = srcu_read_lock(&kvm->srcu); 2807 map = page_address(ind_page); 2808 bit = get_ind_bit(adapter_int->ind_addr, 2809 adapter_int->ind_offset, adapter->swap); 2810 set_bit(bit, map); 2811 mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT); 2812 set_page_dirty_lock(ind_page); 2813 map = page_address(summary_page); 2814 bit = get_ind_bit(adapter_int->summary_addr, 2815 adapter_int->summary_offset, adapter->swap); 2816 summary_set = test_and_set_bit(bit, map); 2817 mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT); 2818 set_page_dirty_lock(summary_page); 2819 srcu_read_unlock(&kvm->srcu, idx); 2820 2821 put_page(ind_page); 2822 put_page(summary_page); 2823 return summary_set ? 0 : 1; 2824} 2825 2826/* 2827 * < 0 - not injected due to error 2828 * = 0 - coalesced, summary indicator already active 2829 * > 0 - injected interrupt 2830 */ 2831static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, 2832 struct kvm *kvm, int irq_source_id, int level, 2833 bool line_status) 2834{ 2835 int ret; 2836 struct s390_io_adapter *adapter; 2837 2838 /* We're only interested in the 0->1 transition. */ 2839 if (!level) 2840 return 0; 2841 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 2842 if (!adapter) 2843 return -1; 2844 ret = adapter_indicators_set(kvm, adapter, &e->adapter); 2845 if ((ret > 0) && !adapter->masked) { 2846 ret = kvm_s390_inject_airq(kvm, adapter); 2847 if (ret == 0) 2848 ret = 1; 2849 } 2850 return ret; 2851} 2852 2853/* 2854 * Inject the machine check to the guest. 2855 */ 2856void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, 2857 struct mcck_volatile_info *mcck_info) 2858{ 2859 struct kvm_s390_interrupt_info inti; 2860 struct kvm_s390_irq irq; 2861 struct kvm_s390_mchk_info *mchk; 2862 union mci mci; 2863 __u64 cr14 = 0; /* upper bits are not used */ 2864 int rc; 2865 2866 mci.val = mcck_info->mcic; 2867 if (mci.sr) 2868 cr14 |= CR14_RECOVERY_SUBMASK; 2869 if (mci.dg) 2870 cr14 |= CR14_DEGRADATION_SUBMASK; 2871 if (mci.w) 2872 cr14 |= CR14_WARNING_SUBMASK; 2873 2874 mchk = mci.ck ? &inti.mchk : &irq.u.mchk; 2875 mchk->cr14 = cr14; 2876 mchk->mcic = mcck_info->mcic; 2877 mchk->ext_damage_code = mcck_info->ext_damage_code; 2878 mchk->failing_storage_address = mcck_info->failing_storage_address; 2879 if (mci.ck) { 2880 /* Inject the floating machine check */ 2881 inti.type = KVM_S390_MCHK; 2882 rc = __inject_vm(vcpu->kvm, &inti); 2883 } else { 2884 /* Inject the machine check to specified vcpu */ 2885 irq.type = KVM_S390_MCHK; 2886 rc = kvm_s390_inject_vcpu(vcpu, &irq); 2887 } 2888 WARN_ON_ONCE(rc); 2889} 2890 2891int kvm_set_routing_entry(struct kvm *kvm, 2892 struct kvm_kernel_irq_routing_entry *e, 2893 const struct kvm_irq_routing_entry *ue) 2894{ 2895 u64 uaddr; 2896 2897 switch (ue->type) { 2898 /* we store the userspace addresses instead of the guest addresses */ 2899 case KVM_IRQ_ROUTING_S390_ADAPTER: 2900 e->set = set_adapter_int; 2901 uaddr = gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr); 2902 if (uaddr == -EFAULT) 2903 return -EFAULT; 2904 e->adapter.summary_addr = uaddr; 2905 uaddr = gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr); 2906 if (uaddr == -EFAULT) 2907 return -EFAULT; 2908 e->adapter.ind_addr = uaddr; 2909 e->adapter.summary_offset = ue->u.adapter.summary_offset; 2910 e->adapter.ind_offset = ue->u.adapter.ind_offset; 2911 e->adapter.adapter_id = ue->u.adapter.adapter_id; 2912 return 0; 2913 default: 2914 return -EINVAL; 2915 } 2916} 2917 2918int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 2919 int irq_source_id, int level, bool line_status) 2920{ 2921 return -EINVAL; 2922} 2923 2924int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) 2925{ 2926 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2927 struct kvm_s390_irq *buf; 2928 int r = 0; 2929 int n; 2930 2931 buf = vmalloc(len); 2932 if (!buf) 2933 return -ENOMEM; 2934 2935 if (copy_from_user((void *) buf, irqstate, len)) { 2936 r = -EFAULT; 2937 goto out_free; 2938 } 2939 2940 /* 2941 * Don't allow setting the interrupt state 2942 * when there are already interrupts pending 2943 */ 2944 spin_lock(&li->lock); 2945 if (li->pending_irqs) { 2946 r = -EBUSY; 2947 goto out_unlock; 2948 } 2949 2950 for (n = 0; n < len / sizeof(*buf); n++) { 2951 r = do_inject_vcpu(vcpu, &buf[n]); 2952 if (r) 2953 break; 2954 } 2955 2956out_unlock: 2957 spin_unlock(&li->lock); 2958out_free: 2959 vfree(buf); 2960 2961 return r; 2962} 2963 2964static void store_local_irq(struct kvm_s390_local_interrupt *li, 2965 struct kvm_s390_irq *irq, 2966 unsigned long irq_type) 2967{ 2968 switch (irq_type) { 2969 case IRQ_PEND_MCHK_EX: 2970 case IRQ_PEND_MCHK_REP: 2971 irq->type = KVM_S390_MCHK; 2972 irq->u.mchk = li->irq.mchk; 2973 break; 2974 case IRQ_PEND_PROG: 2975 irq->type = KVM_S390_PROGRAM_INT; 2976 irq->u.pgm = li->irq.pgm; 2977 break; 2978 case IRQ_PEND_PFAULT_INIT: 2979 irq->type = KVM_S390_INT_PFAULT_INIT; 2980 irq->u.ext = li->irq.ext; 2981 break; 2982 case IRQ_PEND_EXT_EXTERNAL: 2983 irq->type = KVM_S390_INT_EXTERNAL_CALL; 2984 irq->u.extcall = li->irq.extcall; 2985 break; 2986 case IRQ_PEND_EXT_CLOCK_COMP: 2987 irq->type = KVM_S390_INT_CLOCK_COMP; 2988 break; 2989 case IRQ_PEND_EXT_CPU_TIMER: 2990 irq->type = KVM_S390_INT_CPU_TIMER; 2991 break; 2992 case IRQ_PEND_SIGP_STOP: 2993 irq->type = KVM_S390_SIGP_STOP; 2994 irq->u.stop = li->irq.stop; 2995 break; 2996 case IRQ_PEND_RESTART: 2997 irq->type = KVM_S390_RESTART; 2998 break; 2999 case IRQ_PEND_SET_PREFIX: 3000 irq->type = KVM_S390_SIGP_SET_PREFIX; 3001 irq->u.prefix = li->irq.prefix; 3002 break; 3003 } 3004} 3005 3006int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) 3007{ 3008 int scn; 3009 DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); 3010 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 3011 unsigned long pending_irqs; 3012 struct kvm_s390_irq irq; 3013 unsigned long irq_type; 3014 int cpuaddr; 3015 int n = 0; 3016 3017 spin_lock(&li->lock); 3018 pending_irqs = li->pending_irqs; 3019 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending, 3020 sizeof(sigp_emerg_pending)); 3021 spin_unlock(&li->lock); 3022 3023 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) { 3024 memset(&irq, 0, sizeof(irq)); 3025 if (irq_type == IRQ_PEND_EXT_EMERGENCY) 3026 continue; 3027 if (n + sizeof(irq) > len) 3028 return -ENOBUFS; 3029 store_local_irq(&vcpu->arch.local_int, &irq, irq_type); 3030 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 3031 return -EFAULT; 3032 n += sizeof(irq); 3033 } 3034 3035 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) { 3036 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) { 3037 memset(&irq, 0, sizeof(irq)); 3038 if (n + sizeof(irq) > len) 3039 return -ENOBUFS; 3040 irq.type = KVM_S390_INT_EMERGENCY; 3041 irq.u.emerg.code = cpuaddr; 3042 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 3043 return -EFAULT; 3044 n += sizeof(irq); 3045 } 3046 } 3047 3048 if (sca_ext_call_pending(vcpu, &scn)) { 3049 if (n + sizeof(irq) > len) 3050 return -ENOBUFS; 3051 memset(&irq, 0, sizeof(irq)); 3052 irq.type = KVM_S390_INT_EXTERNAL_CALL; 3053 irq.u.extcall.code = scn; 3054 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 3055 return -EFAULT; 3056 n += sizeof(irq); 3057 } 3058 3059 return n; 3060} 3061 3062static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask) 3063{ 3064 int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus); 3065 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3066 struct kvm_vcpu *vcpu; 3067 u8 vcpu_isc_mask; 3068 3069 for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) { 3070 vcpu = kvm_get_vcpu(kvm, vcpu_idx); 3071 if (psw_ioint_disabled(vcpu)) 3072 continue; 3073 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24); 3074 if (deliverable_mask & vcpu_isc_mask) { 3075 /* lately kicked but not yet running */ 3076 if (test_and_set_bit(vcpu_idx, gi->kicked_mask)) 3077 return; 3078 kvm_s390_vcpu_wakeup(vcpu); 3079 return; 3080 } 3081 } 3082} 3083 3084static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer) 3085{ 3086 struct kvm_s390_gisa_interrupt *gi = 3087 container_of(timer, struct kvm_s390_gisa_interrupt, timer); 3088 struct kvm *kvm = 3089 container_of(gi->origin, struct sie_page2, gisa)->kvm; 3090 u8 pending_mask; 3091 3092 pending_mask = gisa_get_ipm_or_restore_iam(gi); 3093 if (pending_mask) { 3094 __airqs_kick_single_vcpu(kvm, pending_mask); 3095 hrtimer_forward_now(timer, ns_to_ktime(gi->expires)); 3096 return HRTIMER_RESTART; 3097 } 3098 3099 return HRTIMER_NORESTART; 3100} 3101 3102#define NULL_GISA_ADDR 0x00000000UL 3103#define NONE_GISA_ADDR 0x00000001UL 3104#define GISA_ADDR_MASK 0xfffff000UL 3105 3106static void process_gib_alert_list(void) 3107{ 3108 struct kvm_s390_gisa_interrupt *gi; 3109 struct kvm_s390_gisa *gisa; 3110 struct kvm *kvm; 3111 u32 final, origin = 0UL; 3112 3113 do { 3114 /* 3115 * If the NONE_GISA_ADDR is still stored in the alert list 3116 * origin, we will leave the outer loop. No further GISA has 3117 * been added to the alert list by millicode while processing 3118 * the current alert list. 3119 */ 3120 final = (origin & NONE_GISA_ADDR); 3121 /* 3122 * Cut off the alert list and store the NONE_GISA_ADDR in the 3123 * alert list origin to avoid further GAL interruptions. 3124 * A new alert list can be build up by millicode in parallel 3125 * for guests not in the yet cut-off alert list. When in the 3126 * final loop, store the NULL_GISA_ADDR instead. This will re- 3127 * enable GAL interruptions on the host again. 3128 */ 3129 origin = xchg(&gib->alert_list_origin, 3130 (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR); 3131 /* 3132 * Loop through the just cut-off alert list and start the 3133 * gisa timers to kick idle vcpus to consume the pending 3134 * interruptions asap. 3135 */ 3136 while (origin & GISA_ADDR_MASK) { 3137 gisa = (struct kvm_s390_gisa *)(u64)origin; 3138 origin = gisa->next_alert; 3139 gisa->next_alert = (u32)(u64)gisa; 3140 kvm = container_of(gisa, struct sie_page2, gisa)->kvm; 3141 gi = &kvm->arch.gisa_int; 3142 if (hrtimer_active(&gi->timer)) 3143 hrtimer_cancel(&gi->timer); 3144 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL); 3145 } 3146 } while (!final); 3147 3148} 3149 3150void kvm_s390_gisa_clear(struct kvm *kvm) 3151{ 3152 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3153 3154 if (!gi->origin) 3155 return; 3156 gisa_clear_ipm(gi->origin); 3157 VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin); 3158} 3159 3160void kvm_s390_gisa_init(struct kvm *kvm) 3161{ 3162 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3163 3164 if (!css_general_characteristics.aiv) 3165 return; 3166 gi->origin = &kvm->arch.sie_page2->gisa; 3167 gi->alert.mask = 0; 3168 spin_lock_init(&gi->alert.ref_lock); 3169 gi->expires = 50 * 1000; /* 50 usec */ 3170 hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3171 gi->timer.function = gisa_vcpu_kicker; 3172 memset(gi->origin, 0, sizeof(struct kvm_s390_gisa)); 3173 gi->origin->next_alert = (u32)(u64)gi->origin; 3174 VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin); 3175} 3176 3177void kvm_s390_gisa_destroy(struct kvm *kvm) 3178{ 3179 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3180 3181 if (!gi->origin) 3182 return; 3183 if (gi->alert.mask) 3184 KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x", 3185 kvm, gi->alert.mask); 3186 while (gisa_in_alert_list(gi->origin)) 3187 cpu_relax(); 3188 hrtimer_cancel(&gi->timer); 3189 gi->origin = NULL; 3190} 3191 3192/** 3193 * kvm_s390_gisc_register - register a guest ISC 3194 * 3195 * @kvm: the kernel vm to work with 3196 * @gisc: the guest interruption sub class to register 3197 * 3198 * The function extends the vm specific alert mask to use. 3199 * The effective IAM mask in the GISA is updated as well 3200 * in case the GISA is not part of the GIB alert list. 3201 * It will be updated latest when the IAM gets restored 3202 * by gisa_get_ipm_or_restore_iam(). 3203 * 3204 * Returns: the nonspecific ISC (NISC) the gib alert mechanism 3205 * has registered with the channel subsystem. 3206 * -ENODEV in case the vm uses no GISA 3207 * -ERANGE in case the guest ISC is invalid 3208 */ 3209int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc) 3210{ 3211 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3212 3213 if (!gi->origin) 3214 return -ENODEV; 3215 if (gisc > MAX_ISC) 3216 return -ERANGE; 3217 3218 spin_lock(&gi->alert.ref_lock); 3219 gi->alert.ref_count[gisc]++; 3220 if (gi->alert.ref_count[gisc] == 1) { 3221 gi->alert.mask |= 0x80 >> gisc; 3222 gisa_set_iam(gi->origin, gi->alert.mask); 3223 } 3224 spin_unlock(&gi->alert.ref_lock); 3225 3226 return gib->nisc; 3227} 3228EXPORT_SYMBOL_GPL(kvm_s390_gisc_register); 3229 3230/** 3231 * kvm_s390_gisc_unregister - unregister a guest ISC 3232 * 3233 * @kvm: the kernel vm to work with 3234 * @gisc: the guest interruption sub class to register 3235 * 3236 * The function reduces the vm specific alert mask to use. 3237 * The effective IAM mask in the GISA is updated as well 3238 * in case the GISA is not part of the GIB alert list. 3239 * It will be updated latest when the IAM gets restored 3240 * by gisa_get_ipm_or_restore_iam(). 3241 * 3242 * Returns: the nonspecific ISC (NISC) the gib alert mechanism 3243 * has registered with the channel subsystem. 3244 * -ENODEV in case the vm uses no GISA 3245 * -ERANGE in case the guest ISC is invalid 3246 * -EINVAL in case the guest ISC is not registered 3247 */ 3248int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc) 3249{ 3250 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3251 int rc = 0; 3252 3253 if (!gi->origin) 3254 return -ENODEV; 3255 if (gisc > MAX_ISC) 3256 return -ERANGE; 3257 3258 spin_lock(&gi->alert.ref_lock); 3259 if (gi->alert.ref_count[gisc] == 0) { 3260 rc = -EINVAL; 3261 goto out; 3262 } 3263 gi->alert.ref_count[gisc]--; 3264 if (gi->alert.ref_count[gisc] == 0) { 3265 gi->alert.mask &= ~(0x80 >> gisc); 3266 gisa_set_iam(gi->origin, gi->alert.mask); 3267 } 3268out: 3269 spin_unlock(&gi->alert.ref_lock); 3270 3271 return rc; 3272} 3273EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister); 3274 3275static void gib_alert_irq_handler(struct airq_struct *airq, bool floating) 3276{ 3277 inc_irq_stat(IRQIO_GAL); 3278 process_gib_alert_list(); 3279} 3280 3281static struct airq_struct gib_alert_irq = { 3282 .handler = gib_alert_irq_handler, 3283 .lsi_ptr = &gib_alert_irq.lsi_mask, 3284}; 3285 3286void kvm_s390_gib_destroy(void) 3287{ 3288 if (!gib) 3289 return; 3290 chsc_sgib(0); 3291 unregister_adapter_interrupt(&gib_alert_irq); 3292 free_page((unsigned long)gib); 3293 gib = NULL; 3294} 3295 3296int kvm_s390_gib_init(u8 nisc) 3297{ 3298 int rc = 0; 3299 3300 if (!css_general_characteristics.aiv) { 3301 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility"); 3302 goto out; 3303 } 3304 3305 gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL | GFP_DMA); 3306 if (!gib) { 3307 rc = -ENOMEM; 3308 goto out; 3309 } 3310 3311 gib_alert_irq.isc = nisc; 3312 if (register_adapter_interrupt(&gib_alert_irq)) { 3313 pr_err("Registering the GIB alert interruption handler failed\n"); 3314 rc = -EIO; 3315 goto out_free_gib; 3316 } 3317 3318 gib->nisc = nisc; 3319 if (chsc_sgib((u32)(u64)gib)) { 3320 pr_err("Associating the GIB with the AIV facility failed\n"); 3321 free_page((unsigned long)gib); 3322 gib = NULL; 3323 rc = -EIO; 3324 goto out_unreg_gal; 3325 } 3326 3327 KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc); 3328 goto out; 3329 3330out_unreg_gal: 3331 unregister_adapter_interrupt(&gib_alert_irq); 3332out_free_gib: 3333 free_page((unsigned long)gib); 3334 gib = NULL; 3335out: 3336 return rc; 3337} 3338