1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Deliver/Emulate exceptions to the guest kernel 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Sanjay Lal <sanjayl@kymasys.com> 10 */ 11 12#include <linux/errno.h> 13#include <linux/err.h> 14#include <linux/kvm_host.h> 15#include <linux/log2.h> 16#include <linux/uaccess.h> 17#include <linux/vmalloc.h> 18#include <asm/mmu_context.h> 19#include <asm/pgalloc.h> 20 21#include "interrupt.h" 22 23static gpa_t kvm_trap_emul_gva_to_gpa_cb(gva_t gva) 24{ 25 gpa_t gpa; 26 gva_t kseg = KSEGX(gva); 27 gva_t gkseg = KVM_GUEST_KSEGX(gva); 28 29 if ((kseg == CKSEG0) || (kseg == CKSEG1)) 30 gpa = CPHYSADDR(gva); 31 else if (gkseg == KVM_GUEST_KSEG0) 32 gpa = KVM_GUEST_CPHYSADDR(gva); 33 else { 34 kvm_err("%s: cannot find GPA for GVA: %#lx\n", __func__, gva); 35 kvm_mips_dump_host_tlbs(); 36 gpa = KVM_INVALID_ADDR; 37 } 38 39 kvm_debug("%s: gva %#lx, gpa: %#llx\n", __func__, gva, gpa); 40 41 return gpa; 42} 43 44static int kvm_trap_emul_no_handler(struct kvm_vcpu *vcpu) 45{ 46 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 47 u32 cause = vcpu->arch.host_cp0_cause; 48 u32 exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; 49 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 50 u32 inst = 0; 51 52 /* 53 * Fetch the instruction. 54 */ 55 if (cause & CAUSEF_BD) 56 opc += 1; 57 kvm_get_badinstr(opc, vcpu, &inst); 58 59 kvm_err("Exception Code: %d not handled @ PC: %p, inst: 0x%08x BadVaddr: %#lx Status: %#x\n", 60 exccode, opc, inst, badvaddr, 61 kvm_read_c0_guest_status(vcpu->arch.cop0)); 62 kvm_arch_vcpu_dump_regs(vcpu); 63 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 64 return RESUME_HOST; 65} 66 67static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu) 68{ 69 struct mips_coproc *cop0 = vcpu->arch.cop0; 70 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 71 u32 cause = vcpu->arch.host_cp0_cause; 72 enum emulation_result er = EMULATE_DONE; 73 int ret = RESUME_GUEST; 74 75 if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) { 76 /* FPU Unusable */ 77 if (!kvm_mips_guest_has_fpu(&vcpu->arch) || 78 (kvm_read_c0_guest_status(cop0) & ST0_CU1) == 0) { 79 /* 80 * Unusable/no FPU in guest: 81 * deliver guest COP1 Unusable Exception 82 */ 83 er = kvm_mips_emulate_fpu_exc(cause, opc, vcpu); 84 } else { 85 /* Restore FPU state */ 86 kvm_own_fpu(vcpu); 87 er = EMULATE_DONE; 88 } 89 } else { 90 er = kvm_mips_emulate_inst(cause, opc, vcpu); 91 } 92 93 switch (er) { 94 case EMULATE_DONE: 95 ret = RESUME_GUEST; 96 break; 97 98 case EMULATE_FAIL: 99 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 100 ret = RESUME_HOST; 101 break; 102 103 case EMULATE_WAIT: 104 vcpu->run->exit_reason = KVM_EXIT_INTR; 105 ret = RESUME_HOST; 106 break; 107 108 case EMULATE_HYPERCALL: 109 ret = kvm_mips_handle_hypcall(vcpu); 110 break; 111 112 default: 113 BUG(); 114 } 115 return ret; 116} 117 118static int kvm_mips_bad_load(u32 cause, u32 *opc, struct kvm_vcpu *vcpu) 119{ 120 enum emulation_result er; 121 union mips_instruction inst; 122 int err; 123 124 /* A code fetch fault doesn't count as an MMIO */ 125 if (kvm_is_ifetch_fault(&vcpu->arch)) { 126 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 127 return RESUME_HOST; 128 } 129 130 /* Fetch the instruction. */ 131 if (cause & CAUSEF_BD) 132 opc += 1; 133 err = kvm_get_badinstr(opc, vcpu, &inst.word); 134 if (err) { 135 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 136 return RESUME_HOST; 137 } 138 139 /* Emulate the load */ 140 er = kvm_mips_emulate_load(inst, cause, vcpu); 141 if (er == EMULATE_FAIL) { 142 kvm_err("Emulate load from MMIO space failed\n"); 143 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 144 } else { 145 vcpu->run->exit_reason = KVM_EXIT_MMIO; 146 } 147 return RESUME_HOST; 148} 149 150static int kvm_mips_bad_store(u32 cause, u32 *opc, struct kvm_vcpu *vcpu) 151{ 152 enum emulation_result er; 153 union mips_instruction inst; 154 int err; 155 156 /* Fetch the instruction. */ 157 if (cause & CAUSEF_BD) 158 opc += 1; 159 err = kvm_get_badinstr(opc, vcpu, &inst.word); 160 if (err) { 161 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 162 return RESUME_HOST; 163 } 164 165 /* Emulate the store */ 166 er = kvm_mips_emulate_store(inst, cause, vcpu); 167 if (er == EMULATE_FAIL) { 168 kvm_err("Emulate store to MMIO space failed\n"); 169 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 170 } else { 171 vcpu->run->exit_reason = KVM_EXIT_MMIO; 172 } 173 return RESUME_HOST; 174} 175 176static int kvm_mips_bad_access(u32 cause, u32 *opc, 177 struct kvm_vcpu *vcpu, bool store) 178{ 179 if (store) 180 return kvm_mips_bad_store(cause, opc, vcpu); 181 else 182 return kvm_mips_bad_load(cause, opc, vcpu); 183} 184 185static int kvm_trap_emul_handle_tlb_mod(struct kvm_vcpu *vcpu) 186{ 187 struct mips_coproc *cop0 = vcpu->arch.cop0; 188 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 189 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 190 u32 cause = vcpu->arch.host_cp0_cause; 191 struct kvm_mips_tlb *tlb; 192 unsigned long entryhi; 193 int index; 194 195 if (KVM_GUEST_KSEGX(badvaddr) < KVM_GUEST_KSEG0 196 || KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG23) { 197 /* 198 * First find the mapping in the guest TLB. If the failure to 199 * write was due to the guest TLB, it should be up to the guest 200 * to handle it. 201 */ 202 entryhi = (badvaddr & VPN2_MASK) | 203 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID); 204 index = kvm_mips_guest_tlb_lookup(vcpu, entryhi); 205 206 /* 207 * These should never happen. 208 * They would indicate stale host TLB entries. 209 */ 210 if (unlikely(index < 0)) { 211 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 212 return RESUME_HOST; 213 } 214 tlb = vcpu->arch.guest_tlb + index; 215 if (unlikely(!TLB_IS_VALID(*tlb, badvaddr))) { 216 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 217 return RESUME_HOST; 218 } 219 220 /* 221 * Guest entry not dirty? That would explain the TLB modified 222 * exception. Relay that on to the guest so it can handle it. 223 */ 224 if (!TLB_IS_DIRTY(*tlb, badvaddr)) { 225 kvm_mips_emulate_tlbmod(cause, opc, vcpu); 226 return RESUME_GUEST; 227 } 228 229 if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, badvaddr, 230 true)) 231 /* Not writable, needs handling as MMIO */ 232 return kvm_mips_bad_store(cause, opc, vcpu); 233 return RESUME_GUEST; 234 } else if (KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG0) { 235 if (kvm_mips_handle_kseg0_tlb_fault(badvaddr, vcpu, true) < 0) 236 /* Not writable, needs handling as MMIO */ 237 return kvm_mips_bad_store(cause, opc, vcpu); 238 return RESUME_GUEST; 239 } else { 240 /* host kernel addresses are all handled as MMIO */ 241 return kvm_mips_bad_store(cause, opc, vcpu); 242 } 243} 244 245static int kvm_trap_emul_handle_tlb_miss(struct kvm_vcpu *vcpu, bool store) 246{ 247 struct kvm_run *run = vcpu->run; 248 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 249 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 250 u32 cause = vcpu->arch.host_cp0_cause; 251 enum emulation_result er = EMULATE_DONE; 252 int ret = RESUME_GUEST; 253 254 if (((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) 255 && KVM_GUEST_KERNEL_MODE(vcpu)) { 256 if (kvm_mips_handle_commpage_tlb_fault(badvaddr, vcpu) < 0) { 257 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 258 ret = RESUME_HOST; 259 } 260 } else if (KVM_GUEST_KSEGX(badvaddr) < KVM_GUEST_KSEG0 261 || KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG23) { 262 kvm_debug("USER ADDR TLB %s fault: cause %#x, PC: %p, BadVaddr: %#lx\n", 263 store ? "ST" : "LD", cause, opc, badvaddr); 264 265 /* 266 * User Address (UA) fault, this could happen if 267 * (1) TLB entry not present/valid in both Guest and shadow host 268 * TLBs, in this case we pass on the fault to the guest 269 * kernel and let it handle it. 270 * (2) TLB entry is present in the Guest TLB but not in the 271 * shadow, in this case we inject the TLB from the Guest TLB 272 * into the shadow host TLB 273 */ 274 275 er = kvm_mips_handle_tlbmiss(cause, opc, vcpu, store); 276 if (er == EMULATE_DONE) 277 ret = RESUME_GUEST; 278 else { 279 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 280 ret = RESUME_HOST; 281 } 282 } else if (KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG0) { 283 /* 284 * All KSEG0 faults are handled by KVM, as the guest kernel does 285 * not expect to ever get them 286 */ 287 if (kvm_mips_handle_kseg0_tlb_fault(badvaddr, vcpu, store) < 0) 288 ret = kvm_mips_bad_access(cause, opc, vcpu, store); 289 } else if (KVM_GUEST_KERNEL_MODE(vcpu) 290 && (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1)) { 291 /* 292 * With EVA we may get a TLB exception instead of an address 293 * error when the guest performs MMIO to KSeg1 addresses. 294 */ 295 ret = kvm_mips_bad_access(cause, opc, vcpu, store); 296 } else { 297 kvm_err("Illegal TLB %s fault address , cause %#x, PC: %p, BadVaddr: %#lx\n", 298 store ? "ST" : "LD", cause, opc, badvaddr); 299 kvm_mips_dump_host_tlbs(); 300 kvm_arch_vcpu_dump_regs(vcpu); 301 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 302 ret = RESUME_HOST; 303 } 304 return ret; 305} 306 307static int kvm_trap_emul_handle_tlb_st_miss(struct kvm_vcpu *vcpu) 308{ 309 return kvm_trap_emul_handle_tlb_miss(vcpu, true); 310} 311 312static int kvm_trap_emul_handle_tlb_ld_miss(struct kvm_vcpu *vcpu) 313{ 314 return kvm_trap_emul_handle_tlb_miss(vcpu, false); 315} 316 317static int kvm_trap_emul_handle_addr_err_st(struct kvm_vcpu *vcpu) 318{ 319 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 320 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 321 u32 cause = vcpu->arch.host_cp0_cause; 322 int ret = RESUME_GUEST; 323 324 if (KVM_GUEST_KERNEL_MODE(vcpu) 325 && (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1)) { 326 ret = kvm_mips_bad_store(cause, opc, vcpu); 327 } else { 328 kvm_err("Address Error (STORE): cause %#x, PC: %p, BadVaddr: %#lx\n", 329 cause, opc, badvaddr); 330 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 331 ret = RESUME_HOST; 332 } 333 return ret; 334} 335 336static int kvm_trap_emul_handle_addr_err_ld(struct kvm_vcpu *vcpu) 337{ 338 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 339 unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr; 340 u32 cause = vcpu->arch.host_cp0_cause; 341 int ret = RESUME_GUEST; 342 343 if (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1) { 344 ret = kvm_mips_bad_load(cause, opc, vcpu); 345 } else { 346 kvm_err("Address Error (LOAD): cause %#x, PC: %p, BadVaddr: %#lx\n", 347 cause, opc, badvaddr); 348 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 349 ret = RESUME_HOST; 350 } 351 return ret; 352} 353 354static int kvm_trap_emul_handle_syscall(struct kvm_vcpu *vcpu) 355{ 356 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 357 u32 cause = vcpu->arch.host_cp0_cause; 358 enum emulation_result er = EMULATE_DONE; 359 int ret = RESUME_GUEST; 360 361 er = kvm_mips_emulate_syscall(cause, opc, vcpu); 362 if (er == EMULATE_DONE) 363 ret = RESUME_GUEST; 364 else { 365 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 366 ret = RESUME_HOST; 367 } 368 return ret; 369} 370 371static int kvm_trap_emul_handle_res_inst(struct kvm_vcpu *vcpu) 372{ 373 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 374 u32 cause = vcpu->arch.host_cp0_cause; 375 enum emulation_result er = EMULATE_DONE; 376 int ret = RESUME_GUEST; 377 378 er = kvm_mips_handle_ri(cause, opc, vcpu); 379 if (er == EMULATE_DONE) 380 ret = RESUME_GUEST; 381 else { 382 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 383 ret = RESUME_HOST; 384 } 385 return ret; 386} 387 388static int kvm_trap_emul_handle_break(struct kvm_vcpu *vcpu) 389{ 390 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 391 u32 cause = vcpu->arch.host_cp0_cause; 392 enum emulation_result er = EMULATE_DONE; 393 int ret = RESUME_GUEST; 394 395 er = kvm_mips_emulate_bp_exc(cause, opc, vcpu); 396 if (er == EMULATE_DONE) 397 ret = RESUME_GUEST; 398 else { 399 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 400 ret = RESUME_HOST; 401 } 402 return ret; 403} 404 405static int kvm_trap_emul_handle_trap(struct kvm_vcpu *vcpu) 406{ 407 u32 __user *opc = (u32 __user *)vcpu->arch.pc; 408 u32 cause = vcpu->arch.host_cp0_cause; 409 enum emulation_result er = EMULATE_DONE; 410 int ret = RESUME_GUEST; 411 412 er = kvm_mips_emulate_trap_exc(cause, opc, vcpu); 413 if (er == EMULATE_DONE) { 414 ret = RESUME_GUEST; 415 } else { 416 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 417 ret = RESUME_HOST; 418 } 419 return ret; 420} 421 422static int kvm_trap_emul_handle_msa_fpe(struct kvm_vcpu *vcpu) 423{ 424 u32 __user *opc = (u32 __user *)vcpu->arch.pc; 425 u32 cause = vcpu->arch.host_cp0_cause; 426 enum emulation_result er = EMULATE_DONE; 427 int ret = RESUME_GUEST; 428 429 er = kvm_mips_emulate_msafpe_exc(cause, opc, vcpu); 430 if (er == EMULATE_DONE) { 431 ret = RESUME_GUEST; 432 } else { 433 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 434 ret = RESUME_HOST; 435 } 436 return ret; 437} 438 439static int kvm_trap_emul_handle_fpe(struct kvm_vcpu *vcpu) 440{ 441 u32 __user *opc = (u32 __user *)vcpu->arch.pc; 442 u32 cause = vcpu->arch.host_cp0_cause; 443 enum emulation_result er = EMULATE_DONE; 444 int ret = RESUME_GUEST; 445 446 er = kvm_mips_emulate_fpe_exc(cause, opc, vcpu); 447 if (er == EMULATE_DONE) { 448 ret = RESUME_GUEST; 449 } else { 450 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 451 ret = RESUME_HOST; 452 } 453 return ret; 454} 455 456/** 457 * kvm_trap_emul_handle_msa_disabled() - Guest used MSA while disabled in root. 458 * @vcpu: Virtual CPU context. 459 * 460 * Handle when the guest attempts to use MSA when it is disabled. 461 */ 462static int kvm_trap_emul_handle_msa_disabled(struct kvm_vcpu *vcpu) 463{ 464 struct mips_coproc *cop0 = vcpu->arch.cop0; 465 u32 __user *opc = (u32 __user *) vcpu->arch.pc; 466 u32 cause = vcpu->arch.host_cp0_cause; 467 enum emulation_result er = EMULATE_DONE; 468 int ret = RESUME_GUEST; 469 470 if (!kvm_mips_guest_has_msa(&vcpu->arch) || 471 (kvm_read_c0_guest_status(cop0) & (ST0_CU1 | ST0_FR)) == ST0_CU1) { 472 /* 473 * No MSA in guest, or FPU enabled and not in FR=1 mode, 474 * guest reserved instruction exception 475 */ 476 er = kvm_mips_emulate_ri_exc(cause, opc, vcpu); 477 } else if (!(kvm_read_c0_guest_config5(cop0) & MIPS_CONF5_MSAEN)) { 478 /* MSA disabled by guest, guest MSA disabled exception */ 479 er = kvm_mips_emulate_msadis_exc(cause, opc, vcpu); 480 } else { 481 /* Restore MSA/FPU state */ 482 kvm_own_msa(vcpu); 483 er = EMULATE_DONE; 484 } 485 486 switch (er) { 487 case EMULATE_DONE: 488 ret = RESUME_GUEST; 489 break; 490 491 case EMULATE_FAIL: 492 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 493 ret = RESUME_HOST; 494 break; 495 496 default: 497 BUG(); 498 } 499 return ret; 500} 501 502static int kvm_trap_emul_hardware_enable(void) 503{ 504 return 0; 505} 506 507static void kvm_trap_emul_hardware_disable(void) 508{ 509} 510 511static int kvm_trap_emul_check_extension(struct kvm *kvm, long ext) 512{ 513 int r; 514 515 switch (ext) { 516 case KVM_CAP_MIPS_TE: 517 r = 1; 518 break; 519 case KVM_CAP_IOEVENTFD: 520 r = 1; 521 break; 522 default: 523 r = 0; 524 break; 525 } 526 527 return r; 528} 529 530static int kvm_trap_emul_vcpu_init(struct kvm_vcpu *vcpu) 531{ 532 struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm; 533 struct mm_struct *user_mm = &vcpu->arch.guest_user_mm; 534 535 /* 536 * Allocate GVA -> HPA page tables. 537 * MIPS doesn't use the mm_struct pointer argument. 538 */ 539 kern_mm->pgd = pgd_alloc(kern_mm); 540 if (!kern_mm->pgd) 541 return -ENOMEM; 542 543 user_mm->pgd = pgd_alloc(user_mm); 544 if (!user_mm->pgd) { 545 pgd_free(kern_mm, kern_mm->pgd); 546 return -ENOMEM; 547 } 548 549 return 0; 550} 551 552static void kvm_mips_emul_free_gva_pt(pgd_t *pgd) 553{ 554 /* Don't free host kernel page tables copied from init_mm.pgd */ 555 const unsigned long end = 0x80000000; 556 unsigned long pgd_va, pud_va, pmd_va; 557 p4d_t *p4d; 558 pud_t *pud; 559 pmd_t *pmd; 560 pte_t *pte; 561 int i, j, k; 562 563 for (i = 0; i < USER_PTRS_PER_PGD; i++) { 564 if (pgd_none(pgd[i])) 565 continue; 566 567 pgd_va = (unsigned long)i << PGDIR_SHIFT; 568 if (pgd_va >= end) 569 break; 570 p4d = p4d_offset(pgd, 0); 571 pud = pud_offset(p4d + i, 0); 572 for (j = 0; j < PTRS_PER_PUD; j++) { 573 if (pud_none(pud[j])) 574 continue; 575 576 pud_va = pgd_va | ((unsigned long)j << PUD_SHIFT); 577 if (pud_va >= end) 578 break; 579 pmd = pmd_offset(pud + j, 0); 580 for (k = 0; k < PTRS_PER_PMD; k++) { 581 if (pmd_none(pmd[k])) 582 continue; 583 584 pmd_va = pud_va | (k << PMD_SHIFT); 585 if (pmd_va >= end) 586 break; 587 pte = pte_offset_kernel(pmd + k, 0); 588 pte_free_kernel(NULL, pte); 589 } 590 pmd_free(NULL, pmd); 591 } 592 pud_free(NULL, pud); 593 } 594 pgd_free(NULL, pgd); 595} 596 597static void kvm_trap_emul_vcpu_uninit(struct kvm_vcpu *vcpu) 598{ 599 kvm_mips_emul_free_gva_pt(vcpu->arch.guest_kernel_mm.pgd); 600 kvm_mips_emul_free_gva_pt(vcpu->arch.guest_user_mm.pgd); 601} 602 603static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu) 604{ 605 struct mips_coproc *cop0 = vcpu->arch.cop0; 606 u32 config, config1; 607 int vcpu_id = vcpu->vcpu_id; 608 609 /* Start off the timer at 100 MHz */ 610 kvm_mips_init_count(vcpu, 100*1000*1000); 611 612 /* 613 * Arch specific stuff, set up config registers properly so that the 614 * guest will come up as expected 615 */ 616#ifndef CONFIG_CPU_MIPSR6 617 /* r2-r5, simulate a MIPS 24kc */ 618 kvm_write_c0_guest_prid(cop0, 0x00019300); 619#else 620 /* r6+, simulate a generic QEMU machine */ 621 kvm_write_c0_guest_prid(cop0, 0x00010000); 622#endif 623 /* 624 * Have config1, Cacheable, noncoherent, write-back, write allocate. 625 * Endianness, arch revision & virtually tagged icache should match 626 * host. 627 */ 628 config = read_c0_config() & MIPS_CONF_AR; 629 config |= MIPS_CONF_M | CONF_CM_CACHABLE_NONCOHERENT | MIPS_CONF_MT_TLB; 630#ifdef CONFIG_CPU_BIG_ENDIAN 631 config |= CONF_BE; 632#endif 633 if (cpu_has_vtag_icache) 634 config |= MIPS_CONF_VI; 635 kvm_write_c0_guest_config(cop0, config); 636 637 /* Read the cache characteristics from the host Config1 Register */ 638 config1 = (read_c0_config1() & ~0x7f); 639 640 /* DCache line size not correctly reported in Config1 on Octeon CPUs */ 641 if (cpu_dcache_line_size()) { 642 config1 &= ~MIPS_CONF1_DL; 643 config1 |= ((ilog2(cpu_dcache_line_size()) - 1) << 644 MIPS_CONF1_DL_SHF) & MIPS_CONF1_DL; 645 } 646 647 /* Set up MMU size */ 648 config1 &= ~(0x3f << 25); 649 config1 |= ((KVM_MIPS_GUEST_TLB_SIZE - 1) << 25); 650 651 /* We unset some bits that we aren't emulating */ 652 config1 &= ~(MIPS_CONF1_C2 | MIPS_CONF1_MD | MIPS_CONF1_PC | 653 MIPS_CONF1_WR | MIPS_CONF1_CA); 654 kvm_write_c0_guest_config1(cop0, config1); 655 656 /* Have config3, no tertiary/secondary caches implemented */ 657 kvm_write_c0_guest_config2(cop0, MIPS_CONF_M); 658 /* MIPS_CONF_M | (read_c0_config2() & 0xfff) */ 659 660 /* Have config4, UserLocal */ 661 kvm_write_c0_guest_config3(cop0, MIPS_CONF_M | MIPS_CONF3_ULRI); 662 663 /* Have config5 */ 664 kvm_write_c0_guest_config4(cop0, MIPS_CONF_M); 665 666 /* No config6 */ 667 kvm_write_c0_guest_config5(cop0, 0); 668 669 /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */ 670 kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10)); 671 672 /* Status */ 673 kvm_write_c0_guest_status(cop0, ST0_BEV | ST0_ERL); 674 675 /* 676 * Setup IntCtl defaults, compatibility mode for timer interrupts (HW5) 677 */ 678 kvm_write_c0_guest_intctl(cop0, 0xFC000000); 679 680 /* Put in vcpu id as CPUNum into Ebase Reg to handle SMP Guests */ 681 kvm_write_c0_guest_ebase(cop0, KVM_GUEST_KSEG0 | 682 (vcpu_id & MIPS_EBASE_CPUNUM)); 683 684 /* Put PC at guest reset vector */ 685 vcpu->arch.pc = KVM_GUEST_CKSEG1ADDR(0x1fc00000); 686 687 return 0; 688} 689 690static void kvm_trap_emul_flush_shadow_all(struct kvm *kvm) 691{ 692 /* Flush GVA page tables and invalidate GVA ASIDs on all VCPUs */ 693 kvm_flush_remote_tlbs(kvm); 694} 695 696static void kvm_trap_emul_flush_shadow_memslot(struct kvm *kvm, 697 const struct kvm_memory_slot *slot) 698{ 699 kvm_trap_emul_flush_shadow_all(kvm); 700} 701 702static u64 kvm_trap_emul_get_one_regs[] = { 703 KVM_REG_MIPS_CP0_INDEX, 704 KVM_REG_MIPS_CP0_ENTRYLO0, 705 KVM_REG_MIPS_CP0_ENTRYLO1, 706 KVM_REG_MIPS_CP0_CONTEXT, 707 KVM_REG_MIPS_CP0_USERLOCAL, 708 KVM_REG_MIPS_CP0_PAGEMASK, 709 KVM_REG_MIPS_CP0_WIRED, 710 KVM_REG_MIPS_CP0_HWRENA, 711 KVM_REG_MIPS_CP0_BADVADDR, 712 KVM_REG_MIPS_CP0_COUNT, 713 KVM_REG_MIPS_CP0_ENTRYHI, 714 KVM_REG_MIPS_CP0_COMPARE, 715 KVM_REG_MIPS_CP0_STATUS, 716 KVM_REG_MIPS_CP0_INTCTL, 717 KVM_REG_MIPS_CP0_CAUSE, 718 KVM_REG_MIPS_CP0_EPC, 719 KVM_REG_MIPS_CP0_PRID, 720 KVM_REG_MIPS_CP0_EBASE, 721 KVM_REG_MIPS_CP0_CONFIG, 722 KVM_REG_MIPS_CP0_CONFIG1, 723 KVM_REG_MIPS_CP0_CONFIG2, 724 KVM_REG_MIPS_CP0_CONFIG3, 725 KVM_REG_MIPS_CP0_CONFIG4, 726 KVM_REG_MIPS_CP0_CONFIG5, 727 KVM_REG_MIPS_CP0_CONFIG7, 728 KVM_REG_MIPS_CP0_ERROREPC, 729 KVM_REG_MIPS_CP0_KSCRATCH1, 730 KVM_REG_MIPS_CP0_KSCRATCH2, 731 KVM_REG_MIPS_CP0_KSCRATCH3, 732 KVM_REG_MIPS_CP0_KSCRATCH4, 733 KVM_REG_MIPS_CP0_KSCRATCH5, 734 KVM_REG_MIPS_CP0_KSCRATCH6, 735 736 KVM_REG_MIPS_COUNT_CTL, 737 KVM_REG_MIPS_COUNT_RESUME, 738 KVM_REG_MIPS_COUNT_HZ, 739}; 740 741static unsigned long kvm_trap_emul_num_regs(struct kvm_vcpu *vcpu) 742{ 743 return ARRAY_SIZE(kvm_trap_emul_get_one_regs); 744} 745 746static int kvm_trap_emul_copy_reg_indices(struct kvm_vcpu *vcpu, 747 u64 __user *indices) 748{ 749 if (copy_to_user(indices, kvm_trap_emul_get_one_regs, 750 sizeof(kvm_trap_emul_get_one_regs))) 751 return -EFAULT; 752 indices += ARRAY_SIZE(kvm_trap_emul_get_one_regs); 753 754 return 0; 755} 756 757static int kvm_trap_emul_get_one_reg(struct kvm_vcpu *vcpu, 758 const struct kvm_one_reg *reg, 759 s64 *v) 760{ 761 struct mips_coproc *cop0 = vcpu->arch.cop0; 762 763 switch (reg->id) { 764 case KVM_REG_MIPS_CP0_INDEX: 765 *v = (long)kvm_read_c0_guest_index(cop0); 766 break; 767 case KVM_REG_MIPS_CP0_ENTRYLO0: 768 *v = kvm_read_c0_guest_entrylo0(cop0); 769 break; 770 case KVM_REG_MIPS_CP0_ENTRYLO1: 771 *v = kvm_read_c0_guest_entrylo1(cop0); 772 break; 773 case KVM_REG_MIPS_CP0_CONTEXT: 774 *v = (long)kvm_read_c0_guest_context(cop0); 775 break; 776 case KVM_REG_MIPS_CP0_USERLOCAL: 777 *v = (long)kvm_read_c0_guest_userlocal(cop0); 778 break; 779 case KVM_REG_MIPS_CP0_PAGEMASK: 780 *v = (long)kvm_read_c0_guest_pagemask(cop0); 781 break; 782 case KVM_REG_MIPS_CP0_WIRED: 783 *v = (long)kvm_read_c0_guest_wired(cop0); 784 break; 785 case KVM_REG_MIPS_CP0_HWRENA: 786 *v = (long)kvm_read_c0_guest_hwrena(cop0); 787 break; 788 case KVM_REG_MIPS_CP0_BADVADDR: 789 *v = (long)kvm_read_c0_guest_badvaddr(cop0); 790 break; 791 case KVM_REG_MIPS_CP0_ENTRYHI: 792 *v = (long)kvm_read_c0_guest_entryhi(cop0); 793 break; 794 case KVM_REG_MIPS_CP0_COMPARE: 795 *v = (long)kvm_read_c0_guest_compare(cop0); 796 break; 797 case KVM_REG_MIPS_CP0_STATUS: 798 *v = (long)kvm_read_c0_guest_status(cop0); 799 break; 800 case KVM_REG_MIPS_CP0_INTCTL: 801 *v = (long)kvm_read_c0_guest_intctl(cop0); 802 break; 803 case KVM_REG_MIPS_CP0_CAUSE: 804 *v = (long)kvm_read_c0_guest_cause(cop0); 805 break; 806 case KVM_REG_MIPS_CP0_EPC: 807 *v = (long)kvm_read_c0_guest_epc(cop0); 808 break; 809 case KVM_REG_MIPS_CP0_PRID: 810 *v = (long)kvm_read_c0_guest_prid(cop0); 811 break; 812 case KVM_REG_MIPS_CP0_EBASE: 813 *v = (long)kvm_read_c0_guest_ebase(cop0); 814 break; 815 case KVM_REG_MIPS_CP0_CONFIG: 816 *v = (long)kvm_read_c0_guest_config(cop0); 817 break; 818 case KVM_REG_MIPS_CP0_CONFIG1: 819 *v = (long)kvm_read_c0_guest_config1(cop0); 820 break; 821 case KVM_REG_MIPS_CP0_CONFIG2: 822 *v = (long)kvm_read_c0_guest_config2(cop0); 823 break; 824 case KVM_REG_MIPS_CP0_CONFIG3: 825 *v = (long)kvm_read_c0_guest_config3(cop0); 826 break; 827 case KVM_REG_MIPS_CP0_CONFIG4: 828 *v = (long)kvm_read_c0_guest_config4(cop0); 829 break; 830 case KVM_REG_MIPS_CP0_CONFIG5: 831 *v = (long)kvm_read_c0_guest_config5(cop0); 832 break; 833 case KVM_REG_MIPS_CP0_CONFIG7: 834 *v = (long)kvm_read_c0_guest_config7(cop0); 835 break; 836 case KVM_REG_MIPS_CP0_COUNT: 837 *v = kvm_mips_read_count(vcpu); 838 break; 839 case KVM_REG_MIPS_COUNT_CTL: 840 *v = vcpu->arch.count_ctl; 841 break; 842 case KVM_REG_MIPS_COUNT_RESUME: 843 *v = ktime_to_ns(vcpu->arch.count_resume); 844 break; 845 case KVM_REG_MIPS_COUNT_HZ: 846 *v = vcpu->arch.count_hz; 847 break; 848 case KVM_REG_MIPS_CP0_ERROREPC: 849 *v = (long)kvm_read_c0_guest_errorepc(cop0); 850 break; 851 case KVM_REG_MIPS_CP0_KSCRATCH1: 852 *v = (long)kvm_read_c0_guest_kscratch1(cop0); 853 break; 854 case KVM_REG_MIPS_CP0_KSCRATCH2: 855 *v = (long)kvm_read_c0_guest_kscratch2(cop0); 856 break; 857 case KVM_REG_MIPS_CP0_KSCRATCH3: 858 *v = (long)kvm_read_c0_guest_kscratch3(cop0); 859 break; 860 case KVM_REG_MIPS_CP0_KSCRATCH4: 861 *v = (long)kvm_read_c0_guest_kscratch4(cop0); 862 break; 863 case KVM_REG_MIPS_CP0_KSCRATCH5: 864 *v = (long)kvm_read_c0_guest_kscratch5(cop0); 865 break; 866 case KVM_REG_MIPS_CP0_KSCRATCH6: 867 *v = (long)kvm_read_c0_guest_kscratch6(cop0); 868 break; 869 default: 870 return -EINVAL; 871 } 872 return 0; 873} 874 875static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu, 876 const struct kvm_one_reg *reg, 877 s64 v) 878{ 879 struct mips_coproc *cop0 = vcpu->arch.cop0; 880 int ret = 0; 881 unsigned int cur, change; 882 883 switch (reg->id) { 884 case KVM_REG_MIPS_CP0_INDEX: 885 kvm_write_c0_guest_index(cop0, v); 886 break; 887 case KVM_REG_MIPS_CP0_ENTRYLO0: 888 kvm_write_c0_guest_entrylo0(cop0, v); 889 break; 890 case KVM_REG_MIPS_CP0_ENTRYLO1: 891 kvm_write_c0_guest_entrylo1(cop0, v); 892 break; 893 case KVM_REG_MIPS_CP0_CONTEXT: 894 kvm_write_c0_guest_context(cop0, v); 895 break; 896 case KVM_REG_MIPS_CP0_USERLOCAL: 897 kvm_write_c0_guest_userlocal(cop0, v); 898 break; 899 case KVM_REG_MIPS_CP0_PAGEMASK: 900 kvm_write_c0_guest_pagemask(cop0, v); 901 break; 902 case KVM_REG_MIPS_CP0_WIRED: 903 kvm_write_c0_guest_wired(cop0, v); 904 break; 905 case KVM_REG_MIPS_CP0_HWRENA: 906 kvm_write_c0_guest_hwrena(cop0, v); 907 break; 908 case KVM_REG_MIPS_CP0_BADVADDR: 909 kvm_write_c0_guest_badvaddr(cop0, v); 910 break; 911 case KVM_REG_MIPS_CP0_ENTRYHI: 912 kvm_write_c0_guest_entryhi(cop0, v); 913 break; 914 case KVM_REG_MIPS_CP0_STATUS: 915 kvm_write_c0_guest_status(cop0, v); 916 break; 917 case KVM_REG_MIPS_CP0_INTCTL: 918 /* No VInt, so no VS, read-only for now */ 919 break; 920 case KVM_REG_MIPS_CP0_EPC: 921 kvm_write_c0_guest_epc(cop0, v); 922 break; 923 case KVM_REG_MIPS_CP0_PRID: 924 kvm_write_c0_guest_prid(cop0, v); 925 break; 926 case KVM_REG_MIPS_CP0_EBASE: 927 /* 928 * Allow core number to be written, but the exception base must 929 * remain in guest KSeg0. 930 */ 931 kvm_change_c0_guest_ebase(cop0, 0x1ffff000 | MIPS_EBASE_CPUNUM, 932 v); 933 break; 934 case KVM_REG_MIPS_CP0_COUNT: 935 kvm_mips_write_count(vcpu, v); 936 break; 937 case KVM_REG_MIPS_CP0_COMPARE: 938 kvm_mips_write_compare(vcpu, v, false); 939 break; 940 case KVM_REG_MIPS_CP0_CAUSE: 941 /* 942 * If the timer is stopped or started (DC bit) it must look 943 * atomic with changes to the interrupt pending bits (TI, IRQ5). 944 * A timer interrupt should not happen in between. 945 */ 946 if ((kvm_read_c0_guest_cause(cop0) ^ v) & CAUSEF_DC) { 947 if (v & CAUSEF_DC) { 948 /* disable timer first */ 949 kvm_mips_count_disable_cause(vcpu); 950 kvm_change_c0_guest_cause(cop0, (u32)~CAUSEF_DC, 951 v); 952 } else { 953 /* enable timer last */ 954 kvm_change_c0_guest_cause(cop0, (u32)~CAUSEF_DC, 955 v); 956 kvm_mips_count_enable_cause(vcpu); 957 } 958 } else { 959 kvm_write_c0_guest_cause(cop0, v); 960 } 961 break; 962 case KVM_REG_MIPS_CP0_CONFIG: 963 /* read-only for now */ 964 break; 965 case KVM_REG_MIPS_CP0_CONFIG1: 966 cur = kvm_read_c0_guest_config1(cop0); 967 change = (cur ^ v) & kvm_mips_config1_wrmask(vcpu); 968 if (change) { 969 v = cur ^ change; 970 kvm_write_c0_guest_config1(cop0, v); 971 } 972 break; 973 case KVM_REG_MIPS_CP0_CONFIG2: 974 /* read-only for now */ 975 break; 976 case KVM_REG_MIPS_CP0_CONFIG3: 977 cur = kvm_read_c0_guest_config3(cop0); 978 change = (cur ^ v) & kvm_mips_config3_wrmask(vcpu); 979 if (change) { 980 v = cur ^ change; 981 kvm_write_c0_guest_config3(cop0, v); 982 } 983 break; 984 case KVM_REG_MIPS_CP0_CONFIG4: 985 cur = kvm_read_c0_guest_config4(cop0); 986 change = (cur ^ v) & kvm_mips_config4_wrmask(vcpu); 987 if (change) { 988 v = cur ^ change; 989 kvm_write_c0_guest_config4(cop0, v); 990 } 991 break; 992 case KVM_REG_MIPS_CP0_CONFIG5: 993 cur = kvm_read_c0_guest_config5(cop0); 994 change = (cur ^ v) & kvm_mips_config5_wrmask(vcpu); 995 if (change) { 996 v = cur ^ change; 997 kvm_write_c0_guest_config5(cop0, v); 998 } 999 break; 1000 case KVM_REG_MIPS_CP0_CONFIG7: 1001 /* writes ignored */ 1002 break; 1003 case KVM_REG_MIPS_COUNT_CTL: 1004 ret = kvm_mips_set_count_ctl(vcpu, v); 1005 break; 1006 case KVM_REG_MIPS_COUNT_RESUME: 1007 ret = kvm_mips_set_count_resume(vcpu, v); 1008 break; 1009 case KVM_REG_MIPS_COUNT_HZ: 1010 ret = kvm_mips_set_count_hz(vcpu, v); 1011 break; 1012 case KVM_REG_MIPS_CP0_ERROREPC: 1013 kvm_write_c0_guest_errorepc(cop0, v); 1014 break; 1015 case KVM_REG_MIPS_CP0_KSCRATCH1: 1016 kvm_write_c0_guest_kscratch1(cop0, v); 1017 break; 1018 case KVM_REG_MIPS_CP0_KSCRATCH2: 1019 kvm_write_c0_guest_kscratch2(cop0, v); 1020 break; 1021 case KVM_REG_MIPS_CP0_KSCRATCH3: 1022 kvm_write_c0_guest_kscratch3(cop0, v); 1023 break; 1024 case KVM_REG_MIPS_CP0_KSCRATCH4: 1025 kvm_write_c0_guest_kscratch4(cop0, v); 1026 break; 1027 case KVM_REG_MIPS_CP0_KSCRATCH5: 1028 kvm_write_c0_guest_kscratch5(cop0, v); 1029 break; 1030 case KVM_REG_MIPS_CP0_KSCRATCH6: 1031 kvm_write_c0_guest_kscratch6(cop0, v); 1032 break; 1033 default: 1034 return -EINVAL; 1035 } 1036 return ret; 1037} 1038 1039static int kvm_trap_emul_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1040{ 1041 struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm; 1042 struct mm_struct *user_mm = &vcpu->arch.guest_user_mm; 1043 struct mm_struct *mm; 1044 1045 /* 1046 * Were we in guest context? If so, restore the appropriate ASID based 1047 * on the mode of the Guest (Kernel/User). 1048 */ 1049 if (current->flags & PF_VCPU) { 1050 mm = KVM_GUEST_KERNEL_MODE(vcpu) ? kern_mm : user_mm; 1051 check_switch_mmu_context(mm); 1052 kvm_mips_suspend_mm(cpu); 1053 ehb(); 1054 } 1055 1056 return 0; 1057} 1058 1059static int kvm_trap_emul_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 1060{ 1061 kvm_lose_fpu(vcpu); 1062 1063 if (current->flags & PF_VCPU) { 1064 /* Restore normal Linux process memory map */ 1065 check_switch_mmu_context(current->mm); 1066 kvm_mips_resume_mm(cpu); 1067 ehb(); 1068 } 1069 1070 return 0; 1071} 1072 1073static void kvm_trap_emul_check_requests(struct kvm_vcpu *vcpu, int cpu, 1074 bool reload_asid) 1075{ 1076 struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm; 1077 struct mm_struct *user_mm = &vcpu->arch.guest_user_mm; 1078 struct mm_struct *mm; 1079 int i; 1080 1081 if (likely(!kvm_request_pending(vcpu))) 1082 return; 1083 1084 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 1085 /* 1086 * Both kernel & user GVA mappings must be invalidated. The 1087 * caller is just about to check whether the ASID is stale 1088 * anyway so no need to reload it here. 1089 */ 1090 kvm_mips_flush_gva_pt(kern_mm->pgd, KMF_GPA | KMF_KERN); 1091 kvm_mips_flush_gva_pt(user_mm->pgd, KMF_GPA | KMF_USER); 1092 for_each_possible_cpu(i) { 1093 set_cpu_context(i, kern_mm, 0); 1094 set_cpu_context(i, user_mm, 0); 1095 } 1096 1097 /* Generate new ASID for current mode */ 1098 if (reload_asid) { 1099 mm = KVM_GUEST_KERNEL_MODE(vcpu) ? kern_mm : user_mm; 1100 get_new_mmu_context(mm); 1101 htw_stop(); 1102 write_c0_entryhi(cpu_asid(cpu, mm)); 1103 TLBMISS_HANDLER_SETUP_PGD(mm->pgd); 1104 htw_start(); 1105 } 1106 } 1107} 1108 1109/** 1110 * kvm_trap_emul_gva_lockless_begin() - Begin lockless access to GVA space. 1111 * @vcpu: VCPU pointer. 1112 * 1113 * Call before a GVA space access outside of guest mode, to ensure that 1114 * asynchronous TLB flush requests are handled or delayed until completion of 1115 * the GVA access (as indicated by a matching kvm_trap_emul_gva_lockless_end()). 1116 * 1117 * Should be called with IRQs already enabled. 1118 */ 1119void kvm_trap_emul_gva_lockless_begin(struct kvm_vcpu *vcpu) 1120{ 1121 /* We re-enable IRQs in kvm_trap_emul_gva_lockless_end() */ 1122 WARN_ON_ONCE(irqs_disabled()); 1123 1124 /* 1125 * The caller is about to access the GVA space, so we set the mode to 1126 * force TLB flush requests to send an IPI, and also disable IRQs to 1127 * delay IPI handling until kvm_trap_emul_gva_lockless_end(). 1128 */ 1129 local_irq_disable(); 1130 1131 /* 1132 * Make sure the read of VCPU requests is not reordered ahead of the 1133 * write to vcpu->mode, or we could miss a TLB flush request while 1134 * the requester sees the VCPU as outside of guest mode and not needing 1135 * an IPI. 1136 */ 1137 smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES); 1138 1139 /* 1140 * If a TLB flush has been requested (potentially while 1141 * OUTSIDE_GUEST_MODE and assumed immediately effective), perform it 1142 * before accessing the GVA space, and be sure to reload the ASID if 1143 * necessary as it'll be immediately used. 1144 * 1145 * TLB flush requests after this check will trigger an IPI due to the 1146 * mode change above, which will be delayed due to IRQs disabled. 1147 */ 1148 kvm_trap_emul_check_requests(vcpu, smp_processor_id(), true); 1149} 1150 1151/** 1152 * kvm_trap_emul_gva_lockless_end() - End lockless access to GVA space. 1153 * @vcpu: VCPU pointer. 1154 * 1155 * Called after a GVA space access outside of guest mode. Should have a matching 1156 * call to kvm_trap_emul_gva_lockless_begin(). 1157 */ 1158void kvm_trap_emul_gva_lockless_end(struct kvm_vcpu *vcpu) 1159{ 1160 /* 1161 * Make sure the write to vcpu->mode is not reordered in front of GVA 1162 * accesses, or a TLB flush requester may not think it necessary to send 1163 * an IPI. 1164 */ 1165 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); 1166 1167 /* 1168 * Now that the access to GVA space is complete, its safe for pending 1169 * TLB flush request IPIs to be handled (which indicates completion). 1170 */ 1171 local_irq_enable(); 1172} 1173 1174static void kvm_trap_emul_vcpu_reenter(struct kvm_vcpu *vcpu) 1175{ 1176 struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm; 1177 struct mm_struct *user_mm = &vcpu->arch.guest_user_mm; 1178 struct mm_struct *mm; 1179 struct mips_coproc *cop0 = vcpu->arch.cop0; 1180 int i, cpu = smp_processor_id(); 1181 unsigned int gasid; 1182 1183 /* 1184 * No need to reload ASID, IRQs are disabled already so there's no rush, 1185 * and we'll check if we need to regenerate below anyway before 1186 * re-entering the guest. 1187 */ 1188 kvm_trap_emul_check_requests(vcpu, cpu, false); 1189 1190 if (KVM_GUEST_KERNEL_MODE(vcpu)) { 1191 mm = kern_mm; 1192 } else { 1193 mm = user_mm; 1194 1195 /* 1196 * Lazy host ASID regeneration / PT flush for guest user mode. 1197 * If the guest ASID has changed since the last guest usermode 1198 * execution, invalidate the stale TLB entries and flush GVA PT 1199 * entries too. 1200 */ 1201 gasid = kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID; 1202 if (gasid != vcpu->arch.last_user_gasid) { 1203 kvm_mips_flush_gva_pt(user_mm->pgd, KMF_USER); 1204 for_each_possible_cpu(i) 1205 set_cpu_context(i, user_mm, 0); 1206 vcpu->arch.last_user_gasid = gasid; 1207 } 1208 } 1209 1210 /* 1211 * Check if ASID is stale. This may happen due to a TLB flush request or 1212 * a lazy user MM invalidation. 1213 */ 1214 check_mmu_context(mm); 1215} 1216 1217static int kvm_trap_emul_vcpu_run(struct kvm_vcpu *vcpu) 1218{ 1219 int cpu = smp_processor_id(); 1220 int r; 1221 1222 /* Check if we have any exceptions/interrupts pending */ 1223 kvm_mips_deliver_interrupts(vcpu, 1224 kvm_read_c0_guest_cause(vcpu->arch.cop0)); 1225 1226 kvm_trap_emul_vcpu_reenter(vcpu); 1227 1228 /* 1229 * We use user accessors to access guest memory, but we don't want to 1230 * invoke Linux page faulting. 1231 */ 1232 pagefault_disable(); 1233 1234 /* Disable hardware page table walking while in guest */ 1235 htw_stop(); 1236 1237 /* 1238 * While in guest context we're in the guest's address space, not the 1239 * host process address space, so we need to be careful not to confuse 1240 * e.g. cache management IPIs. 1241 */ 1242 kvm_mips_suspend_mm(cpu); 1243 1244 r = vcpu->arch.vcpu_run(vcpu); 1245 1246 /* We may have migrated while handling guest exits */ 1247 cpu = smp_processor_id(); 1248 1249 /* Restore normal Linux process memory map */ 1250 check_switch_mmu_context(current->mm); 1251 kvm_mips_resume_mm(cpu); 1252 1253 htw_start(); 1254 1255 pagefault_enable(); 1256 1257 return r; 1258} 1259 1260static struct kvm_mips_callbacks kvm_trap_emul_callbacks = { 1261 /* exit handlers */ 1262 .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable, 1263 .handle_tlb_mod = kvm_trap_emul_handle_tlb_mod, 1264 .handle_tlb_st_miss = kvm_trap_emul_handle_tlb_st_miss, 1265 .handle_tlb_ld_miss = kvm_trap_emul_handle_tlb_ld_miss, 1266 .handle_addr_err_st = kvm_trap_emul_handle_addr_err_st, 1267 .handle_addr_err_ld = kvm_trap_emul_handle_addr_err_ld, 1268 .handle_syscall = kvm_trap_emul_handle_syscall, 1269 .handle_res_inst = kvm_trap_emul_handle_res_inst, 1270 .handle_break = kvm_trap_emul_handle_break, 1271 .handle_trap = kvm_trap_emul_handle_trap, 1272 .handle_msa_fpe = kvm_trap_emul_handle_msa_fpe, 1273 .handle_fpe = kvm_trap_emul_handle_fpe, 1274 .handle_msa_disabled = kvm_trap_emul_handle_msa_disabled, 1275 .handle_guest_exit = kvm_trap_emul_no_handler, 1276 1277 .hardware_enable = kvm_trap_emul_hardware_enable, 1278 .hardware_disable = kvm_trap_emul_hardware_disable, 1279 .check_extension = kvm_trap_emul_check_extension, 1280 .vcpu_init = kvm_trap_emul_vcpu_init, 1281 .vcpu_uninit = kvm_trap_emul_vcpu_uninit, 1282 .vcpu_setup = kvm_trap_emul_vcpu_setup, 1283 .flush_shadow_all = kvm_trap_emul_flush_shadow_all, 1284 .flush_shadow_memslot = kvm_trap_emul_flush_shadow_memslot, 1285 .gva_to_gpa = kvm_trap_emul_gva_to_gpa_cb, 1286 .queue_timer_int = kvm_mips_queue_timer_int_cb, 1287 .dequeue_timer_int = kvm_mips_dequeue_timer_int_cb, 1288 .queue_io_int = kvm_mips_queue_io_int_cb, 1289 .dequeue_io_int = kvm_mips_dequeue_io_int_cb, 1290 .irq_deliver = kvm_mips_irq_deliver_cb, 1291 .irq_clear = kvm_mips_irq_clear_cb, 1292 .num_regs = kvm_trap_emul_num_regs, 1293 .copy_reg_indices = kvm_trap_emul_copy_reg_indices, 1294 .get_one_reg = kvm_trap_emul_get_one_reg, 1295 .set_one_reg = kvm_trap_emul_set_one_reg, 1296 .vcpu_load = kvm_trap_emul_vcpu_load, 1297 .vcpu_put = kvm_trap_emul_vcpu_put, 1298 .vcpu_run = kvm_trap_emul_vcpu_run, 1299 .vcpu_reenter = kvm_trap_emul_vcpu_reenter, 1300}; 1301 1302int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks) 1303{ 1304 *install_callbacks = &kvm_trap_emul_callbacks; 1305 return 0; 1306} 1307