1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ASM_X86_PARAVIRT_H 3#define _ASM_X86_PARAVIRT_H 4/* Various instructions on x86 need to be replaced for 5 * para-virtualization: those hooks are defined here. */ 6 7#include <asm/paravirt_types.h> 8 9#ifdef CONFIG_PARAVIRT 10#include <asm/pgtable_types.h> 11#include <asm/asm.h> 12#include <asm/nospec-branch.h> 13 14#ifndef __ASSEMBLY__ 15#include <linux/bug.h> 16#include <linux/types.h> 17#include <linux/cpumask.h> 18#include <linux/static_call_types.h> 19#include <asm/frame.h> 20 21u64 dummy_steal_clock(int cpu); 22u64 dummy_sched_clock(void); 23 24DECLARE_STATIC_CALL(pv_steal_clock, dummy_steal_clock); 25DECLARE_STATIC_CALL(pv_sched_clock, dummy_sched_clock); 26 27void paravirt_set_sched_clock(u64 (*func)(void)); 28 29static __always_inline u64 paravirt_sched_clock(void) 30{ 31 return static_call(pv_sched_clock)(); 32} 33 34struct static_key; 35extern struct static_key paravirt_steal_enabled; 36extern struct static_key paravirt_steal_rq_enabled; 37 38__visible void __native_queued_spin_unlock(struct qspinlock *lock); 39bool pv_is_native_spin_unlock(void); 40__visible bool __native_vcpu_is_preempted(long cpu); 41bool pv_is_native_vcpu_is_preempted(void); 42 43static inline u64 paravirt_steal_clock(int cpu) 44{ 45 return static_call(pv_steal_clock)(cpu); 46} 47 48#ifdef CONFIG_PARAVIRT_SPINLOCKS 49void __init paravirt_set_cap(void); 50#endif 51 52/* The paravirtualized I/O functions */ 53static inline void slow_down_io(void) 54{ 55 PVOP_VCALL0(cpu.io_delay); 56#ifdef REALLY_SLOW_IO 57 PVOP_VCALL0(cpu.io_delay); 58 PVOP_VCALL0(cpu.io_delay); 59 PVOP_VCALL0(cpu.io_delay); 60#endif 61} 62 63void native_flush_tlb_local(void); 64void native_flush_tlb_global(void); 65void native_flush_tlb_one_user(unsigned long addr); 66void native_flush_tlb_multi(const struct cpumask *cpumask, 67 const struct flush_tlb_info *info); 68 69static inline void __flush_tlb_local(void) 70{ 71 PVOP_VCALL0(mmu.flush_tlb_user); 72} 73 74static inline void __flush_tlb_global(void) 75{ 76 PVOP_VCALL0(mmu.flush_tlb_kernel); 77} 78 79static inline void __flush_tlb_one_user(unsigned long addr) 80{ 81 PVOP_VCALL1(mmu.flush_tlb_one_user, addr); 82} 83 84static inline void __flush_tlb_multi(const struct cpumask *cpumask, 85 const struct flush_tlb_info *info) 86{ 87 PVOP_VCALL2(mmu.flush_tlb_multi, cpumask, info); 88} 89 90static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) 91{ 92 PVOP_VCALL2(mmu.tlb_remove_table, tlb, table); 93} 94 95static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 96{ 97 PVOP_VCALL1(mmu.exit_mmap, mm); 98} 99 100static inline void notify_page_enc_status_changed(unsigned long pfn, 101 int npages, bool enc) 102{ 103 PVOP_VCALL3(mmu.notify_page_enc_status_changed, pfn, npages, enc); 104} 105 106#ifdef CONFIG_PARAVIRT_XXL 107static inline void load_sp0(unsigned long sp0) 108{ 109 PVOP_VCALL1(cpu.load_sp0, sp0); 110} 111 112/* The paravirtualized CPUID instruction. */ 113static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 114 unsigned int *ecx, unsigned int *edx) 115{ 116 PVOP_VCALL4(cpu.cpuid, eax, ebx, ecx, edx); 117} 118 119/* 120 * These special macros can be used to get or set a debugging register 121 */ 122static __always_inline unsigned long paravirt_get_debugreg(int reg) 123{ 124 return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg); 125} 126#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 127static __always_inline void set_debugreg(unsigned long val, int reg) 128{ 129 PVOP_VCALL2(cpu.set_debugreg, reg, val); 130} 131 132static inline unsigned long read_cr0(void) 133{ 134 return PVOP_CALL0(unsigned long, cpu.read_cr0); 135} 136 137static inline void write_cr0(unsigned long x) 138{ 139 PVOP_VCALL1(cpu.write_cr0, x); 140} 141 142static __always_inline unsigned long read_cr2(void) 143{ 144 return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2, 145 "mov %%cr2, %%rax;", 146 ALT_NOT(X86_FEATURE_XENPV)); 147} 148 149static __always_inline void write_cr2(unsigned long x) 150{ 151 PVOP_VCALL1(mmu.write_cr2, x); 152} 153 154static inline unsigned long __read_cr3(void) 155{ 156 return PVOP_ALT_CALL0(unsigned long, mmu.read_cr3, 157 "mov %%cr3, %%rax;", ALT_NOT(X86_FEATURE_XENPV)); 158} 159 160static inline void write_cr3(unsigned long x) 161{ 162 PVOP_ALT_VCALL1(mmu.write_cr3, x, 163 "mov %%rdi, %%cr3", ALT_NOT(X86_FEATURE_XENPV)); 164} 165 166static inline void __write_cr4(unsigned long x) 167{ 168 PVOP_VCALL1(cpu.write_cr4, x); 169} 170 171static __always_inline void arch_safe_halt(void) 172{ 173 PVOP_VCALL0(irq.safe_halt); 174} 175 176static inline void halt(void) 177{ 178 PVOP_VCALL0(irq.halt); 179} 180 181extern noinstr void pv_native_wbinvd(void); 182 183static __always_inline void wbinvd(void) 184{ 185 PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT(X86_FEATURE_XENPV)); 186} 187 188static inline u64 paravirt_read_msr(unsigned msr) 189{ 190 return PVOP_CALL1(u64, cpu.read_msr, msr); 191} 192 193static inline void paravirt_write_msr(unsigned msr, 194 unsigned low, unsigned high) 195{ 196 PVOP_VCALL3(cpu.write_msr, msr, low, high); 197} 198 199static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) 200{ 201 return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err); 202} 203 204static inline int paravirt_write_msr_safe(unsigned msr, 205 unsigned low, unsigned high) 206{ 207 return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high); 208} 209 210#define rdmsr(msr, val1, val2) \ 211do { \ 212 u64 _l = paravirt_read_msr(msr); \ 213 val1 = (u32)_l; \ 214 val2 = _l >> 32; \ 215} while (0) 216 217#define wrmsr(msr, val1, val2) \ 218do { \ 219 paravirt_write_msr(msr, val1, val2); \ 220} while (0) 221 222#define rdmsrl(msr, val) \ 223do { \ 224 val = paravirt_read_msr(msr); \ 225} while (0) 226 227static inline void wrmsrl(unsigned msr, u64 val) 228{ 229 wrmsr(msr, (u32)val, (u32)(val>>32)); 230} 231 232#define wrmsr_safe(msr, a, b) paravirt_write_msr_safe(msr, a, b) 233 234/* rdmsr with exception handling */ 235#define rdmsr_safe(msr, a, b) \ 236({ \ 237 int _err; \ 238 u64 _l = paravirt_read_msr_safe(msr, &_err); \ 239 (*a) = (u32)_l; \ 240 (*b) = _l >> 32; \ 241 _err; \ 242}) 243 244static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) 245{ 246 int err; 247 248 *p = paravirt_read_msr_safe(msr, &err); 249 return err; 250} 251 252static inline unsigned long long paravirt_read_pmc(int counter) 253{ 254 return PVOP_CALL1(u64, cpu.read_pmc, counter); 255} 256 257#define rdpmc(counter, low, high) \ 258do { \ 259 u64 _l = paravirt_read_pmc(counter); \ 260 low = (u32)_l; \ 261 high = _l >> 32; \ 262} while (0) 263 264#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter)) 265 266static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 267{ 268 PVOP_VCALL2(cpu.alloc_ldt, ldt, entries); 269} 270 271static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 272{ 273 PVOP_VCALL2(cpu.free_ldt, ldt, entries); 274} 275 276static inline void load_TR_desc(void) 277{ 278 PVOP_VCALL0(cpu.load_tr_desc); 279} 280static inline void load_gdt(const struct desc_ptr *dtr) 281{ 282 PVOP_VCALL1(cpu.load_gdt, dtr); 283} 284static inline void load_idt(const struct desc_ptr *dtr) 285{ 286 PVOP_VCALL1(cpu.load_idt, dtr); 287} 288static inline void set_ldt(const void *addr, unsigned entries) 289{ 290 PVOP_VCALL2(cpu.set_ldt, addr, entries); 291} 292static inline unsigned long paravirt_store_tr(void) 293{ 294 return PVOP_CALL0(unsigned long, cpu.store_tr); 295} 296 297#define store_tr(tr) ((tr) = paravirt_store_tr()) 298static inline void load_TLS(struct thread_struct *t, unsigned cpu) 299{ 300 PVOP_VCALL2(cpu.load_tls, t, cpu); 301} 302 303static inline void load_gs_index(unsigned int gs) 304{ 305 PVOP_VCALL1(cpu.load_gs_index, gs); 306} 307 308static inline void write_ldt_entry(struct desc_struct *dt, int entry, 309 const void *desc) 310{ 311 PVOP_VCALL3(cpu.write_ldt_entry, dt, entry, desc); 312} 313 314static inline void write_gdt_entry(struct desc_struct *dt, int entry, 315 void *desc, int type) 316{ 317 PVOP_VCALL4(cpu.write_gdt_entry, dt, entry, desc, type); 318} 319 320static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 321{ 322 PVOP_VCALL3(cpu.write_idt_entry, dt, entry, g); 323} 324 325#ifdef CONFIG_X86_IOPL_IOPERM 326static inline void tss_invalidate_io_bitmap(void) 327{ 328 PVOP_VCALL0(cpu.invalidate_io_bitmap); 329} 330 331static inline void tss_update_io_bitmap(void) 332{ 333 PVOP_VCALL0(cpu.update_io_bitmap); 334} 335#endif 336 337static inline void paravirt_enter_mmap(struct mm_struct *next) 338{ 339 PVOP_VCALL1(mmu.enter_mmap, next); 340} 341 342static inline int paravirt_pgd_alloc(struct mm_struct *mm) 343{ 344 return PVOP_CALL1(int, mmu.pgd_alloc, mm); 345} 346 347static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 348{ 349 PVOP_VCALL2(mmu.pgd_free, mm, pgd); 350} 351 352static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 353{ 354 PVOP_VCALL2(mmu.alloc_pte, mm, pfn); 355} 356static inline void paravirt_release_pte(unsigned long pfn) 357{ 358 PVOP_VCALL1(mmu.release_pte, pfn); 359} 360 361static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 362{ 363 PVOP_VCALL2(mmu.alloc_pmd, mm, pfn); 364} 365 366static inline void paravirt_release_pmd(unsigned long pfn) 367{ 368 PVOP_VCALL1(mmu.release_pmd, pfn); 369} 370 371static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 372{ 373 PVOP_VCALL2(mmu.alloc_pud, mm, pfn); 374} 375static inline void paravirt_release_pud(unsigned long pfn) 376{ 377 PVOP_VCALL1(mmu.release_pud, pfn); 378} 379 380static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) 381{ 382 PVOP_VCALL2(mmu.alloc_p4d, mm, pfn); 383} 384 385static inline void paravirt_release_p4d(unsigned long pfn) 386{ 387 PVOP_VCALL1(mmu.release_p4d, pfn); 388} 389 390static inline pte_t __pte(pteval_t val) 391{ 392 return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, mmu.make_pte, val, 393 "mov %%rdi, %%rax", 394 ALT_NOT(X86_FEATURE_XENPV)) }; 395} 396 397static inline pteval_t pte_val(pte_t pte) 398{ 399 return PVOP_ALT_CALLEE1(pteval_t, mmu.pte_val, pte.pte, 400 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 401} 402 403static inline pgd_t __pgd(pgdval_t val) 404{ 405 return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, mmu.make_pgd, val, 406 "mov %%rdi, %%rax", 407 ALT_NOT(X86_FEATURE_XENPV)) }; 408} 409 410static inline pgdval_t pgd_val(pgd_t pgd) 411{ 412 return PVOP_ALT_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd, 413 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 414} 415 416#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 417static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, 418 pte_t *ptep) 419{ 420 pteval_t ret; 421 422 ret = PVOP_CALL3(pteval_t, mmu.ptep_modify_prot_start, vma, addr, ptep); 423 424 return (pte_t) { .pte = ret }; 425} 426 427static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, 428 pte_t *ptep, pte_t old_pte, pte_t pte) 429{ 430 431 PVOP_VCALL4(mmu.ptep_modify_prot_commit, vma, addr, ptep, pte.pte); 432} 433 434static inline void set_pte(pte_t *ptep, pte_t pte) 435{ 436 PVOP_VCALL2(mmu.set_pte, ptep, pte.pte); 437} 438 439static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 440{ 441 PVOP_VCALL2(mmu.set_pmd, pmdp, native_pmd_val(pmd)); 442} 443 444static inline pmd_t __pmd(pmdval_t val) 445{ 446 return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, mmu.make_pmd, val, 447 "mov %%rdi, %%rax", 448 ALT_NOT(X86_FEATURE_XENPV)) }; 449} 450 451static inline pmdval_t pmd_val(pmd_t pmd) 452{ 453 return PVOP_ALT_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd, 454 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 455} 456 457static inline void set_pud(pud_t *pudp, pud_t pud) 458{ 459 PVOP_VCALL2(mmu.set_pud, pudp, native_pud_val(pud)); 460} 461 462static inline pud_t __pud(pudval_t val) 463{ 464 pudval_t ret; 465 466 ret = PVOP_ALT_CALLEE1(pudval_t, mmu.make_pud, val, 467 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 468 469 return (pud_t) { ret }; 470} 471 472static inline pudval_t pud_val(pud_t pud) 473{ 474 return PVOP_ALT_CALLEE1(pudval_t, mmu.pud_val, pud.pud, 475 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 476} 477 478static inline void pud_clear(pud_t *pudp) 479{ 480 set_pud(pudp, native_make_pud(0)); 481} 482 483static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 484{ 485 p4dval_t val = native_p4d_val(p4d); 486 487 PVOP_VCALL2(mmu.set_p4d, p4dp, val); 488} 489 490#if CONFIG_PGTABLE_LEVELS >= 5 491 492static inline p4d_t __p4d(p4dval_t val) 493{ 494 p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val, 495 "mov %%rdi, %%rax", 496 ALT_NOT(X86_FEATURE_XENPV)); 497 498 return (p4d_t) { ret }; 499} 500 501static inline p4dval_t p4d_val(p4d_t p4d) 502{ 503 return PVOP_ALT_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d, 504 "mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV)); 505} 506 507static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) 508{ 509 PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd)); 510} 511 512#define set_pgd(pgdp, pgdval) do { \ 513 if (pgtable_l5_enabled()) \ 514 __set_pgd(pgdp, pgdval); \ 515 else \ 516 set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd }); \ 517} while (0) 518 519#define pgd_clear(pgdp) do { \ 520 if (pgtable_l5_enabled()) \ 521 set_pgd(pgdp, native_make_pgd(0)); \ 522} while (0) 523 524#endif /* CONFIG_PGTABLE_LEVELS == 5 */ 525 526static inline void p4d_clear(p4d_t *p4dp) 527{ 528 set_p4d(p4dp, native_make_p4d(0)); 529} 530 531static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 532{ 533 set_pte(ptep, pte); 534} 535 536static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 537 pte_t *ptep) 538{ 539 set_pte(ptep, native_make_pte(0)); 540} 541 542static inline void pmd_clear(pmd_t *pmdp) 543{ 544 set_pmd(pmdp, native_make_pmd(0)); 545} 546 547#define __HAVE_ARCH_START_CONTEXT_SWITCH 548static inline void arch_start_context_switch(struct task_struct *prev) 549{ 550 PVOP_VCALL1(cpu.start_context_switch, prev); 551} 552 553static inline void arch_end_context_switch(struct task_struct *next) 554{ 555 PVOP_VCALL1(cpu.end_context_switch, next); 556} 557 558#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 559static inline void arch_enter_lazy_mmu_mode(void) 560{ 561 PVOP_VCALL0(mmu.lazy_mode.enter); 562} 563 564static inline void arch_leave_lazy_mmu_mode(void) 565{ 566 PVOP_VCALL0(mmu.lazy_mode.leave); 567} 568 569static inline void arch_flush_lazy_mmu_mode(void) 570{ 571 PVOP_VCALL0(mmu.lazy_mode.flush); 572} 573 574static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 575 phys_addr_t phys, pgprot_t flags) 576{ 577 pv_ops.mmu.set_fixmap(idx, phys, flags); 578} 579#endif 580 581#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 582 583static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 584 u32 val) 585{ 586 PVOP_VCALL2(lock.queued_spin_lock_slowpath, lock, val); 587} 588 589static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 590{ 591 PVOP_ALT_VCALLEE1(lock.queued_spin_unlock, lock, 592 "movb $0, (%%" _ASM_ARG1 ");", 593 ALT_NOT(X86_FEATURE_PVUNLOCK)); 594} 595 596static __always_inline void pv_wait(u8 *ptr, u8 val) 597{ 598 PVOP_VCALL2(lock.wait, ptr, val); 599} 600 601static __always_inline void pv_kick(int cpu) 602{ 603 PVOP_VCALL1(lock.kick, cpu); 604} 605 606static __always_inline bool pv_vcpu_is_preempted(long cpu) 607{ 608 return PVOP_ALT_CALLEE1(bool, lock.vcpu_is_preempted, cpu, 609 "xor %%" _ASM_AX ", %%" _ASM_AX ";", 610 ALT_NOT(X86_FEATURE_VCPUPREEMPT)); 611} 612 613void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock); 614bool __raw_callee_save___native_vcpu_is_preempted(long cpu); 615 616#endif /* SMP && PARAVIRT_SPINLOCKS */ 617 618#ifdef CONFIG_X86_32 619/* save and restore all caller-save registers, except return value */ 620#define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;" 621#define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;" 622#else 623/* save and restore all caller-save registers, except return value */ 624#define PV_SAVE_ALL_CALLER_REGS \ 625 "push %rcx;" \ 626 "push %rdx;" \ 627 "push %rsi;" \ 628 "push %rdi;" \ 629 "push %r8;" \ 630 "push %r9;" \ 631 "push %r10;" \ 632 "push %r11;" 633#define PV_RESTORE_ALL_CALLER_REGS \ 634 "pop %r11;" \ 635 "pop %r10;" \ 636 "pop %r9;" \ 637 "pop %r8;" \ 638 "pop %rdi;" \ 639 "pop %rsi;" \ 640 "pop %rdx;" \ 641 "pop %rcx;" 642#endif 643 644/* 645 * Generate a thunk around a function which saves all caller-save 646 * registers except for the return value. This allows C functions to 647 * be called from assembler code where fewer than normal registers are 648 * available. It may also help code generation around calls from C 649 * code if the common case doesn't use many registers. 650 * 651 * When a callee is wrapped in a thunk, the caller can assume that all 652 * arg regs and all scratch registers are preserved across the 653 * call. The return value in rax/eax will not be saved, even for void 654 * functions. 655 */ 656#define PV_THUNK_NAME(func) "__raw_callee_save_" #func 657#define __PV_CALLEE_SAVE_REGS_THUNK(func, section) \ 658 extern typeof(func) __raw_callee_save_##func; \ 659 \ 660 asm(".pushsection " section ", \"ax\";" \ 661 ".globl " PV_THUNK_NAME(func) ";" \ 662 ".type " PV_THUNK_NAME(func) ", @function;" \ 663 ASM_FUNC_ALIGN \ 664 PV_THUNK_NAME(func) ":" \ 665 ASM_ENDBR \ 666 FRAME_BEGIN \ 667 PV_SAVE_ALL_CALLER_REGS \ 668 "call " #func ";" \ 669 PV_RESTORE_ALL_CALLER_REGS \ 670 FRAME_END \ 671 ASM_RET \ 672 ".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \ 673 ".popsection") 674 675#define PV_CALLEE_SAVE_REGS_THUNK(func) \ 676 __PV_CALLEE_SAVE_REGS_THUNK(func, ".text") 677 678/* Get a reference to a callee-save function */ 679#define PV_CALLEE_SAVE(func) \ 680 ((struct paravirt_callee_save) { __raw_callee_save_##func }) 681 682/* Promise that "func" already uses the right calling convention */ 683#define __PV_IS_CALLEE_SAVE(func) \ 684 ((struct paravirt_callee_save) { func }) 685 686#ifdef CONFIG_PARAVIRT_XXL 687static __always_inline unsigned long arch_local_save_flags(void) 688{ 689 return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;", 690 ALT_NOT(X86_FEATURE_XENPV)); 691} 692 693static __always_inline void arch_local_irq_disable(void) 694{ 695 PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV)); 696} 697 698static __always_inline void arch_local_irq_enable(void) 699{ 700 PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV)); 701} 702 703static __always_inline unsigned long arch_local_irq_save(void) 704{ 705 unsigned long f; 706 707 f = arch_local_save_flags(); 708 arch_local_irq_disable(); 709 return f; 710} 711#endif 712 713 714/* Make sure as little as possible of this mess escapes. */ 715#undef PARAVIRT_CALL 716#undef __PVOP_CALL 717#undef __PVOP_VCALL 718#undef PVOP_VCALL0 719#undef PVOP_CALL0 720#undef PVOP_VCALL1 721#undef PVOP_CALL1 722#undef PVOP_VCALL2 723#undef PVOP_CALL2 724#undef PVOP_VCALL3 725#undef PVOP_CALL3 726#undef PVOP_VCALL4 727#undef PVOP_CALL4 728 729#define DEFINE_PARAVIRT_ASM(func, instr, sec) \ 730 asm (".pushsection " #sec ", \"ax\"\n" \ 731 ".global " #func "\n\t" \ 732 ".type " #func ", @function\n\t" \ 733 ASM_FUNC_ALIGN "\n" \ 734 #func ":\n\t" \ 735 ASM_ENDBR \ 736 instr "\n\t" \ 737 ASM_RET \ 738 ".size " #func ", . - " #func "\n\t" \ 739 ".popsection") 740 741extern void default_banner(void); 742void native_pv_lock_init(void) __init; 743 744#else /* __ASSEMBLY__ */ 745 746#define _PVSITE(ptype, ops, word, algn) \ 747771:; \ 748 ops; \ 749772:; \ 750 .pushsection .parainstructions,"a"; \ 751 .align algn; \ 752 word 771b; \ 753 .byte ptype; \ 754 .byte 772b-771b; \ 755 _ASM_ALIGN; \ 756 .popsection 757 758 759#ifdef CONFIG_X86_64 760#ifdef CONFIG_PARAVIRT_XXL 761 762#define PARA_PATCH(off) ((off) / 8) 763#define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8) 764#define PARA_INDIRECT(addr) *addr(%rip) 765 766#ifdef CONFIG_DEBUG_ENTRY 767.macro PARA_IRQ_save_fl 768 PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), 769 ANNOTATE_RETPOLINE_SAFE; 770 call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);) 771.endm 772 773#define SAVE_FLAGS ALTERNATIVE "PARA_IRQ_save_fl;", "pushf; pop %rax;", \ 774 ALT_NOT(X86_FEATURE_XENPV) 775#endif 776#endif /* CONFIG_PARAVIRT_XXL */ 777#endif /* CONFIG_X86_64 */ 778 779#endif /* __ASSEMBLY__ */ 780#else /* CONFIG_PARAVIRT */ 781# define default_banner x86_init_noop 782 783#ifndef __ASSEMBLY__ 784static inline void native_pv_lock_init(void) 785{ 786} 787#endif 788#endif /* !CONFIG_PARAVIRT */ 789 790#ifndef __ASSEMBLY__ 791#ifndef CONFIG_PARAVIRT_XXL 792static inline void paravirt_enter_mmap(struct mm_struct *mm) 793{ 794} 795#endif 796 797#ifndef CONFIG_PARAVIRT 798static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 799{ 800} 801#endif 802 803#ifndef CONFIG_PARAVIRT_SPINLOCKS 804static inline void paravirt_set_cap(void) 805{ 806} 807#endif 808#endif /* __ASSEMBLY__ */ 809#endif /* _ASM_X86_PARAVIRT_H */ 810