1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Xen event channels 4 * 5 * Xen models interrupts with abstract event channels. Because each 6 * domain gets 1024 event channels, but NR_IRQ is not that large, we 7 * must dynamically map irqs<->event channels. The event channels 8 * interface with the rest of the kernel by defining a xen interrupt 9 * chip. When an event is received, it is mapped to an irq and sent 10 * through the normal interrupt processing path. 11 * 12 * There are four kinds of events which can be mapped to an event 13 * channel: 14 * 15 * 1. Inter-domain notifications. This includes all the virtual 16 * device events, since they're driven by front-ends in another domain 17 * (typically dom0). 18 * 2. VIRQs, typically used for timers. These are per-cpu events. 19 * 3. IPIs. 20 * 4. PIRQs - Hardware interrupts. 21 * 22 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 23 */ 24 25#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 26 27#include <linux/linkage.h> 28#include <linux/interrupt.h> 29#include <linux/irq.h> 30#include <linux/moduleparam.h> 31#include <linux/string.h> 32#include <linux/memblock.h> 33#include <linux/slab.h> 34#include <linux/irqnr.h> 35#include <linux/pci.h> 36#include <linux/rcupdate.h> 37#include <linux/spinlock.h> 38#include <linux/cpuhotplug.h> 39#include <linux/atomic.h> 40#include <linux/ktime.h> 41 42#ifdef CONFIG_X86 43#include <asm/desc.h> 44#include <asm/ptrace.h> 45#include <asm/idtentry.h> 46#include <asm/irq.h> 47#include <asm/io_apic.h> 48#include <asm/i8259.h> 49#include <asm/xen/pci.h> 50#endif 51#include <asm/sync_bitops.h> 52#include <asm/xen/hypercall.h> 53#include <asm/xen/hypervisor.h> 54#include <xen/page.h> 55 56#include <xen/xen.h> 57#include <xen/hvm.h> 58#include <xen/xen-ops.h> 59#include <xen/events.h> 60#include <xen/interface/xen.h> 61#include <xen/interface/event_channel.h> 62#include <xen/interface/hvm/hvm_op.h> 63#include <xen/interface/hvm/params.h> 64#include <xen/interface/physdev.h> 65#include <xen/interface/sched.h> 66#include <xen/interface/vcpu.h> 67#include <asm/hw_irq.h> 68 69#include "events_internal.h" 70 71#undef MODULE_PARAM_PREFIX 72#define MODULE_PARAM_PREFIX "xen." 73 74/* Interrupt types. */ 75enum xen_irq_type { 76 IRQT_UNBOUND = 0, 77 IRQT_PIRQ, 78 IRQT_VIRQ, 79 IRQT_IPI, 80 IRQT_EVTCHN 81}; 82 83/* 84 * Packed IRQ information: 85 * type - enum xen_irq_type 86 * event channel - irq->event channel mapping 87 * cpu - cpu this event channel is bound to 88 * index - type-specific information: 89 * PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM 90 * guest, or GSI (real passthrough IRQ) of the device. 91 * VIRQ - virq number 92 * IPI - IPI vector 93 * EVTCHN - 94 */ 95struct irq_info { 96 struct list_head list; 97 struct list_head eoi_list; 98 struct rcu_work rwork; 99 short refcnt; 100 short spurious_cnt; 101 short type; /* type */ 102 u8 mask_reason; /* Why is event channel masked */ 103#define EVT_MASK_REASON_EXPLICIT 0x01 104#define EVT_MASK_REASON_TEMPORARY 0x02 105#define EVT_MASK_REASON_EOI_PENDING 0x04 106 u8 is_active; /* Is event just being handled? */ 107 unsigned irq; 108 evtchn_port_t evtchn; /* event channel */ 109 unsigned short cpu; /* cpu bound */ 110 unsigned short eoi_cpu; /* EOI must happen on this cpu-1 */ 111 unsigned int irq_epoch; /* If eoi_cpu valid: irq_epoch of event */ 112 u64 eoi_time; /* Time in jiffies when to EOI. */ 113 raw_spinlock_t lock; 114 115 union { 116 unsigned short virq; 117 enum ipi_vector ipi; 118 struct { 119 unsigned short pirq; 120 unsigned short gsi; 121 unsigned char vector; 122 unsigned char flags; 123 uint16_t domid; 124 } pirq; 125 } u; 126}; 127 128#define PIRQ_NEEDS_EOI (1 << 0) 129#define PIRQ_SHAREABLE (1 << 1) 130#define PIRQ_MSI_GROUP (1 << 2) 131 132static uint __read_mostly event_loop_timeout = 2; 133module_param(event_loop_timeout, uint, 0644); 134 135static uint __read_mostly event_eoi_delay = 10; 136module_param(event_eoi_delay, uint, 0644); 137 138const struct evtchn_ops *evtchn_ops; 139 140/* 141 * This lock protects updates to the following mapping and reference-count 142 * arrays. The lock does not need to be acquired to read the mapping tables. 143 */ 144static DEFINE_MUTEX(irq_mapping_update_lock); 145 146/* 147 * Lock hierarchy: 148 * 149 * irq_mapping_update_lock 150 * IRQ-desc lock 151 * percpu eoi_list_lock 152 * irq_info->lock 153 */ 154 155static LIST_HEAD(xen_irq_list_head); 156 157/* IRQ <-> VIRQ mapping. */ 158static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1}; 159 160/* IRQ <-> IPI mapping */ 161static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1}; 162 163static int **evtchn_to_irq; 164#ifdef CONFIG_X86 165static unsigned long *pirq_eoi_map; 166#endif 167static bool (*pirq_needs_eoi)(unsigned irq); 168 169#define EVTCHN_ROW(e) (e / (PAGE_SIZE/sizeof(**evtchn_to_irq))) 170#define EVTCHN_COL(e) (e % (PAGE_SIZE/sizeof(**evtchn_to_irq))) 171#define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq)) 172 173/* Xen will never allocate port zero for any purpose. */ 174#define VALID_EVTCHN(chn) ((chn) != 0) 175 176static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY]; 177 178static struct irq_chip xen_dynamic_chip; 179static struct irq_chip xen_lateeoi_chip; 180static struct irq_chip xen_percpu_chip; 181static struct irq_chip xen_pirq_chip; 182static void enable_dynirq(struct irq_data *data); 183static void disable_dynirq(struct irq_data *data); 184 185static DEFINE_PER_CPU(unsigned int, irq_epoch); 186 187static void clear_evtchn_to_irq_row(int *evtchn_row) 188{ 189 unsigned col; 190 191 for (col = 0; col < EVTCHN_PER_ROW; col++) 192 WRITE_ONCE(evtchn_row[col], -1); 193} 194 195static void clear_evtchn_to_irq_all(void) 196{ 197 unsigned row; 198 199 for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) { 200 if (evtchn_to_irq[row] == NULL) 201 continue; 202 clear_evtchn_to_irq_row(evtchn_to_irq[row]); 203 } 204} 205 206static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq) 207{ 208 unsigned row; 209 unsigned col; 210 int *evtchn_row; 211 212 if (evtchn >= xen_evtchn_max_channels()) 213 return -EINVAL; 214 215 row = EVTCHN_ROW(evtchn); 216 col = EVTCHN_COL(evtchn); 217 218 if (evtchn_to_irq[row] == NULL) { 219 /* Unallocated irq entries return -1 anyway */ 220 if (irq == -1) 221 return 0; 222 223 evtchn_row = (int *) __get_free_pages(GFP_KERNEL, 0); 224 if (evtchn_row == NULL) 225 return -ENOMEM; 226 227 clear_evtchn_to_irq_row(evtchn_row); 228 229 /* 230 * We've prepared an empty row for the mapping. If a different 231 * thread was faster inserting it, we can drop ours. 232 */ 233 if (cmpxchg(&evtchn_to_irq[row], NULL, evtchn_row) != NULL) 234 free_page((unsigned long) evtchn_row); 235 } 236 237 WRITE_ONCE(evtchn_to_irq[row][col], irq); 238 return 0; 239} 240 241int get_evtchn_to_irq(evtchn_port_t evtchn) 242{ 243 if (evtchn >= xen_evtchn_max_channels()) 244 return -1; 245 if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL) 246 return -1; 247 return READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]); 248} 249 250/* Get info for IRQ */ 251static struct irq_info *info_for_irq(unsigned irq) 252{ 253 if (irq < nr_legacy_irqs()) 254 return legacy_info_ptrs[irq]; 255 else 256 return irq_get_chip_data(irq); 257} 258 259static void set_info_for_irq(unsigned int irq, struct irq_info *info) 260{ 261 if (irq < nr_legacy_irqs()) 262 legacy_info_ptrs[irq] = info; 263 else 264 irq_set_chip_data(irq, info); 265} 266 267static void delayed_free_irq(struct work_struct *work) 268{ 269 struct irq_info *info = container_of(to_rcu_work(work), struct irq_info, 270 rwork); 271 unsigned int irq = info->irq; 272 273 /* Remove the info pointer only now, with no potential users left. */ 274 set_info_for_irq(irq, NULL); 275 276 kfree(info); 277 278 /* Legacy IRQ descriptors are managed by the arch. */ 279 if (irq >= nr_legacy_irqs()) 280 irq_free_desc(irq); 281} 282 283/* Constructors for packed IRQ information. */ 284static int xen_irq_info_common_setup(struct irq_info *info, 285 unsigned irq, 286 enum xen_irq_type type, 287 evtchn_port_t evtchn, 288 unsigned short cpu) 289{ 290 int ret; 291 292 BUG_ON(info->type != IRQT_UNBOUND && info->type != type); 293 294 info->type = type; 295 info->irq = irq; 296 info->evtchn = evtchn; 297 info->cpu = cpu; 298 info->mask_reason = EVT_MASK_REASON_EXPLICIT; 299 raw_spin_lock_init(&info->lock); 300 301 ret = set_evtchn_to_irq(evtchn, irq); 302 if (ret < 0) 303 return ret; 304 305 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN); 306 307 return xen_evtchn_port_setup(evtchn); 308} 309 310static int xen_irq_info_evtchn_setup(unsigned irq, 311 evtchn_port_t evtchn) 312{ 313 struct irq_info *info = info_for_irq(irq); 314 315 return xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0); 316} 317 318static int xen_irq_info_ipi_setup(unsigned cpu, 319 unsigned irq, 320 evtchn_port_t evtchn, 321 enum ipi_vector ipi) 322{ 323 struct irq_info *info = info_for_irq(irq); 324 325 info->u.ipi = ipi; 326 327 per_cpu(ipi_to_irq, cpu)[ipi] = irq; 328 329 return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0); 330} 331 332static int xen_irq_info_virq_setup(unsigned cpu, 333 unsigned irq, 334 evtchn_port_t evtchn, 335 unsigned virq) 336{ 337 struct irq_info *info = info_for_irq(irq); 338 339 info->u.virq = virq; 340 341 per_cpu(virq_to_irq, cpu)[virq] = irq; 342 343 return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0); 344} 345 346static int xen_irq_info_pirq_setup(unsigned irq, 347 evtchn_port_t evtchn, 348 unsigned pirq, 349 unsigned gsi, 350 uint16_t domid, 351 unsigned char flags) 352{ 353 struct irq_info *info = info_for_irq(irq); 354 355 info->u.pirq.pirq = pirq; 356 info->u.pirq.gsi = gsi; 357 info->u.pirq.domid = domid; 358 info->u.pirq.flags = flags; 359 360 return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0); 361} 362 363static void xen_irq_info_cleanup(struct irq_info *info) 364{ 365 set_evtchn_to_irq(info->evtchn, -1); 366 xen_evtchn_port_remove(info->evtchn, info->cpu); 367 info->evtchn = 0; 368} 369 370/* 371 * Accessors for packed IRQ information. 372 */ 373evtchn_port_t evtchn_from_irq(unsigned irq) 374{ 375 const struct irq_info *info = NULL; 376 377 if (likely(irq < nr_irqs)) 378 info = info_for_irq(irq); 379 if (!info) 380 return 0; 381 382 return info->evtchn; 383} 384 385unsigned int irq_from_evtchn(evtchn_port_t evtchn) 386{ 387 return get_evtchn_to_irq(evtchn); 388} 389EXPORT_SYMBOL_GPL(irq_from_evtchn); 390 391int irq_from_virq(unsigned int cpu, unsigned int virq) 392{ 393 return per_cpu(virq_to_irq, cpu)[virq]; 394} 395 396static enum ipi_vector ipi_from_irq(unsigned irq) 397{ 398 struct irq_info *info = info_for_irq(irq); 399 400 BUG_ON(info == NULL); 401 BUG_ON(info->type != IRQT_IPI); 402 403 return info->u.ipi; 404} 405 406static unsigned virq_from_irq(unsigned irq) 407{ 408 struct irq_info *info = info_for_irq(irq); 409 410 BUG_ON(info == NULL); 411 BUG_ON(info->type != IRQT_VIRQ); 412 413 return info->u.virq; 414} 415 416static unsigned pirq_from_irq(unsigned irq) 417{ 418 struct irq_info *info = info_for_irq(irq); 419 420 BUG_ON(info == NULL); 421 BUG_ON(info->type != IRQT_PIRQ); 422 423 return info->u.pirq.pirq; 424} 425 426static enum xen_irq_type type_from_irq(unsigned irq) 427{ 428 return info_for_irq(irq)->type; 429} 430 431static unsigned cpu_from_irq(unsigned irq) 432{ 433 return info_for_irq(irq)->cpu; 434} 435 436unsigned int cpu_from_evtchn(evtchn_port_t evtchn) 437{ 438 int irq = get_evtchn_to_irq(evtchn); 439 unsigned ret = 0; 440 441 if (irq != -1) 442 ret = cpu_from_irq(irq); 443 444 return ret; 445} 446 447static void do_mask(struct irq_info *info, u8 reason) 448{ 449 unsigned long flags; 450 451 raw_spin_lock_irqsave(&info->lock, flags); 452 453 if (!info->mask_reason) 454 mask_evtchn(info->evtchn); 455 456 info->mask_reason |= reason; 457 458 raw_spin_unlock_irqrestore(&info->lock, flags); 459} 460 461static void do_unmask(struct irq_info *info, u8 reason) 462{ 463 unsigned long flags; 464 465 raw_spin_lock_irqsave(&info->lock, flags); 466 467 info->mask_reason &= ~reason; 468 469 if (!info->mask_reason) 470 unmask_evtchn(info->evtchn); 471 472 raw_spin_unlock_irqrestore(&info->lock, flags); 473} 474 475#ifdef CONFIG_X86 476static bool pirq_check_eoi_map(unsigned irq) 477{ 478 return test_bit(pirq_from_irq(irq), pirq_eoi_map); 479} 480#endif 481 482static bool pirq_needs_eoi_flag(unsigned irq) 483{ 484 struct irq_info *info = info_for_irq(irq); 485 BUG_ON(info->type != IRQT_PIRQ); 486 487 return info->u.pirq.flags & PIRQ_NEEDS_EOI; 488} 489 490static void bind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int cpu) 491{ 492 int irq = get_evtchn_to_irq(evtchn); 493 struct irq_info *info = info_for_irq(irq); 494 495 BUG_ON(irq == -1); 496#ifdef CONFIG_SMP 497 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu)); 498#endif 499 xen_evtchn_port_bind_to_cpu(evtchn, cpu, info->cpu); 500 501 info->cpu = cpu; 502} 503 504/** 505 * notify_remote_via_irq - send event to remote end of event channel via irq 506 * @irq: irq of event channel to send event to 507 * 508 * Unlike notify_remote_via_evtchn(), this is safe to use across 509 * save/restore. Notifications on a broken connection are silently 510 * dropped. 511 */ 512void notify_remote_via_irq(int irq) 513{ 514 evtchn_port_t evtchn = evtchn_from_irq(irq); 515 516 if (VALID_EVTCHN(evtchn)) 517 notify_remote_via_evtchn(evtchn); 518} 519EXPORT_SYMBOL_GPL(notify_remote_via_irq); 520 521struct lateeoi_work { 522 struct delayed_work delayed; 523 spinlock_t eoi_list_lock; 524 struct list_head eoi_list; 525}; 526 527static DEFINE_PER_CPU(struct lateeoi_work, lateeoi); 528 529static void lateeoi_list_del(struct irq_info *info) 530{ 531 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); 532 unsigned long flags; 533 534 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 535 list_del_init(&info->eoi_list); 536 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 537} 538 539static void lateeoi_list_add(struct irq_info *info) 540{ 541 struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu); 542 struct irq_info *elem; 543 u64 now = get_jiffies_64(); 544 unsigned long delay; 545 unsigned long flags; 546 547 if (now < info->eoi_time) 548 delay = info->eoi_time - now; 549 else 550 delay = 1; 551 552 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 553 554 elem = list_first_entry_or_null(&eoi->eoi_list, struct irq_info, 555 eoi_list); 556 if (!elem || info->eoi_time < elem->eoi_time) { 557 list_add(&info->eoi_list, &eoi->eoi_list); 558 mod_delayed_work_on(info->eoi_cpu, system_wq, 559 &eoi->delayed, delay); 560 } else { 561 list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) { 562 if (elem->eoi_time <= info->eoi_time) 563 break; 564 } 565 list_add(&info->eoi_list, &elem->eoi_list); 566 } 567 568 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 569} 570 571static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious) 572{ 573 evtchn_port_t evtchn; 574 unsigned int cpu; 575 unsigned int delay = 0; 576 577 evtchn = info->evtchn; 578 if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list)) 579 return; 580 581 if (spurious) { 582 if ((1 << info->spurious_cnt) < (HZ << 2)) 583 info->spurious_cnt++; 584 if (info->spurious_cnt > 1) { 585 delay = 1 << (info->spurious_cnt - 2); 586 if (delay > HZ) 587 delay = HZ; 588 if (!info->eoi_time) 589 info->eoi_cpu = smp_processor_id(); 590 info->eoi_time = get_jiffies_64() + delay; 591 } 592 } else { 593 info->spurious_cnt = 0; 594 } 595 596 cpu = info->eoi_cpu; 597 if (info->eoi_time && 598 (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) { 599 lateeoi_list_add(info); 600 return; 601 } 602 603 info->eoi_time = 0; 604 605 /* is_active hasn't been reset yet, do it now. */ 606 smp_store_release(&info->is_active, 0); 607 do_unmask(info, EVT_MASK_REASON_EOI_PENDING); 608} 609 610static void xen_irq_lateeoi_worker(struct work_struct *work) 611{ 612 struct lateeoi_work *eoi; 613 struct irq_info *info; 614 u64 now = get_jiffies_64(); 615 unsigned long flags; 616 617 eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed); 618 619 rcu_read_lock(); 620 621 while (true) { 622 spin_lock_irqsave(&eoi->eoi_list_lock, flags); 623 624 info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info, 625 eoi_list); 626 627 if (info == NULL) 628 break; 629 630 if (now < info->eoi_time) { 631 mod_delayed_work_on(info->eoi_cpu, system_wq, 632 &eoi->delayed, 633 info->eoi_time - now); 634 break; 635 } 636 637 list_del_init(&info->eoi_list); 638 639 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 640 641 info->eoi_time = 0; 642 643 xen_irq_lateeoi_locked(info, false); 644 } 645 646 spin_unlock_irqrestore(&eoi->eoi_list_lock, flags); 647 648 rcu_read_unlock(); 649} 650 651static void xen_cpu_init_eoi(unsigned int cpu) 652{ 653 struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu); 654 655 INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker); 656 spin_lock_init(&eoi->eoi_list_lock); 657 INIT_LIST_HEAD(&eoi->eoi_list); 658} 659 660void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags) 661{ 662 struct irq_info *info; 663 664 rcu_read_lock(); 665 666 info = info_for_irq(irq); 667 668 if (info) 669 xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS); 670 671 rcu_read_unlock(); 672} 673EXPORT_SYMBOL_GPL(xen_irq_lateeoi); 674 675static void xen_irq_init(unsigned irq) 676{ 677 struct irq_info *info; 678 679#ifdef CONFIG_SMP 680 /* By default all event channels notify CPU#0. */ 681 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0)); 682#endif 683 684 info = kzalloc(sizeof(*info), GFP_KERNEL); 685 if (info == NULL) 686 panic("Unable to allocate metadata for IRQ%d\n", irq); 687 688 info->type = IRQT_UNBOUND; 689 info->refcnt = -1; 690 INIT_RCU_WORK(&info->rwork, delayed_free_irq); 691 692 set_info_for_irq(irq, info); 693 694 INIT_LIST_HEAD(&info->eoi_list); 695 list_add_tail(&info->list, &xen_irq_list_head); 696} 697 698static int __must_check xen_allocate_irqs_dynamic(int nvec) 699{ 700 int i, irq = irq_alloc_descs(-1, 0, nvec, -1); 701 702 if (irq >= 0) { 703 for (i = 0; i < nvec; i++) 704 xen_irq_init(irq + i); 705 } 706 707 return irq; 708} 709 710static inline int __must_check xen_allocate_irq_dynamic(void) 711{ 712 713 return xen_allocate_irqs_dynamic(1); 714} 715 716static int __must_check xen_allocate_irq_gsi(unsigned gsi) 717{ 718 int irq; 719 720 /* 721 * A PV guest has no concept of a GSI (since it has no ACPI 722 * nor access to/knowledge of the physical APICs). Therefore 723 * all IRQs are dynamically allocated from the entire IRQ 724 * space. 725 */ 726 if (xen_pv_domain() && !xen_initial_domain()) 727 return xen_allocate_irq_dynamic(); 728 729 /* Legacy IRQ descriptors are already allocated by the arch. */ 730 if (gsi < nr_legacy_irqs()) 731 irq = gsi; 732 else 733 irq = irq_alloc_desc_at(gsi, -1); 734 735 xen_irq_init(irq); 736 737 return irq; 738} 739 740static void xen_free_irq(unsigned irq) 741{ 742 struct irq_info *info = info_for_irq(irq); 743 744 if (WARN_ON(!info)) 745 return; 746 747 if (!list_empty(&info->eoi_list)) 748 lateeoi_list_del(info); 749 750 list_del(&info->list); 751 752 WARN_ON(info->refcnt > 0); 753 754 queue_rcu_work(system_wq, &info->rwork); 755} 756 757static void xen_evtchn_close(evtchn_port_t port) 758{ 759 struct evtchn_close close; 760 761 close.port = port; 762 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0) 763 BUG(); 764} 765 766static void event_handler_exit(struct irq_info *info) 767{ 768 smp_store_release(&info->is_active, 0); 769 clear_evtchn(info->evtchn); 770} 771 772static void pirq_query_unmask(int irq) 773{ 774 struct physdev_irq_status_query irq_status; 775 struct irq_info *info = info_for_irq(irq); 776 777 BUG_ON(info->type != IRQT_PIRQ); 778 779 irq_status.irq = pirq_from_irq(irq); 780 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 781 irq_status.flags = 0; 782 783 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI; 784 if (irq_status.flags & XENIRQSTAT_needs_eoi) 785 info->u.pirq.flags |= PIRQ_NEEDS_EOI; 786} 787 788static void eoi_pirq(struct irq_data *data) 789{ 790 struct irq_info *info = info_for_irq(data->irq); 791 evtchn_port_t evtchn = info ? info->evtchn : 0; 792 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) }; 793 int rc = 0; 794 795 if (!VALID_EVTCHN(evtchn)) 796 return; 797 798 if (unlikely(irqd_is_setaffinity_pending(data)) && 799 likely(!irqd_irq_disabled(data))) { 800 do_mask(info, EVT_MASK_REASON_TEMPORARY); 801 802 event_handler_exit(info); 803 804 irq_move_masked_irq(data); 805 806 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 807 } else 808 event_handler_exit(info); 809 810 if (pirq_needs_eoi(data->irq)) { 811 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); 812 WARN_ON(rc); 813 } 814} 815 816static void mask_ack_pirq(struct irq_data *data) 817{ 818 disable_dynirq(data); 819 eoi_pirq(data); 820} 821 822static unsigned int __startup_pirq(unsigned int irq) 823{ 824 struct evtchn_bind_pirq bind_pirq; 825 struct irq_info *info = info_for_irq(irq); 826 evtchn_port_t evtchn = evtchn_from_irq(irq); 827 int rc; 828 829 BUG_ON(info->type != IRQT_PIRQ); 830 831 if (VALID_EVTCHN(evtchn)) 832 goto out; 833 834 bind_pirq.pirq = pirq_from_irq(irq); 835 /* NB. We are happy to share unless we are probing. */ 836 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ? 837 BIND_PIRQ__WILL_SHARE : 0; 838 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq); 839 if (rc != 0) { 840 pr_warn("Failed to obtain physical IRQ %d\n", irq); 841 return 0; 842 } 843 evtchn = bind_pirq.port; 844 845 pirq_query_unmask(irq); 846 847 rc = set_evtchn_to_irq(evtchn, irq); 848 if (rc) 849 goto err; 850 851 info->evtchn = evtchn; 852 bind_evtchn_to_cpu(evtchn, 0); 853 854 rc = xen_evtchn_port_setup(evtchn); 855 if (rc) 856 goto err; 857 858out: 859 do_unmask(info, EVT_MASK_REASON_EXPLICIT); 860 861 eoi_pirq(irq_get_irq_data(irq)); 862 863 return 0; 864 865err: 866 pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc); 867 xen_evtchn_close(evtchn); 868 return 0; 869} 870 871static unsigned int startup_pirq(struct irq_data *data) 872{ 873 return __startup_pirq(data->irq); 874} 875 876static void shutdown_pirq(struct irq_data *data) 877{ 878 unsigned int irq = data->irq; 879 struct irq_info *info = info_for_irq(irq); 880 evtchn_port_t evtchn = evtchn_from_irq(irq); 881 882 BUG_ON(info->type != IRQT_PIRQ); 883 884 if (!VALID_EVTCHN(evtchn)) 885 return; 886 887 do_mask(info, EVT_MASK_REASON_EXPLICIT); 888 xen_evtchn_close(evtchn); 889 xen_irq_info_cleanup(info); 890} 891 892static void enable_pirq(struct irq_data *data) 893{ 894 enable_dynirq(data); 895} 896 897static void disable_pirq(struct irq_data *data) 898{ 899 disable_dynirq(data); 900} 901 902int xen_irq_from_gsi(unsigned gsi) 903{ 904 struct irq_info *info; 905 906 list_for_each_entry(info, &xen_irq_list_head, list) { 907 if (info->type != IRQT_PIRQ) 908 continue; 909 910 if (info->u.pirq.gsi == gsi) 911 return info->irq; 912 } 913 914 return -1; 915} 916EXPORT_SYMBOL_GPL(xen_irq_from_gsi); 917 918static void __unbind_from_irq(unsigned int irq) 919{ 920 evtchn_port_t evtchn = evtchn_from_irq(irq); 921 struct irq_info *info = info_for_irq(irq); 922 923 if (info->refcnt > 0) { 924 info->refcnt--; 925 if (info->refcnt != 0) 926 return; 927 } 928 929 if (VALID_EVTCHN(evtchn)) { 930 unsigned int cpu = cpu_from_irq(irq); 931 932 xen_evtchn_close(evtchn); 933 934 switch (type_from_irq(irq)) { 935 case IRQT_VIRQ: 936 per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1; 937 break; 938 case IRQT_IPI: 939 per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1; 940 break; 941 default: 942 break; 943 } 944 945 xen_irq_info_cleanup(info); 946 } 947 948 xen_free_irq(irq); 949} 950 951/* 952 * Do not make any assumptions regarding the relationship between the 953 * IRQ number returned here and the Xen pirq argument. 954 * 955 * Note: We don't assign an event channel until the irq actually started 956 * up. Return an existing irq if we've already got one for the gsi. 957 * 958 * Shareable implies level triggered, not shareable implies edge 959 * triggered here. 960 */ 961int xen_bind_pirq_gsi_to_irq(unsigned gsi, 962 unsigned pirq, int shareable, char *name) 963{ 964 int irq = -1; 965 struct physdev_irq irq_op; 966 int ret; 967 968 mutex_lock(&irq_mapping_update_lock); 969 970 irq = xen_irq_from_gsi(gsi); 971 if (irq != -1) { 972 pr_info("%s: returning irq %d for gsi %u\n", 973 __func__, irq, gsi); 974 goto out; 975 } 976 977 irq = xen_allocate_irq_gsi(gsi); 978 if (irq < 0) 979 goto out; 980 981 irq_op.irq = irq; 982 irq_op.vector = 0; 983 984 /* Only the privileged domain can do this. For non-priv, the pcifront 985 * driver provides a PCI bus that does the call to do exactly 986 * this in the priv domain. */ 987 if (xen_initial_domain() && 988 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) { 989 xen_free_irq(irq); 990 irq = -ENOSPC; 991 goto out; 992 } 993 994 ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF, 995 shareable ? PIRQ_SHAREABLE : 0); 996 if (ret < 0) { 997 __unbind_from_irq(irq); 998 irq = ret; 999 goto out; 1000 } 1001 1002 pirq_query_unmask(irq); 1003 /* We try to use the handler with the appropriate semantic for the 1004 * type of interrupt: if the interrupt is an edge triggered 1005 * interrupt we use handle_edge_irq. 1006 * 1007 * On the other hand if the interrupt is level triggered we use 1008 * handle_fasteoi_irq like the native code does for this kind of 1009 * interrupts. 1010 * 1011 * Depending on the Xen version, pirq_needs_eoi might return true 1012 * not only for level triggered interrupts but for edge triggered 1013 * interrupts too. In any case Xen always honors the eoi mechanism, 1014 * not injecting any more pirqs of the same kind if the first one 1015 * hasn't received an eoi yet. Therefore using the fasteoi handler 1016 * is the right choice either way. 1017 */ 1018 if (shareable) 1019 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, 1020 handle_fasteoi_irq, name); 1021 else 1022 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, 1023 handle_edge_irq, name); 1024 1025out: 1026 mutex_unlock(&irq_mapping_update_lock); 1027 1028 return irq; 1029} 1030 1031#ifdef CONFIG_PCI_MSI 1032int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc) 1033{ 1034 int rc; 1035 struct physdev_get_free_pirq op_get_free_pirq; 1036 1037 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI; 1038 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq); 1039 1040 WARN_ONCE(rc == -ENOSYS, 1041 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n"); 1042 1043 return rc ? -1 : op_get_free_pirq.pirq; 1044} 1045 1046int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc, 1047 int pirq, int nvec, const char *name, domid_t domid) 1048{ 1049 int i, irq, ret; 1050 1051 mutex_lock(&irq_mapping_update_lock); 1052 1053 irq = xen_allocate_irqs_dynamic(nvec); 1054 if (irq < 0) 1055 goto out; 1056 1057 for (i = 0; i < nvec; i++) { 1058 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name); 1059 1060 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid, 1061 i == 0 ? 0 : PIRQ_MSI_GROUP); 1062 if (ret < 0) 1063 goto error_irq; 1064 } 1065 1066 ret = irq_set_msi_desc(irq, msidesc); 1067 if (ret < 0) 1068 goto error_irq; 1069out: 1070 mutex_unlock(&irq_mapping_update_lock); 1071 return irq; 1072error_irq: 1073 while (nvec--) 1074 __unbind_from_irq(irq + nvec); 1075 mutex_unlock(&irq_mapping_update_lock); 1076 return ret; 1077} 1078#endif 1079 1080int xen_destroy_irq(int irq) 1081{ 1082 struct physdev_unmap_pirq unmap_irq; 1083 struct irq_info *info = info_for_irq(irq); 1084 int rc = -ENOENT; 1085 1086 mutex_lock(&irq_mapping_update_lock); 1087 1088 /* 1089 * If trying to remove a vector in a MSI group different 1090 * than the first one skip the PIRQ unmap unless this vector 1091 * is the first one in the group. 1092 */ 1093 if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) { 1094 unmap_irq.pirq = info->u.pirq.pirq; 1095 unmap_irq.domid = info->u.pirq.domid; 1096 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq); 1097 /* If another domain quits without making the pci_disable_msix 1098 * call, the Xen hypervisor takes care of freeing the PIRQs 1099 * (free_domain_pirqs). 1100 */ 1101 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF)) 1102 pr_info("domain %d does not have %d anymore\n", 1103 info->u.pirq.domid, info->u.pirq.pirq); 1104 else if (rc) { 1105 pr_warn("unmap irq failed %d\n", rc); 1106 goto out; 1107 } 1108 } 1109 1110 xen_free_irq(irq); 1111 1112out: 1113 mutex_unlock(&irq_mapping_update_lock); 1114 return rc; 1115} 1116 1117int xen_irq_from_pirq(unsigned pirq) 1118{ 1119 int irq; 1120 1121 struct irq_info *info; 1122 1123 mutex_lock(&irq_mapping_update_lock); 1124 1125 list_for_each_entry(info, &xen_irq_list_head, list) { 1126 if (info->type != IRQT_PIRQ) 1127 continue; 1128 irq = info->irq; 1129 if (info->u.pirq.pirq == pirq) 1130 goto out; 1131 } 1132 irq = -1; 1133out: 1134 mutex_unlock(&irq_mapping_update_lock); 1135 1136 return irq; 1137} 1138 1139 1140int xen_pirq_from_irq(unsigned irq) 1141{ 1142 return pirq_from_irq(irq); 1143} 1144EXPORT_SYMBOL_GPL(xen_pirq_from_irq); 1145 1146static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip) 1147{ 1148 int irq; 1149 int ret; 1150 1151 if (evtchn >= xen_evtchn_max_channels()) 1152 return -ENOMEM; 1153 1154 mutex_lock(&irq_mapping_update_lock); 1155 1156 irq = get_evtchn_to_irq(evtchn); 1157 1158 if (irq == -1) { 1159 irq = xen_allocate_irq_dynamic(); 1160 if (irq < 0) 1161 goto out; 1162 1163 irq_set_chip_and_handler_name(irq, chip, 1164 handle_edge_irq, "event"); 1165 1166 ret = xen_irq_info_evtchn_setup(irq, evtchn); 1167 if (ret < 0) { 1168 __unbind_from_irq(irq); 1169 irq = ret; 1170 goto out; 1171 } 1172 /* New interdomain events are bound to VCPU 0. */ 1173 bind_evtchn_to_cpu(evtchn, 0); 1174 } else { 1175 struct irq_info *info = info_for_irq(irq); 1176 WARN_ON(info == NULL || info->type != IRQT_EVTCHN); 1177 } 1178 1179out: 1180 mutex_unlock(&irq_mapping_update_lock); 1181 1182 return irq; 1183} 1184 1185int bind_evtchn_to_irq(evtchn_port_t evtchn) 1186{ 1187 return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip); 1188} 1189EXPORT_SYMBOL_GPL(bind_evtchn_to_irq); 1190 1191int bind_evtchn_to_irq_lateeoi(evtchn_port_t evtchn) 1192{ 1193 return bind_evtchn_to_irq_chip(evtchn, &xen_lateeoi_chip); 1194} 1195EXPORT_SYMBOL_GPL(bind_evtchn_to_irq_lateeoi); 1196 1197static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu) 1198{ 1199 struct evtchn_bind_ipi bind_ipi; 1200 evtchn_port_t evtchn; 1201 int ret, irq; 1202 1203 mutex_lock(&irq_mapping_update_lock); 1204 1205 irq = per_cpu(ipi_to_irq, cpu)[ipi]; 1206 1207 if (irq == -1) { 1208 irq = xen_allocate_irq_dynamic(); 1209 if (irq < 0) 1210 goto out; 1211 1212 irq_set_chip_and_handler_name(irq, &xen_percpu_chip, 1213 handle_percpu_irq, "ipi"); 1214 1215 bind_ipi.vcpu = xen_vcpu_nr(cpu); 1216 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 1217 &bind_ipi) != 0) 1218 BUG(); 1219 evtchn = bind_ipi.port; 1220 1221 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi); 1222 if (ret < 0) { 1223 __unbind_from_irq(irq); 1224 irq = ret; 1225 goto out; 1226 } 1227 bind_evtchn_to_cpu(evtchn, cpu); 1228 } else { 1229 struct irq_info *info = info_for_irq(irq); 1230 WARN_ON(info == NULL || info->type != IRQT_IPI); 1231 } 1232 1233 out: 1234 mutex_unlock(&irq_mapping_update_lock); 1235 return irq; 1236} 1237 1238static int bind_interdomain_evtchn_to_irq_chip(unsigned int remote_domain, 1239 evtchn_port_t remote_port, 1240 struct irq_chip *chip) 1241{ 1242 struct evtchn_bind_interdomain bind_interdomain; 1243 int err; 1244 1245 bind_interdomain.remote_dom = remote_domain; 1246 bind_interdomain.remote_port = remote_port; 1247 1248 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, 1249 &bind_interdomain); 1250 1251 return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port, 1252 chip); 1253} 1254 1255int bind_interdomain_evtchn_to_irq_lateeoi(unsigned int remote_domain, 1256 evtchn_port_t remote_port) 1257{ 1258 return bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, 1259 &xen_lateeoi_chip); 1260} 1261EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi); 1262 1263static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn) 1264{ 1265 struct evtchn_status status; 1266 evtchn_port_t port; 1267 int rc = -ENOENT; 1268 1269 memset(&status, 0, sizeof(status)); 1270 for (port = 0; port < xen_evtchn_max_channels(); port++) { 1271 status.dom = DOMID_SELF; 1272 status.port = port; 1273 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status); 1274 if (rc < 0) 1275 continue; 1276 if (status.status != EVTCHNSTAT_virq) 1277 continue; 1278 if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) { 1279 *evtchn = port; 1280 break; 1281 } 1282 } 1283 return rc; 1284} 1285 1286/** 1287 * xen_evtchn_nr_channels - number of usable event channel ports 1288 * 1289 * This may be less than the maximum supported by the current 1290 * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum 1291 * supported. 1292 */ 1293unsigned xen_evtchn_nr_channels(void) 1294{ 1295 return evtchn_ops->nr_channels(); 1296} 1297EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels); 1298 1299int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu) 1300{ 1301 struct evtchn_bind_virq bind_virq; 1302 evtchn_port_t evtchn = 0; 1303 int irq, ret; 1304 1305 mutex_lock(&irq_mapping_update_lock); 1306 1307 irq = per_cpu(virq_to_irq, cpu)[virq]; 1308 1309 if (irq == -1) { 1310 irq = xen_allocate_irq_dynamic(); 1311 if (irq < 0) 1312 goto out; 1313 1314 if (percpu) 1315 irq_set_chip_and_handler_name(irq, &xen_percpu_chip, 1316 handle_percpu_irq, "virq"); 1317 else 1318 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip, 1319 handle_edge_irq, "virq"); 1320 1321 bind_virq.virq = virq; 1322 bind_virq.vcpu = xen_vcpu_nr(cpu); 1323 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1324 &bind_virq); 1325 if (ret == 0) 1326 evtchn = bind_virq.port; 1327 else { 1328 if (ret == -EEXIST) 1329 ret = find_virq(virq, cpu, &evtchn); 1330 BUG_ON(ret < 0); 1331 } 1332 1333 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq); 1334 if (ret < 0) { 1335 __unbind_from_irq(irq); 1336 irq = ret; 1337 goto out; 1338 } 1339 1340 bind_evtchn_to_cpu(evtchn, cpu); 1341 } else { 1342 struct irq_info *info = info_for_irq(irq); 1343 WARN_ON(info == NULL || info->type != IRQT_VIRQ); 1344 } 1345 1346out: 1347 mutex_unlock(&irq_mapping_update_lock); 1348 1349 return irq; 1350} 1351 1352static void unbind_from_irq(unsigned int irq) 1353{ 1354 mutex_lock(&irq_mapping_update_lock); 1355 __unbind_from_irq(irq); 1356 mutex_unlock(&irq_mapping_update_lock); 1357} 1358 1359static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn, 1360 irq_handler_t handler, 1361 unsigned long irqflags, 1362 const char *devname, void *dev_id, 1363 struct irq_chip *chip) 1364{ 1365 int irq, retval; 1366 1367 irq = bind_evtchn_to_irq_chip(evtchn, chip); 1368 if (irq < 0) 1369 return irq; 1370 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1371 if (retval != 0) { 1372 unbind_from_irq(irq); 1373 return retval; 1374 } 1375 1376 return irq; 1377} 1378 1379int bind_evtchn_to_irqhandler(evtchn_port_t evtchn, 1380 irq_handler_t handler, 1381 unsigned long irqflags, 1382 const char *devname, void *dev_id) 1383{ 1384 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, 1385 devname, dev_id, 1386 &xen_dynamic_chip); 1387} 1388EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler); 1389 1390int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn, 1391 irq_handler_t handler, 1392 unsigned long irqflags, 1393 const char *devname, void *dev_id) 1394{ 1395 return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags, 1396 devname, dev_id, 1397 &xen_lateeoi_chip); 1398} 1399EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi); 1400 1401static int bind_interdomain_evtchn_to_irqhandler_chip( 1402 unsigned int remote_domain, evtchn_port_t remote_port, 1403 irq_handler_t handler, unsigned long irqflags, 1404 const char *devname, void *dev_id, struct irq_chip *chip) 1405{ 1406 int irq, retval; 1407 1408 irq = bind_interdomain_evtchn_to_irq_chip(remote_domain, remote_port, 1409 chip); 1410 if (irq < 0) 1411 return irq; 1412 1413 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1414 if (retval != 0) { 1415 unbind_from_irq(irq); 1416 return retval; 1417 } 1418 1419 return irq; 1420} 1421 1422int bind_interdomain_evtchn_to_irqhandler_lateeoi(unsigned int remote_domain, 1423 evtchn_port_t remote_port, 1424 irq_handler_t handler, 1425 unsigned long irqflags, 1426 const char *devname, 1427 void *dev_id) 1428{ 1429 return bind_interdomain_evtchn_to_irqhandler_chip(remote_domain, 1430 remote_port, handler, irqflags, devname, 1431 dev_id, &xen_lateeoi_chip); 1432} 1433EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi); 1434 1435int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu, 1436 irq_handler_t handler, 1437 unsigned long irqflags, const char *devname, void *dev_id) 1438{ 1439 int irq, retval; 1440 1441 irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU); 1442 if (irq < 0) 1443 return irq; 1444 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1445 if (retval != 0) { 1446 unbind_from_irq(irq); 1447 return retval; 1448 } 1449 1450 return irq; 1451} 1452EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler); 1453 1454int bind_ipi_to_irqhandler(enum ipi_vector ipi, 1455 unsigned int cpu, 1456 irq_handler_t handler, 1457 unsigned long irqflags, 1458 const char *devname, 1459 void *dev_id) 1460{ 1461 int irq, retval; 1462 1463 irq = bind_ipi_to_irq(ipi, cpu); 1464 if (irq < 0) 1465 return irq; 1466 1467 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME; 1468 retval = request_irq(irq, handler, irqflags, devname, dev_id); 1469 if (retval != 0) { 1470 unbind_from_irq(irq); 1471 return retval; 1472 } 1473 1474 return irq; 1475} 1476 1477void unbind_from_irqhandler(unsigned int irq, void *dev_id) 1478{ 1479 struct irq_info *info = info_for_irq(irq); 1480 1481 if (WARN_ON(!info)) 1482 return; 1483 free_irq(irq, dev_id); 1484 unbind_from_irq(irq); 1485} 1486EXPORT_SYMBOL_GPL(unbind_from_irqhandler); 1487 1488/** 1489 * xen_set_irq_priority() - set an event channel priority. 1490 * @irq:irq bound to an event channel. 1491 * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN. 1492 */ 1493int xen_set_irq_priority(unsigned irq, unsigned priority) 1494{ 1495 struct evtchn_set_priority set_priority; 1496 1497 set_priority.port = evtchn_from_irq(irq); 1498 set_priority.priority = priority; 1499 1500 return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority, 1501 &set_priority); 1502} 1503EXPORT_SYMBOL_GPL(xen_set_irq_priority); 1504 1505int evtchn_make_refcounted(evtchn_port_t evtchn) 1506{ 1507 int irq = get_evtchn_to_irq(evtchn); 1508 struct irq_info *info; 1509 1510 if (irq == -1) 1511 return -ENOENT; 1512 1513 info = info_for_irq(irq); 1514 1515 if (!info) 1516 return -ENOENT; 1517 1518 WARN_ON(info->refcnt != -1); 1519 1520 info->refcnt = 1; 1521 1522 return 0; 1523} 1524EXPORT_SYMBOL_GPL(evtchn_make_refcounted); 1525 1526int evtchn_get(evtchn_port_t evtchn) 1527{ 1528 int irq; 1529 struct irq_info *info; 1530 int err = -ENOENT; 1531 1532 if (evtchn >= xen_evtchn_max_channels()) 1533 return -EINVAL; 1534 1535 mutex_lock(&irq_mapping_update_lock); 1536 1537 irq = get_evtchn_to_irq(evtchn); 1538 if (irq == -1) 1539 goto done; 1540 1541 info = info_for_irq(irq); 1542 1543 if (!info) 1544 goto done; 1545 1546 err = -EINVAL; 1547 if (info->refcnt <= 0 || info->refcnt == SHRT_MAX) 1548 goto done; 1549 1550 info->refcnt++; 1551 err = 0; 1552 done: 1553 mutex_unlock(&irq_mapping_update_lock); 1554 1555 return err; 1556} 1557EXPORT_SYMBOL_GPL(evtchn_get); 1558 1559void evtchn_put(evtchn_port_t evtchn) 1560{ 1561 int irq = get_evtchn_to_irq(evtchn); 1562 if (WARN_ON(irq == -1)) 1563 return; 1564 unbind_from_irq(irq); 1565} 1566EXPORT_SYMBOL_GPL(evtchn_put); 1567 1568void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector) 1569{ 1570 int irq; 1571 1572#ifdef CONFIG_X86 1573 if (unlikely(vector == XEN_NMI_VECTOR)) { 1574 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu), 1575 NULL); 1576 if (rc < 0) 1577 printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc); 1578 return; 1579 } 1580#endif 1581 irq = per_cpu(ipi_to_irq, cpu)[vector]; 1582 BUG_ON(irq < 0); 1583 notify_remote_via_irq(irq); 1584} 1585 1586struct evtchn_loop_ctrl { 1587 ktime_t timeout; 1588 unsigned count; 1589 bool defer_eoi; 1590}; 1591 1592void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl) 1593{ 1594 int irq; 1595 struct irq_info *info; 1596 1597 irq = get_evtchn_to_irq(port); 1598 if (irq == -1) 1599 return; 1600 1601 /* 1602 * Check for timeout every 256 events. 1603 * We are setting the timeout value only after the first 256 1604 * events in order to not hurt the common case of few loop 1605 * iterations. The 256 is basically an arbitrary value. 1606 * 1607 * In case we are hitting the timeout we need to defer all further 1608 * EOIs in order to ensure to leave the event handling loop rather 1609 * sooner than later. 1610 */ 1611 if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) { 1612 ktime_t kt = ktime_get(); 1613 1614 if (!ctrl->timeout) { 1615 kt = ktime_add_ms(kt, 1616 jiffies_to_msecs(event_loop_timeout)); 1617 ctrl->timeout = kt; 1618 } else if (kt > ctrl->timeout) { 1619 ctrl->defer_eoi = true; 1620 } 1621 } 1622 1623 info = info_for_irq(irq); 1624 if (xchg_acquire(&info->is_active, 1)) 1625 return; 1626 1627 if (ctrl->defer_eoi) { 1628 info->eoi_cpu = smp_processor_id(); 1629 info->irq_epoch = __this_cpu_read(irq_epoch); 1630 info->eoi_time = get_jiffies_64() + event_eoi_delay; 1631 } 1632 1633 generic_handle_irq(irq); 1634} 1635 1636static void __xen_evtchn_do_upcall(void) 1637{ 1638 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu); 1639 int cpu = smp_processor_id(); 1640 struct evtchn_loop_ctrl ctrl = { 0 }; 1641 1642 /* 1643 * When closing an event channel the associated IRQ must not be freed 1644 * until all cpus have left the event handling loop. This is ensured 1645 * by taking the rcu_read_lock() while handling events, as freeing of 1646 * the IRQ is handled via queue_rcu_work() _after_ closing the event 1647 * channel. 1648 */ 1649 rcu_read_lock(); 1650 1651 do { 1652 vcpu_info->evtchn_upcall_pending = 0; 1653 1654 xen_evtchn_handle_events(cpu, &ctrl); 1655 1656 BUG_ON(!irqs_disabled()); 1657 1658 virt_rmb(); /* Hypervisor can set upcall pending. */ 1659 1660 } while (vcpu_info->evtchn_upcall_pending); 1661 1662 rcu_read_unlock(); 1663 1664 /* 1665 * Increment irq_epoch only now to defer EOIs only for 1666 * xen_irq_lateeoi() invocations occurring from inside the loop 1667 * above. 1668 */ 1669 __this_cpu_inc(irq_epoch); 1670} 1671 1672void xen_evtchn_do_upcall(struct pt_regs *regs) 1673{ 1674 struct pt_regs *old_regs = set_irq_regs(regs); 1675 1676 irq_enter(); 1677 1678 __xen_evtchn_do_upcall(); 1679 1680 irq_exit(); 1681 set_irq_regs(old_regs); 1682} 1683 1684void xen_hvm_evtchn_do_upcall(void) 1685{ 1686 __xen_evtchn_do_upcall(); 1687} 1688EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall); 1689 1690/* Rebind a new event channel to an existing irq. */ 1691void rebind_evtchn_irq(evtchn_port_t evtchn, int irq) 1692{ 1693 struct irq_info *info = info_for_irq(irq); 1694 1695 if (WARN_ON(!info)) 1696 return; 1697 1698 /* Make sure the irq is masked, since the new event channel 1699 will also be masked. */ 1700 disable_irq(irq); 1701 1702 mutex_lock(&irq_mapping_update_lock); 1703 1704 /* After resume the irq<->evtchn mappings are all cleared out */ 1705 BUG_ON(get_evtchn_to_irq(evtchn) != -1); 1706 /* Expect irq to have been bound before, 1707 so there should be a proper type */ 1708 BUG_ON(info->type == IRQT_UNBOUND); 1709 1710 (void)xen_irq_info_evtchn_setup(irq, evtchn); 1711 1712 mutex_unlock(&irq_mapping_update_lock); 1713 1714 bind_evtchn_to_cpu(evtchn, info->cpu); 1715 /* This will be deferred until interrupt is processed */ 1716 irq_set_affinity(irq, cpumask_of(info->cpu)); 1717 1718 /* Unmask the event channel. */ 1719 enable_irq(irq); 1720} 1721 1722/* Rebind an evtchn so that it gets delivered to a specific cpu */ 1723static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu) 1724{ 1725 struct evtchn_bind_vcpu bind_vcpu; 1726 evtchn_port_t evtchn = info ? info->evtchn : 0; 1727 1728 if (!VALID_EVTCHN(evtchn)) 1729 return -1; 1730 1731 if (!xen_support_evtchn_rebind()) 1732 return -1; 1733 1734 /* Send future instances of this interrupt to other vcpu. */ 1735 bind_vcpu.port = evtchn; 1736 bind_vcpu.vcpu = xen_vcpu_nr(tcpu); 1737 1738 /* 1739 * Mask the event while changing the VCPU binding to prevent 1740 * it being delivered on an unexpected VCPU. 1741 */ 1742 do_mask(info, EVT_MASK_REASON_TEMPORARY); 1743 1744 /* 1745 * If this fails, it usually just indicates that we're dealing with a 1746 * virq or IPI channel, which don't actually need to be rebound. Ignore 1747 * it, but don't do the xenlinux-level rebind in that case. 1748 */ 1749 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0) 1750 bind_evtchn_to_cpu(evtchn, tcpu); 1751 1752 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 1753 1754 return 0; 1755} 1756 1757static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest, 1758 bool force) 1759{ 1760 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask); 1761 int ret = xen_rebind_evtchn_to_cpu(info_for_irq(data->irq), tcpu); 1762 1763 if (!ret) 1764 irq_data_update_effective_affinity(data, cpumask_of(tcpu)); 1765 1766 return ret; 1767} 1768 1769/* To be called with desc->lock held. */ 1770int xen_set_affinity_evtchn(struct irq_desc *desc, unsigned int tcpu) 1771{ 1772 struct irq_data *d = irq_desc_get_irq_data(desc); 1773 1774 return set_affinity_irq(d, cpumask_of(tcpu), false); 1775} 1776EXPORT_SYMBOL_GPL(xen_set_affinity_evtchn); 1777 1778static void enable_dynirq(struct irq_data *data) 1779{ 1780 struct irq_info *info = info_for_irq(data->irq); 1781 evtchn_port_t evtchn = info ? info->evtchn : 0; 1782 1783 if (VALID_EVTCHN(evtchn)) 1784 do_unmask(info, EVT_MASK_REASON_EXPLICIT); 1785} 1786 1787static void disable_dynirq(struct irq_data *data) 1788{ 1789 struct irq_info *info = info_for_irq(data->irq); 1790 evtchn_port_t evtchn = info ? info->evtchn : 0; 1791 1792 if (VALID_EVTCHN(evtchn)) 1793 do_mask(info, EVT_MASK_REASON_EXPLICIT); 1794} 1795 1796static void ack_dynirq(struct irq_data *data) 1797{ 1798 struct irq_info *info = info_for_irq(data->irq); 1799 evtchn_port_t evtchn = info ? info->evtchn : 0; 1800 1801 if (!VALID_EVTCHN(evtchn)) 1802 return; 1803 1804 if (unlikely(irqd_is_setaffinity_pending(data)) && 1805 likely(!irqd_irq_disabled(data))) { 1806 do_mask(info, EVT_MASK_REASON_TEMPORARY); 1807 1808 event_handler_exit(info); 1809 1810 irq_move_masked_irq(data); 1811 1812 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 1813 } else 1814 event_handler_exit(info); 1815} 1816 1817static void mask_ack_dynirq(struct irq_data *data) 1818{ 1819 disable_dynirq(data); 1820 ack_dynirq(data); 1821} 1822 1823static void lateeoi_ack_dynirq(struct irq_data *data) 1824{ 1825 struct irq_info *info = info_for_irq(data->irq); 1826 evtchn_port_t evtchn = info ? info->evtchn : 0; 1827 1828 if (!VALID_EVTCHN(evtchn)) 1829 return; 1830 1831 do_mask(info, EVT_MASK_REASON_EOI_PENDING); 1832 1833 if (unlikely(irqd_is_setaffinity_pending(data)) && 1834 likely(!irqd_irq_disabled(data))) { 1835 do_mask(info, EVT_MASK_REASON_TEMPORARY); 1836 1837 clear_evtchn(evtchn); 1838 1839 irq_move_masked_irq(data); 1840 1841 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 1842 } else 1843 clear_evtchn(evtchn); 1844} 1845 1846static void lateeoi_mask_ack_dynirq(struct irq_data *data) 1847{ 1848 struct irq_info *info = info_for_irq(data->irq); 1849 evtchn_port_t evtchn = info ? info->evtchn : 0; 1850 1851 if (VALID_EVTCHN(evtchn)) { 1852 do_mask(info, EVT_MASK_REASON_EXPLICIT); 1853 ack_dynirq(data); 1854 } 1855} 1856 1857static int retrigger_dynirq(struct irq_data *data) 1858{ 1859 struct irq_info *info = info_for_irq(data->irq); 1860 evtchn_port_t evtchn = info ? info->evtchn : 0; 1861 1862 if (!VALID_EVTCHN(evtchn)) 1863 return 0; 1864 1865 do_mask(info, EVT_MASK_REASON_TEMPORARY); 1866 set_evtchn(evtchn); 1867 do_unmask(info, EVT_MASK_REASON_TEMPORARY); 1868 1869 return 1; 1870} 1871 1872static void restore_pirqs(void) 1873{ 1874 int pirq, rc, irq, gsi; 1875 struct physdev_map_pirq map_irq; 1876 struct irq_info *info; 1877 1878 list_for_each_entry(info, &xen_irq_list_head, list) { 1879 if (info->type != IRQT_PIRQ) 1880 continue; 1881 1882 pirq = info->u.pirq.pirq; 1883 gsi = info->u.pirq.gsi; 1884 irq = info->irq; 1885 1886 /* save/restore of PT devices doesn't work, so at this point the 1887 * only devices present are GSI based emulated devices */ 1888 if (!gsi) 1889 continue; 1890 1891 map_irq.domid = DOMID_SELF; 1892 map_irq.type = MAP_PIRQ_TYPE_GSI; 1893 map_irq.index = gsi; 1894 map_irq.pirq = pirq; 1895 1896 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq); 1897 if (rc) { 1898 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n", 1899 gsi, irq, pirq, rc); 1900 xen_free_irq(irq); 1901 continue; 1902 } 1903 1904 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq); 1905 1906 __startup_pirq(irq); 1907 } 1908} 1909 1910static void restore_cpu_virqs(unsigned int cpu) 1911{ 1912 struct evtchn_bind_virq bind_virq; 1913 evtchn_port_t evtchn; 1914 int virq, irq; 1915 1916 for (virq = 0; virq < NR_VIRQS; virq++) { 1917 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) 1918 continue; 1919 1920 BUG_ON(virq_from_irq(irq) != virq); 1921 1922 /* Get a new binding from Xen. */ 1923 bind_virq.virq = virq; 1924 bind_virq.vcpu = xen_vcpu_nr(cpu); 1925 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, 1926 &bind_virq) != 0) 1927 BUG(); 1928 evtchn = bind_virq.port; 1929 1930 /* Record the new mapping. */ 1931 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq); 1932 bind_evtchn_to_cpu(evtchn, cpu); 1933 } 1934} 1935 1936static void restore_cpu_ipis(unsigned int cpu) 1937{ 1938 struct evtchn_bind_ipi bind_ipi; 1939 evtchn_port_t evtchn; 1940 int ipi, irq; 1941 1942 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) { 1943 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) 1944 continue; 1945 1946 BUG_ON(ipi_from_irq(irq) != ipi); 1947 1948 /* Get a new binding from Xen. */ 1949 bind_ipi.vcpu = xen_vcpu_nr(cpu); 1950 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, 1951 &bind_ipi) != 0) 1952 BUG(); 1953 evtchn = bind_ipi.port; 1954 1955 /* Record the new mapping. */ 1956 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi); 1957 bind_evtchn_to_cpu(evtchn, cpu); 1958 } 1959} 1960 1961/* Clear an irq's pending state, in preparation for polling on it */ 1962void xen_clear_irq_pending(int irq) 1963{ 1964 struct irq_info *info = info_for_irq(irq); 1965 evtchn_port_t evtchn = info ? info->evtchn : 0; 1966 1967 if (VALID_EVTCHN(evtchn)) 1968 event_handler_exit(info); 1969} 1970EXPORT_SYMBOL(xen_clear_irq_pending); 1971void xen_set_irq_pending(int irq) 1972{ 1973 evtchn_port_t evtchn = evtchn_from_irq(irq); 1974 1975 if (VALID_EVTCHN(evtchn)) 1976 set_evtchn(evtchn); 1977} 1978 1979bool xen_test_irq_pending(int irq) 1980{ 1981 evtchn_port_t evtchn = evtchn_from_irq(irq); 1982 bool ret = false; 1983 1984 if (VALID_EVTCHN(evtchn)) 1985 ret = test_evtchn(evtchn); 1986 1987 return ret; 1988} 1989 1990/* Poll waiting for an irq to become pending with timeout. In the usual case, 1991 * the irq will be disabled so it won't deliver an interrupt. */ 1992void xen_poll_irq_timeout(int irq, u64 timeout) 1993{ 1994 evtchn_port_t evtchn = evtchn_from_irq(irq); 1995 1996 if (VALID_EVTCHN(evtchn)) { 1997 struct sched_poll poll; 1998 1999 poll.nr_ports = 1; 2000 poll.timeout = timeout; 2001 set_xen_guest_handle(poll.ports, &evtchn); 2002 2003 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0) 2004 BUG(); 2005 } 2006} 2007EXPORT_SYMBOL(xen_poll_irq_timeout); 2008/* Poll waiting for an irq to become pending. In the usual case, the 2009 * irq will be disabled so it won't deliver an interrupt. */ 2010void xen_poll_irq(int irq) 2011{ 2012 xen_poll_irq_timeout(irq, 0 /* no timeout */); 2013} 2014 2015/* Check whether the IRQ line is shared with other guests. */ 2016int xen_test_irq_shared(int irq) 2017{ 2018 struct irq_info *info = info_for_irq(irq); 2019 struct physdev_irq_status_query irq_status; 2020 2021 if (WARN_ON(!info)) 2022 return -ENOENT; 2023 2024 irq_status.irq = info->u.pirq.pirq; 2025 2026 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status)) 2027 return 0; 2028 return !(irq_status.flags & XENIRQSTAT_shared); 2029} 2030EXPORT_SYMBOL_GPL(xen_test_irq_shared); 2031 2032void xen_irq_resume(void) 2033{ 2034 unsigned int cpu; 2035 struct irq_info *info; 2036 2037 /* New event-channel space is not 'live' yet. */ 2038 xen_evtchn_resume(); 2039 2040 /* No IRQ <-> event-channel mappings. */ 2041 list_for_each_entry(info, &xen_irq_list_head, list) 2042 info->evtchn = 0; /* zap event-channel binding */ 2043 2044 clear_evtchn_to_irq_all(); 2045 2046 for_each_possible_cpu(cpu) { 2047 restore_cpu_virqs(cpu); 2048 restore_cpu_ipis(cpu); 2049 } 2050 2051 restore_pirqs(); 2052} 2053 2054static struct irq_chip xen_dynamic_chip __read_mostly = { 2055 .name = "xen-dyn", 2056 2057 .irq_disable = disable_dynirq, 2058 .irq_mask = disable_dynirq, 2059 .irq_unmask = enable_dynirq, 2060 2061 .irq_ack = ack_dynirq, 2062 .irq_mask_ack = mask_ack_dynirq, 2063 2064 .irq_set_affinity = set_affinity_irq, 2065 .irq_retrigger = retrigger_dynirq, 2066}; 2067 2068static struct irq_chip xen_lateeoi_chip __read_mostly = { 2069 /* The chip name needs to contain "xen-dyn" for irqbalance to work. */ 2070 .name = "xen-dyn-lateeoi", 2071 2072 .irq_disable = disable_dynirq, 2073 .irq_mask = disable_dynirq, 2074 .irq_unmask = enable_dynirq, 2075 2076 .irq_ack = lateeoi_ack_dynirq, 2077 .irq_mask_ack = lateeoi_mask_ack_dynirq, 2078 2079 .irq_set_affinity = set_affinity_irq, 2080 .irq_retrigger = retrigger_dynirq, 2081}; 2082 2083static struct irq_chip xen_pirq_chip __read_mostly = { 2084 .name = "xen-pirq", 2085 2086 .irq_startup = startup_pirq, 2087 .irq_shutdown = shutdown_pirq, 2088 .irq_enable = enable_pirq, 2089 .irq_disable = disable_pirq, 2090 2091 .irq_mask = disable_dynirq, 2092 .irq_unmask = enable_dynirq, 2093 2094 .irq_ack = eoi_pirq, 2095 .irq_eoi = eoi_pirq, 2096 .irq_mask_ack = mask_ack_pirq, 2097 2098 .irq_set_affinity = set_affinity_irq, 2099 2100 .irq_retrigger = retrigger_dynirq, 2101}; 2102 2103static struct irq_chip xen_percpu_chip __read_mostly = { 2104 .name = "xen-percpu", 2105 2106 .irq_disable = disable_dynirq, 2107 .irq_mask = disable_dynirq, 2108 .irq_unmask = enable_dynirq, 2109 2110 .irq_ack = ack_dynirq, 2111}; 2112 2113#ifdef CONFIG_XEN_PVHVM 2114/* Vector callbacks are better than PCI interrupts to receive event 2115 * channel notifications because we can receive vector callbacks on any 2116 * vcpu and we don't need PCI support or APIC interactions. */ 2117void xen_setup_callback_vector(void) 2118{ 2119 uint64_t callback_via; 2120 2121 if (xen_have_vector_callback) { 2122 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR); 2123 if (xen_set_callback_via(callback_via)) { 2124 pr_err("Request for Xen HVM callback vector failed\n"); 2125 xen_have_vector_callback = 0; 2126 } 2127 } 2128} 2129 2130static __init void xen_alloc_callback_vector(void) 2131{ 2132 if (!xen_have_vector_callback) 2133 return; 2134 2135 pr_info("Xen HVM callback vector for event delivery is enabled\n"); 2136 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); 2137} 2138#else 2139void xen_setup_callback_vector(void) {} 2140static inline void xen_alloc_callback_vector(void) {} 2141#endif 2142 2143bool xen_fifo_events = true; 2144module_param_named(fifo_events, xen_fifo_events, bool, 0); 2145 2146static int xen_evtchn_cpu_prepare(unsigned int cpu) 2147{ 2148 int ret = 0; 2149 2150 xen_cpu_init_eoi(cpu); 2151 2152 if (evtchn_ops->percpu_init) 2153 ret = evtchn_ops->percpu_init(cpu); 2154 2155 return ret; 2156} 2157 2158static int xen_evtchn_cpu_dead(unsigned int cpu) 2159{ 2160 int ret = 0; 2161 2162 if (evtchn_ops->percpu_deinit) 2163 ret = evtchn_ops->percpu_deinit(cpu); 2164 2165 return ret; 2166} 2167 2168void __init xen_init_IRQ(void) 2169{ 2170 int ret = -EINVAL; 2171 evtchn_port_t evtchn; 2172 2173 if (xen_fifo_events) 2174 ret = xen_evtchn_fifo_init(); 2175 if (ret < 0) { 2176 xen_evtchn_2l_init(); 2177 xen_fifo_events = false; 2178 } 2179 2180 xen_cpu_init_eoi(smp_processor_id()); 2181 2182 cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE, 2183 "xen/evtchn:prepare", 2184 xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead); 2185 2186 evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()), 2187 sizeof(*evtchn_to_irq), GFP_KERNEL); 2188 BUG_ON(!evtchn_to_irq); 2189 2190 /* No event channels are 'live' right now. */ 2191 for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++) 2192 mask_evtchn(evtchn); 2193 2194 pirq_needs_eoi = pirq_needs_eoi_flag; 2195 2196#ifdef CONFIG_X86 2197 if (xen_pv_domain()) { 2198 if (xen_initial_domain()) 2199 pci_xen_initial_domain(); 2200 } 2201 if (xen_feature(XENFEAT_hvm_callback_vector)) { 2202 xen_setup_callback_vector(); 2203 xen_alloc_callback_vector(); 2204 } 2205 2206 if (xen_hvm_domain()) { 2207 native_init_IRQ(); 2208 /* pci_xen_hvm_init must be called after native_init_IRQ so that 2209 * __acpi_register_gsi can point at the right function */ 2210 pci_xen_hvm_init(); 2211 } else { 2212 int rc; 2213 struct physdev_pirq_eoi_gmfn eoi_gmfn; 2214 2215 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO); 2216 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map); 2217 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn); 2218 if (rc != 0) { 2219 free_page((unsigned long) pirq_eoi_map); 2220 pirq_eoi_map = NULL; 2221 } else 2222 pirq_needs_eoi = pirq_check_eoi_map; 2223 } 2224#endif 2225} 2226