1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
4 *     Author: Alex Williamson <alex.williamson@redhat.com>
5 *
6 * Derived from original vfio:
7 * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
8 * Author: Tom Lyon, pugs@cisco.com
9 */
10
11#include <linux/mutex.h>
12#include <linux/pci.h>
13#include <linux/irqbypass.h>
14#include <linux/types.h>
15#include <linux/uuid.h>
16#include <linux/notifier.h>
17
18#ifndef VFIO_PCI_PRIVATE_H
19#define VFIO_PCI_PRIVATE_H
20
21#define VFIO_PCI_OFFSET_SHIFT   40
22
23#define VFIO_PCI_OFFSET_TO_INDEX(off)	(off >> VFIO_PCI_OFFSET_SHIFT)
24#define VFIO_PCI_INDEX_TO_OFFSET(index)	((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
25#define VFIO_PCI_OFFSET_MASK	(((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
26
27/* Special capability IDs predefined access */
28#define PCI_CAP_ID_INVALID		0xFF	/* default raw access */
29#define PCI_CAP_ID_INVALID_VIRT		0xFE	/* default virt access */
30
31/* Cap maximum number of ioeventfds per device (arbitrary) */
32#define VFIO_PCI_IOEVENTFD_MAX		1000
33
34struct vfio_pci_ioeventfd {
35	struct list_head	next;
36	struct vfio_pci_device	*vdev;
37	struct virqfd		*virqfd;
38	void __iomem		*addr;
39	uint64_t		data;
40	loff_t			pos;
41	int			bar;
42	int			count;
43	bool			test_mem;
44};
45
46struct vfio_pci_irq_ctx {
47	struct eventfd_ctx	*trigger;
48	struct virqfd		*unmask;
49	struct virqfd		*mask;
50	char			*name;
51	bool			masked;
52	struct irq_bypass_producer	producer;
53};
54
55struct vfio_pci_device;
56struct vfio_pci_region;
57
58struct vfio_pci_regops {
59	size_t	(*rw)(struct vfio_pci_device *vdev, char __user *buf,
60		      size_t count, loff_t *ppos, bool iswrite);
61	void	(*release)(struct vfio_pci_device *vdev,
62			   struct vfio_pci_region *region);
63	int	(*mmap)(struct vfio_pci_device *vdev,
64			struct vfio_pci_region *region,
65			struct vm_area_struct *vma);
66	int	(*add_capability)(struct vfio_pci_device *vdev,
67				  struct vfio_pci_region *region,
68				  struct vfio_info_cap *caps);
69};
70
71struct vfio_pci_region {
72	u32				type;
73	u32				subtype;
74	const struct vfio_pci_regops	*ops;
75	void				*data;
76	size_t				size;
77	u32				flags;
78};
79
80struct vfio_pci_dummy_resource {
81	struct resource		resource;
82	int			index;
83	struct list_head	res_next;
84};
85
86struct vfio_pci_reflck {
87	struct kref		kref;
88	struct mutex		lock;
89};
90
91struct vfio_pci_vf_token {
92	struct mutex		lock;
93	uuid_t			uuid;
94	int			users;
95};
96
97struct vfio_pci_mmap_vma {
98	struct vm_area_struct	*vma;
99	struct list_head	vma_next;
100};
101
102struct vfio_pci_device {
103	struct pci_dev		*pdev;
104	void __iomem		*barmap[PCI_STD_NUM_BARS];
105	bool			bar_mmap_supported[PCI_STD_NUM_BARS];
106	u8			*pci_config_map;
107	u8			*vconfig;
108	struct perm_bits	*msi_perm;
109	spinlock_t		irqlock;
110	struct mutex		igate;
111	struct vfio_pci_irq_ctx	*ctx;
112	int			num_ctx;
113	int			irq_type;
114	int			num_regions;
115	struct vfio_pci_region	*region;
116	u8			msi_qmax;
117	u8			msix_bar;
118	u16			msix_size;
119	u32			msix_offset;
120	u32			rbar[7];
121	bool			pci_2_3;
122	bool			virq_disabled;
123	bool			reset_works;
124	bool			extended_caps;
125	bool			bardirty;
126	bool			has_vga;
127	bool			needs_reset;
128	bool			nointx;
129	bool			needs_pm_restore;
130	struct pci_saved_state	*pci_saved_state;
131	struct pci_saved_state	*pm_save;
132	struct vfio_pci_reflck	*reflck;
133	int			refcnt;
134	int			ioeventfds_nr;
135	struct eventfd_ctx	*err_trigger;
136	struct eventfd_ctx	*req_trigger;
137	struct list_head	dummy_resources_list;
138	struct mutex		ioeventfds_lock;
139	struct list_head	ioeventfds_list;
140	struct vfio_pci_vf_token	*vf_token;
141	struct notifier_block	nb;
142	struct mutex		vma_lock;
143	struct list_head	vma_list;
144	struct rw_semaphore	memory_lock;
145};
146
147#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX)
148#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX)
149#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX)
150#define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev)))
151#define irq_is(vdev, type) (vdev->irq_type == type)
152
153extern void vfio_pci_intx_mask(struct vfio_pci_device *vdev);
154extern void vfio_pci_intx_unmask(struct vfio_pci_device *vdev);
155
156extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev,
157				   uint32_t flags, unsigned index,
158				   unsigned start, unsigned count, void *data);
159
160extern ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev,
161				  char __user *buf, size_t count,
162				  loff_t *ppos, bool iswrite);
163
164extern ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
165			       size_t count, loff_t *ppos, bool iswrite);
166
167extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf,
168			       size_t count, loff_t *ppos, bool iswrite);
169
170extern long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
171			       uint64_t data, int count, int fd);
172
173extern int vfio_pci_init_perm_bits(void);
174extern void vfio_pci_uninit_perm_bits(void);
175
176extern int vfio_config_init(struct vfio_pci_device *vdev);
177extern void vfio_config_free(struct vfio_pci_device *vdev);
178
179extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
180					unsigned int type, unsigned int subtype,
181					const struct vfio_pci_regops *ops,
182					size_t size, u32 flags, void *data);
183
184extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev,
185				    pci_power_t state);
186
187extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev);
188extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device
189						    *vdev);
190extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev);
191extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev,
192					       u16 cmd);
193
194#ifdef CONFIG_VFIO_PCI_IGD
195extern int vfio_pci_igd_init(struct vfio_pci_device *vdev);
196#else
197static inline int vfio_pci_igd_init(struct vfio_pci_device *vdev)
198{
199	return -ENODEV;
200}
201#endif
202#ifdef CONFIG_VFIO_PCI_NVLINK2
203extern int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev);
204extern int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev);
205#else
206static inline int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev)
207{
208	return -ENODEV;
209}
210
211static inline int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev)
212{
213	return -ENODEV;
214}
215#endif
216
217#ifdef CONFIG_VFIO_PCI_ZDEV
218extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
219				       struct vfio_info_cap *caps);
220#else
221static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
222					      struct vfio_info_cap *caps)
223{
224	return -ENODEV;
225}
226#endif
227
228#endif /* VFIO_PCI_PRIVATE_H */
229