1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2/* Copyright(c) 2014 - 2020 Intel Corporation */ 3#include <linux/kernel.h> 4#include <linux/init.h> 5#include <linux/types.h> 6#include <linux/pci.h> 7#include <linux/slab.h> 8#include <linux/errno.h> 9#include <linux/interrupt.h> 10#include "adf_accel_devices.h" 11#include "adf_common_drv.h" 12#include "adf_cfg.h" 13#include "adf_cfg_strings.h" 14#include "adf_cfg_common.h" 15#include "adf_transport_access_macros.h" 16#include "adf_transport_internal.h" 17 18#define ADF_MAX_NUM_VFS 32 19 20static int adf_enable_msix(struct adf_accel_dev *accel_dev) 21{ 22 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 23 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 24 u32 msix_num_entries = 1; 25 26 /* If SR-IOV is disabled, add entries for each bank */ 27 if (!accel_dev->pf.vf_info) { 28 int i; 29 30 msix_num_entries += hw_data->num_banks; 31 for (i = 0; i < msix_num_entries; i++) 32 pci_dev_info->msix_entries.entries[i].entry = i; 33 } else { 34 pci_dev_info->msix_entries.entries[0].entry = 35 hw_data->num_banks; 36 } 37 38 if (pci_enable_msix_exact(pci_dev_info->pci_dev, 39 pci_dev_info->msix_entries.entries, 40 msix_num_entries)) { 41 dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n"); 42 return -EFAULT; 43 } 44 return 0; 45} 46 47static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) 48{ 49 pci_disable_msix(pci_dev_info->pci_dev); 50} 51 52static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) 53{ 54 struct adf_etr_bank_data *bank = bank_ptr; 55 56 WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); 57 tasklet_hi_schedule(&bank->resp_handler); 58 return IRQ_HANDLED; 59} 60 61static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) 62{ 63 struct adf_accel_dev *accel_dev = dev_ptr; 64 65#ifdef CONFIG_PCI_IOV 66 /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ 67 if (accel_dev->pf.vf_info) { 68 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 69 struct adf_bar *pmisc = 70 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; 71 void __iomem *pmisc_bar_addr = pmisc->virt_addr; 72 unsigned long vf_mask; 73 74 /* Get the interrupt sources triggered by VFs */ 75 vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU5) & 76 0x0000FFFF) << 16) | 77 ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU3) & 78 0x01FFFE00) >> 9); 79 80 if (vf_mask) { 81 struct adf_accel_vf_info *vf_info; 82 bool irq_handled = false; 83 int i; 84 85 /* Disable VF2PF interrupts for VFs with pending ints */ 86 adf_disable_vf2pf_interrupts(accel_dev, vf_mask); 87 88 /* 89 * Schedule tasklets to handle VF2PF interrupt BHs 90 * unless the VF is malicious and is attempting to 91 * flood the host OS with VF2PF interrupts. 92 */ 93 for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) { 94 vf_info = accel_dev->pf.vf_info + i; 95 96 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { 97 dev_info(&GET_DEV(accel_dev), 98 "Too many ints from VF%d\n", 99 vf_info->vf_nr + 1); 100 continue; 101 } 102 103 /* Tasklet will re-enable ints from this VF */ 104 tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet); 105 irq_handled = true; 106 } 107 108 if (irq_handled) 109 return IRQ_HANDLED; 110 } 111 } 112#endif /* CONFIG_PCI_IOV */ 113 114 dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", 115 accel_dev->accel_id); 116 117 return IRQ_NONE; 118} 119 120static int adf_request_irqs(struct adf_accel_dev *accel_dev) 121{ 122 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 123 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 124 struct msix_entry *msixe = pci_dev_info->msix_entries.entries; 125 struct adf_etr_data *etr_data = accel_dev->transport; 126 int ret, i = 0; 127 char *name; 128 129 /* Request msix irq for all banks unless SR-IOV enabled */ 130 if (!accel_dev->pf.vf_info) { 131 for (i = 0; i < hw_data->num_banks; i++) { 132 struct adf_etr_bank_data *bank = &etr_data->banks[i]; 133 unsigned int cpu, cpus = num_online_cpus(); 134 135 name = *(pci_dev_info->msix_entries.names + i); 136 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, 137 "qat%d-bundle%d", accel_dev->accel_id, i); 138 ret = request_irq(msixe[i].vector, 139 adf_msix_isr_bundle, 0, name, bank); 140 if (ret) { 141 dev_err(&GET_DEV(accel_dev), 142 "failed to enable irq %d for %s\n", 143 msixe[i].vector, name); 144 return ret; 145 } 146 147 cpu = ((accel_dev->accel_id * hw_data->num_banks) + 148 i) % cpus; 149 irq_set_affinity_hint(msixe[i].vector, 150 get_cpu_mask(cpu)); 151 } 152 } 153 154 /* Request msix irq for AE */ 155 name = *(pci_dev_info->msix_entries.names + i); 156 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, 157 "qat%d-ae-cluster", accel_dev->accel_id); 158 ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev); 159 if (ret) { 160 dev_err(&GET_DEV(accel_dev), 161 "failed to enable irq %d, for %s\n", 162 msixe[i].vector, name); 163 return ret; 164 } 165 return ret; 166} 167 168static void adf_free_irqs(struct adf_accel_dev *accel_dev) 169{ 170 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 171 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 172 struct msix_entry *msixe = pci_dev_info->msix_entries.entries; 173 struct adf_etr_data *etr_data = accel_dev->transport; 174 int i = 0; 175 176 if (pci_dev_info->msix_entries.num_entries > 1) { 177 for (i = 0; i < hw_data->num_banks; i++) { 178 irq_set_affinity_hint(msixe[i].vector, NULL); 179 free_irq(msixe[i].vector, &etr_data->banks[i]); 180 } 181 } 182 irq_set_affinity_hint(msixe[i].vector, NULL); 183 free_irq(msixe[i].vector, accel_dev); 184} 185 186static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) 187{ 188 int i; 189 char **names; 190 struct msix_entry *entries; 191 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 192 u32 msix_num_entries = 1; 193 194 /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ 195 if (!accel_dev->pf.vf_info) 196 msix_num_entries += hw_data->num_banks; 197 198 entries = kcalloc_node(msix_num_entries, sizeof(*entries), 199 GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); 200 if (!entries) 201 return -ENOMEM; 202 203 names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL); 204 if (!names) { 205 kfree(entries); 206 return -ENOMEM; 207 } 208 for (i = 0; i < msix_num_entries; i++) { 209 *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); 210 if (!(*(names + i))) 211 goto err; 212 } 213 accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; 214 accel_dev->accel_pci_dev.msix_entries.entries = entries; 215 accel_dev->accel_pci_dev.msix_entries.names = names; 216 return 0; 217err: 218 for (i = 0; i < msix_num_entries; i++) 219 kfree(*(names + i)); 220 kfree(entries); 221 kfree(names); 222 return -ENOMEM; 223} 224 225static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) 226{ 227 char **names = accel_dev->accel_pci_dev.msix_entries.names; 228 int i; 229 230 kfree(accel_dev->accel_pci_dev.msix_entries.entries); 231 for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) 232 kfree(*(names + i)); 233 kfree(names); 234} 235 236static int adf_setup_bh(struct adf_accel_dev *accel_dev) 237{ 238 struct adf_etr_data *priv_data = accel_dev->transport; 239 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 240 int i; 241 242 for (i = 0; i < hw_data->num_banks; i++) 243 tasklet_init(&priv_data->banks[i].resp_handler, 244 adf_response_handler, 245 (unsigned long)&priv_data->banks[i]); 246 return 0; 247} 248 249static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) 250{ 251 struct adf_etr_data *priv_data = accel_dev->transport; 252 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 253 int i; 254 255 for (i = 0; i < hw_data->num_banks; i++) { 256 tasklet_disable(&priv_data->banks[i].resp_handler); 257 tasklet_kill(&priv_data->banks[i].resp_handler); 258 } 259} 260 261/** 262 * adf_isr_resource_free() - Free IRQ for acceleration device 263 * @accel_dev: Pointer to acceleration device. 264 * 265 * Function frees interrupts for acceleration device. 266 */ 267void adf_isr_resource_free(struct adf_accel_dev *accel_dev) 268{ 269 adf_free_irqs(accel_dev); 270 adf_cleanup_bh(accel_dev); 271 adf_disable_msix(&accel_dev->accel_pci_dev); 272 adf_isr_free_msix_entry_table(accel_dev); 273} 274EXPORT_SYMBOL_GPL(adf_isr_resource_free); 275 276/** 277 * adf_isr_resource_alloc() - Allocate IRQ for acceleration device 278 * @accel_dev: Pointer to acceleration device. 279 * 280 * Function allocates interrupts for acceleration device. 281 * 282 * Return: 0 on success, error code otherwise. 283 */ 284int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) 285{ 286 int ret; 287 288 ret = adf_isr_alloc_msix_entry_table(accel_dev); 289 if (ret) 290 goto err_out; 291 292 ret = adf_enable_msix(accel_dev); 293 if (ret) 294 goto err_free_msix_table; 295 296 ret = adf_setup_bh(accel_dev); 297 if (ret) 298 goto err_disable_msix; 299 300 ret = adf_request_irqs(accel_dev); 301 if (ret) 302 goto err_cleanup_bh; 303 304 return 0; 305 306err_cleanup_bh: 307 adf_cleanup_bh(accel_dev); 308 309err_disable_msix: 310 adf_disable_msix(&accel_dev->accel_pci_dev); 311 312err_free_msix_table: 313 adf_isr_free_msix_entry_table(accel_dev); 314 315err_out: 316 return ret; 317} 318EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); 319