1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3#include <linux/init.h> 4#include <linux/kernel.h> 5#include <linux/module.h> 6#include <linux/pci.h> 7#include <linux/device.h> 8#include <linux/sched/task.h> 9#include <linux/intel-svm.h> 10#include <linux/io-64-nonatomic-lo-hi.h> 11#include <linux/cdev.h> 12#include <linux/fs.h> 13#include <linux/poll.h> 14#include <uapi/linux/idxd.h> 15#include "registers.h" 16#include "idxd.h" 17 18struct idxd_cdev_context { 19 const char *name; 20 dev_t devt; 21 struct ida minor_ida; 22}; 23 24/* 25 * ictx is an array based off of accelerator types. enum idxd_type 26 * is used as index 27 */ 28static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 29 { .name = "dsa" }, 30}; 31 32struct idxd_user_context { 33 struct idxd_wq *wq; 34 struct task_struct *task; 35 unsigned int flags; 36}; 37 38static void idxd_cdev_dev_release(struct device *dev) 39{ 40 struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev); 41 struct idxd_cdev_context *cdev_ctx; 42 struct idxd_wq *wq = idxd_cdev->wq; 43 44 cdev_ctx = &ictx[wq->idxd->type]; 45 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 46 kfree(idxd_cdev); 47} 48 49static struct device_type idxd_cdev_device_type = { 50 .name = "idxd_cdev", 51 .release = idxd_cdev_dev_release, 52}; 53 54static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 55{ 56 struct cdev *cdev = inode->i_cdev; 57 58 return container_of(cdev, struct idxd_cdev, cdev); 59} 60 61static inline struct idxd_wq *inode_wq(struct inode *inode) 62{ 63 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 64 65 return idxd_cdev->wq; 66} 67 68static int idxd_cdev_open(struct inode *inode, struct file *filp) 69{ 70 struct idxd_user_context *ctx; 71 struct idxd_device *idxd; 72 struct idxd_wq *wq; 73 struct device *dev; 74 int rc = 0; 75 76 wq = inode_wq(inode); 77 idxd = wq->idxd; 78 dev = &idxd->pdev->dev; 79 80 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 81 82 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 83 if (!ctx) 84 return -ENOMEM; 85 86 mutex_lock(&wq->wq_lock); 87 88 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 89 rc = -EBUSY; 90 goto failed; 91 } 92 93 ctx->wq = wq; 94 filp->private_data = ctx; 95 idxd_wq_get(wq); 96 mutex_unlock(&wq->wq_lock); 97 return 0; 98 99 failed: 100 mutex_unlock(&wq->wq_lock); 101 kfree(ctx); 102 return rc; 103} 104 105static int idxd_cdev_release(struct inode *node, struct file *filep) 106{ 107 struct idxd_user_context *ctx = filep->private_data; 108 struct idxd_wq *wq = ctx->wq; 109 struct idxd_device *idxd = wq->idxd; 110 struct device *dev = &idxd->pdev->dev; 111 112 dev_dbg(dev, "%s called\n", __func__); 113 filep->private_data = NULL; 114 115 /* Wait for in-flight operations to complete. */ 116 idxd_wq_drain(wq); 117 118 kfree(ctx); 119 mutex_lock(&wq->wq_lock); 120 idxd_wq_put(wq); 121 mutex_unlock(&wq->wq_lock); 122 return 0; 123} 124 125static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 126 const char *func) 127{ 128 struct device *dev = &wq->idxd->pdev->dev; 129 130 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 131 dev_info_ratelimited(dev, 132 "%s: %s: mapping too large: %lu\n", 133 current->comm, func, 134 vma->vm_end - vma->vm_start); 135 return -EINVAL; 136 } 137 138 return 0; 139} 140 141static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 142{ 143 struct idxd_user_context *ctx = filp->private_data; 144 struct idxd_wq *wq = ctx->wq; 145 struct idxd_device *idxd = wq->idxd; 146 struct pci_dev *pdev = idxd->pdev; 147 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 148 unsigned long pfn; 149 int rc; 150 151 dev_dbg(&pdev->dev, "%s called\n", __func__); 152 rc = check_vma(wq, vma, __func__); 153 if (rc < 0) 154 return rc; 155 156 vma->vm_flags |= VM_DONTCOPY; 157 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 158 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 159 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 160 vma->vm_private_data = ctx; 161 162 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 163 vma->vm_page_prot); 164} 165 166static __poll_t idxd_cdev_poll(struct file *filp, 167 struct poll_table_struct *wait) 168{ 169 struct idxd_user_context *ctx = filp->private_data; 170 struct idxd_wq *wq = ctx->wq; 171 struct idxd_device *idxd = wq->idxd; 172 unsigned long flags; 173 __poll_t out = 0; 174 175 poll_wait(filp, &wq->err_queue, wait); 176 spin_lock_irqsave(&idxd->dev_lock, flags); 177 if (idxd->sw_err.valid) 178 out = EPOLLIN | EPOLLRDNORM; 179 spin_unlock_irqrestore(&idxd->dev_lock, flags); 180 181 return out; 182} 183 184static const struct file_operations idxd_cdev_fops = { 185 .owner = THIS_MODULE, 186 .open = idxd_cdev_open, 187 .release = idxd_cdev_release, 188 .mmap = idxd_cdev_mmap, 189 .poll = idxd_cdev_poll, 190}; 191 192int idxd_cdev_get_major(struct idxd_device *idxd) 193{ 194 return MAJOR(ictx[idxd->type].devt); 195} 196 197int idxd_wq_add_cdev(struct idxd_wq *wq) 198{ 199 struct idxd_device *idxd = wq->idxd; 200 struct idxd_cdev *idxd_cdev; 201 struct cdev *cdev; 202 struct device *dev; 203 struct idxd_cdev_context *cdev_ctx; 204 int rc, minor; 205 206 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 207 if (!idxd_cdev) 208 return -ENOMEM; 209 210 idxd_cdev->wq = wq; 211 cdev = &idxd_cdev->cdev; 212 dev = &idxd_cdev->dev; 213 cdev_ctx = &ictx[wq->idxd->type]; 214 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 215 if (minor < 0) { 216 kfree(idxd_cdev); 217 return minor; 218 } 219 idxd_cdev->minor = minor; 220 221 device_initialize(dev); 222 dev->parent = &wq->conf_dev; 223 dev->bus = idxd_get_bus_type(idxd); 224 dev->type = &idxd_cdev_device_type; 225 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 226 227 rc = dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd), 228 idxd->id, wq->id); 229 if (rc < 0) 230 goto err; 231 232 wq->idxd_cdev = idxd_cdev; 233 cdev_init(cdev, &idxd_cdev_fops); 234 rc = cdev_device_add(cdev, dev); 235 if (rc) { 236 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 237 goto err; 238 } 239 240 return 0; 241 242 err: 243 put_device(dev); 244 wq->idxd_cdev = NULL; 245 return rc; 246} 247 248void idxd_wq_del_cdev(struct idxd_wq *wq) 249{ 250 struct idxd_cdev *idxd_cdev; 251 struct idxd_cdev_context *cdev_ctx; 252 253 cdev_ctx = &ictx[wq->idxd->type]; 254 idxd_cdev = wq->idxd_cdev; 255 wq->idxd_cdev = NULL; 256 cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev); 257 put_device(&idxd_cdev->dev); 258} 259 260int idxd_cdev_register(void) 261{ 262 int rc, i; 263 264 for (i = 0; i < IDXD_TYPE_MAX; i++) { 265 ida_init(&ictx[i].minor_ida); 266 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 267 ictx[i].name); 268 if (rc) 269 goto err_free_chrdev_region; 270 } 271 272 return 0; 273 274err_free_chrdev_region: 275 for (i--; i >= 0; i--) 276 unregister_chrdev_region(ictx[i].devt, MINORMASK); 277 278 return rc; 279} 280 281void idxd_cdev_remove(void) 282{ 283 int i; 284 285 for (i = 0; i < IDXD_TYPE_MAX; i++) { 286 unregister_chrdev_region(ictx[i].devt, MINORMASK); 287 ida_destroy(&ictx[i].minor_ida); 288 } 289} 290