1// SPDX-License-Identifier: GPL-2.0 2/* 3 * CDX bus driver. 4 * 5 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. 6 */ 7 8/* 9 * Architecture Overview 10 * ===================== 11 * CDX is a Hardware Architecture designed for AMD FPGA devices. It 12 * consists of sophisticated mechanism for interaction between FPGA, 13 * Firmware and the APUs (Application CPUs). 14 * 15 * Firmware resides on RPU (Realtime CPUs) which interacts with 16 * the FPGA program manager and the APUs. The RPU provides memory-mapped 17 * interface (RPU if) which is used to communicate with APUs. 18 * 19 * The diagram below shows an overview of the CDX architecture: 20 * 21 * +--------------------------------------+ 22 * | Application CPUs (APU) | 23 * | | 24 * | CDX device drivers| 25 * | Linux OS | | 26 * | CDX bus | 27 * | | | 28 * | CDX controller | 29 * | | | 30 * +-----------------------------|--------+ 31 * | (discover, config, 32 * | reset, rescan) 33 * | 34 * +------------------------| RPU if |----+ 35 * | | | 36 * | V | 37 * | Realtime CPUs (RPU) | 38 * | | 39 * +--------------------------------------+ 40 * | 41 * +---------------------|----------------+ 42 * | FPGA | | 43 * | +-----------------------+ | 44 * | | | | | 45 * | +-------+ +-------+ +-------+ | 46 * | | dev 1 | | dev 2 | | dev 3 | | 47 * | +-------+ +-------+ +-------+ | 48 * +--------------------------------------+ 49 * 50 * The RPU firmware extracts the device information from the loaded FPGA 51 * image and implements a mechanism that allows the APU drivers to 52 * enumerate such devices (device personality and resource details) via 53 * a dedicated communication channel. RPU mediates operations such as 54 * discover, reset and rescan of the FPGA devices for the APU. This is 55 * done using memory mapped interface provided by the RPU to APU. 56 */ 57 58#include <linux/init.h> 59#include <linux/kernel.h> 60#include <linux/of_device.h> 61#include <linux/slab.h> 62#include <linux/mm.h> 63#include <linux/xarray.h> 64#include <linux/cdx/cdx_bus.h> 65#include <linux/iommu.h> 66#include <linux/dma-map-ops.h> 67#include "cdx.h" 68 69/* Default DMA mask for devices on a CDX bus */ 70#define CDX_DEFAULT_DMA_MASK (~0ULL) 71#define MAX_CDX_CONTROLLERS 16 72 73/* CDX controllers registered with the CDX bus */ 74static DEFINE_XARRAY_ALLOC(cdx_controllers); 75 76/** 77 * cdx_dev_reset - Reset a CDX device 78 * @dev: CDX device 79 * 80 * Return: -errno on failure, 0 on success. 81 */ 82int cdx_dev_reset(struct device *dev) 83{ 84 struct cdx_device *cdx_dev = to_cdx_device(dev); 85 struct cdx_controller *cdx = cdx_dev->cdx; 86 struct cdx_device_config dev_config = {0}; 87 struct cdx_driver *cdx_drv; 88 int ret; 89 90 cdx_drv = to_cdx_driver(dev->driver); 91 /* Notify driver that device is being reset */ 92 if (cdx_drv && cdx_drv->reset_prepare) 93 cdx_drv->reset_prepare(cdx_dev); 94 95 dev_config.type = CDX_DEV_RESET_CONF; 96 ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num, 97 cdx_dev->dev_num, &dev_config); 98 if (ret) 99 dev_err(dev, "cdx device reset failed\n"); 100 101 /* Notify driver that device reset is complete */ 102 if (cdx_drv && cdx_drv->reset_done) 103 cdx_drv->reset_done(cdx_dev); 104 105 return ret; 106} 107EXPORT_SYMBOL_GPL(cdx_dev_reset); 108 109/** 110 * cdx_unregister_device - Unregister a CDX device 111 * @dev: CDX device 112 * @data: This is always passed as NULL, and is not used in this API, 113 * but is required here as the bus_for_each_dev() API expects 114 * the passed function (cdx_unregister_device) to have this 115 * as an argument. 116 * 117 * Return: 0 on success. 118 */ 119static int cdx_unregister_device(struct device *dev, 120 void *data) 121{ 122 struct cdx_device *cdx_dev = to_cdx_device(dev); 123 124 kfree(cdx_dev->driver_override); 125 cdx_dev->driver_override = NULL; 126 /* 127 * Do not free cdx_dev here as it would be freed in 128 * cdx_device_release() called from within put_device(). 129 */ 130 device_del(&cdx_dev->dev); 131 put_device(&cdx_dev->dev); 132 133 return 0; 134} 135 136static void cdx_unregister_devices(struct bus_type *bus) 137{ 138 /* Reset all the devices attached to cdx bus */ 139 bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device); 140} 141 142/** 143 * cdx_match_one_device - Tell if a CDX device structure has a matching 144 * CDX device id structure 145 * @id: single CDX device id structure to match 146 * @dev: the CDX device structure to match against 147 * 148 * Return: matching cdx_device_id structure or NULL if there is no match. 149 */ 150static inline const struct cdx_device_id * 151cdx_match_one_device(const struct cdx_device_id *id, 152 const struct cdx_device *dev) 153{ 154 /* Use vendor ID and device ID for matching */ 155 if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) && 156 (id->device == CDX_ANY_ID || id->device == dev->device)) 157 return id; 158 return NULL; 159} 160 161/** 162 * cdx_match_id - See if a CDX device matches a given cdx_id table 163 * @ids: array of CDX device ID structures to search in 164 * @dev: the CDX device structure to match against. 165 * 166 * Used by a driver to check whether a CDX device is in its list of 167 * supported devices. Returns the matching cdx_device_id structure or 168 * NULL if there is no match. 169 * 170 * Return: matching cdx_device_id structure or NULL if there is no match. 171 */ 172static inline const struct cdx_device_id * 173cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev) 174{ 175 if (ids) { 176 while (ids->vendor || ids->device) { 177 if (cdx_match_one_device(ids, dev)) 178 return ids; 179 ids++; 180 } 181 } 182 return NULL; 183} 184 185/** 186 * cdx_bus_match - device to driver matching callback 187 * @dev: the cdx device to match against 188 * @drv: the device driver to search for matching cdx device 189 * structures 190 * 191 * Return: true on success, false otherwise. 192 */ 193static int cdx_bus_match(struct device *dev, struct device_driver *drv) 194{ 195 struct cdx_device *cdx_dev = to_cdx_device(dev); 196 struct cdx_driver *cdx_drv = to_cdx_driver(drv); 197 const struct cdx_device_id *found_id = NULL; 198 const struct cdx_device_id *ids; 199 200 ids = cdx_drv->match_id_table; 201 202 /* When driver_override is set, only bind to the matching driver */ 203 if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name)) 204 return false; 205 206 found_id = cdx_match_id(ids, cdx_dev); 207 if (!found_id) 208 return false; 209 210 do { 211 /* 212 * In case override_only was set, enforce driver_override 213 * matching. 214 */ 215 if (!found_id->override_only) 216 return true; 217 if (cdx_dev->driver_override) 218 return true; 219 220 ids = found_id + 1; 221 found_id = cdx_match_id(ids, cdx_dev); 222 } while (found_id); 223 224 return false; 225} 226 227static int cdx_probe(struct device *dev) 228{ 229 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 230 struct cdx_device *cdx_dev = to_cdx_device(dev); 231 int error; 232 233 error = cdx_drv->probe(cdx_dev); 234 if (error) { 235 dev_err_probe(dev, error, "%s failed\n", __func__); 236 return error; 237 } 238 239 return 0; 240} 241 242static void cdx_remove(struct device *dev) 243{ 244 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 245 struct cdx_device *cdx_dev = to_cdx_device(dev); 246 247 if (cdx_drv && cdx_drv->remove) 248 cdx_drv->remove(cdx_dev); 249} 250 251static void cdx_shutdown(struct device *dev) 252{ 253 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 254 struct cdx_device *cdx_dev = to_cdx_device(dev); 255 256 if (cdx_drv && cdx_drv->shutdown) 257 cdx_drv->shutdown(cdx_dev); 258} 259 260static int cdx_dma_configure(struct device *dev) 261{ 262 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 263 struct cdx_device *cdx_dev = to_cdx_device(dev); 264 u32 input_id = cdx_dev->req_id; 265 int ret; 266 267 ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id); 268 if (ret && ret != -EPROBE_DEFER) { 269 dev_err(dev, "of_dma_configure_id() failed\n"); 270 return ret; 271 } 272 273 if (!ret && !cdx_drv->driver_managed_dma) { 274 ret = iommu_device_use_default_domain(dev); 275 if (ret) 276 arch_teardown_dma_ops(dev); 277 } 278 279 return 0; 280} 281 282static void cdx_dma_cleanup(struct device *dev) 283{ 284 struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); 285 286 if (!cdx_drv->driver_managed_dma) 287 iommu_device_unuse_default_domain(dev); 288} 289 290/* show configuration fields */ 291#define cdx_config_attr(field, format_string) \ 292static ssize_t \ 293field##_show(struct device *dev, struct device_attribute *attr, char *buf) \ 294{ \ 295 struct cdx_device *cdx_dev = to_cdx_device(dev); \ 296 return sysfs_emit(buf, format_string, cdx_dev->field); \ 297} \ 298static DEVICE_ATTR_RO(field) 299 300cdx_config_attr(vendor, "0x%04x\n"); 301cdx_config_attr(device, "0x%04x\n"); 302 303static ssize_t remove_store(struct device *dev, 304 struct device_attribute *attr, 305 const char *buf, size_t count) 306{ 307 bool val; 308 309 if (kstrtobool(buf, &val) < 0) 310 return -EINVAL; 311 312 if (!val) 313 return -EINVAL; 314 315 if (device_remove_file_self(dev, attr)) { 316 int ret; 317 318 ret = cdx_unregister_device(dev, NULL); 319 if (ret) 320 return ret; 321 } 322 323 return count; 324} 325static DEVICE_ATTR_WO(remove); 326 327static ssize_t reset_store(struct device *dev, struct device_attribute *attr, 328 const char *buf, size_t count) 329{ 330 bool val; 331 int ret; 332 333 if (kstrtobool(buf, &val) < 0) 334 return -EINVAL; 335 336 if (!val) 337 return -EINVAL; 338 339 ret = cdx_dev_reset(dev); 340 if (ret) 341 return ret; 342 343 return count; 344} 345static DEVICE_ATTR_WO(reset); 346 347static ssize_t driver_override_store(struct device *dev, 348 struct device_attribute *attr, 349 const char *buf, size_t count) 350{ 351 struct cdx_device *cdx_dev = to_cdx_device(dev); 352 int ret; 353 354 if (WARN_ON(dev->bus != &cdx_bus_type)) 355 return -EINVAL; 356 357 ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count); 358 if (ret) 359 return ret; 360 361 return count; 362} 363 364static ssize_t driver_override_show(struct device *dev, 365 struct device_attribute *attr, char *buf) 366{ 367 struct cdx_device *cdx_dev = to_cdx_device(dev); 368 369 return sysfs_emit(buf, "%s\n", cdx_dev->driver_override); 370} 371static DEVICE_ATTR_RW(driver_override); 372 373static struct attribute *cdx_dev_attrs[] = { 374 &dev_attr_remove.attr, 375 &dev_attr_reset.attr, 376 &dev_attr_vendor.attr, 377 &dev_attr_device.attr, 378 &dev_attr_driver_override.attr, 379 NULL, 380}; 381ATTRIBUTE_GROUPS(cdx_dev); 382 383static ssize_t rescan_store(const struct bus_type *bus, 384 const char *buf, size_t count) 385{ 386 struct cdx_controller *cdx; 387 unsigned long index; 388 bool val; 389 390 if (kstrtobool(buf, &val) < 0) 391 return -EINVAL; 392 393 if (!val) 394 return -EINVAL; 395 396 /* Unregister all the devices on the bus */ 397 cdx_unregister_devices(&cdx_bus_type); 398 399 /* Rescan all the devices */ 400 xa_for_each(&cdx_controllers, index, cdx) { 401 int ret; 402 403 ret = cdx->ops->scan(cdx); 404 if (ret) 405 dev_err(cdx->dev, "cdx bus scanning failed\n"); 406 } 407 408 return count; 409} 410static BUS_ATTR_WO(rescan); 411 412static struct attribute *cdx_bus_attrs[] = { 413 &bus_attr_rescan.attr, 414 NULL, 415}; 416ATTRIBUTE_GROUPS(cdx_bus); 417 418struct bus_type cdx_bus_type = { 419 .name = "cdx", 420 .match = cdx_bus_match, 421 .probe = cdx_probe, 422 .remove = cdx_remove, 423 .shutdown = cdx_shutdown, 424 .dma_configure = cdx_dma_configure, 425 .dma_cleanup = cdx_dma_cleanup, 426 .bus_groups = cdx_bus_groups, 427 .dev_groups = cdx_dev_groups, 428}; 429EXPORT_SYMBOL_GPL(cdx_bus_type); 430 431int __cdx_driver_register(struct cdx_driver *cdx_driver, 432 struct module *owner) 433{ 434 int error; 435 436 cdx_driver->driver.owner = owner; 437 cdx_driver->driver.bus = &cdx_bus_type; 438 439 error = driver_register(&cdx_driver->driver); 440 if (error) { 441 pr_err("driver_register() failed for %s: %d\n", 442 cdx_driver->driver.name, error); 443 return error; 444 } 445 446 return 0; 447} 448EXPORT_SYMBOL_GPL(__cdx_driver_register); 449 450void cdx_driver_unregister(struct cdx_driver *cdx_driver) 451{ 452 driver_unregister(&cdx_driver->driver); 453} 454EXPORT_SYMBOL_GPL(cdx_driver_unregister); 455 456static void cdx_device_release(struct device *dev) 457{ 458 struct cdx_device *cdx_dev = to_cdx_device(dev); 459 460 kfree(cdx_dev); 461} 462 463int cdx_device_add(struct cdx_dev_params *dev_params) 464{ 465 struct cdx_controller *cdx = dev_params->cdx; 466 struct device *parent = cdx->dev; 467 struct cdx_device *cdx_dev; 468 int ret; 469 470 cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL); 471 if (!cdx_dev) 472 return -ENOMEM; 473 474 /* Populate resource */ 475 memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) * 476 dev_params->res_count); 477 cdx_dev->res_count = dev_params->res_count; 478 479 /* Populate CDX dev params */ 480 cdx_dev->req_id = dev_params->req_id; 481 cdx_dev->vendor = dev_params->vendor; 482 cdx_dev->device = dev_params->device; 483 cdx_dev->bus_num = dev_params->bus_num; 484 cdx_dev->dev_num = dev_params->dev_num; 485 cdx_dev->cdx = dev_params->cdx; 486 cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK; 487 488 /* Initialize generic device */ 489 device_initialize(&cdx_dev->dev); 490 cdx_dev->dev.parent = parent; 491 cdx_dev->dev.bus = &cdx_bus_type; 492 cdx_dev->dev.dma_mask = &cdx_dev->dma_mask; 493 cdx_dev->dev.release = cdx_device_release; 494 495 /* Set Name */ 496 dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", 497 ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)), 498 cdx_dev->dev_num); 499 500 ret = device_add(&cdx_dev->dev); 501 if (ret) { 502 dev_err(&cdx_dev->dev, 503 "cdx device add failed: %d", ret); 504 goto fail; 505 } 506 507 return 0; 508fail: 509 /* 510 * Do not free cdx_dev here as it would be freed in 511 * cdx_device_release() called from put_device(). 512 */ 513 put_device(&cdx_dev->dev); 514 515 return ret; 516} 517EXPORT_SYMBOL_GPL(cdx_device_add); 518 519int cdx_register_controller(struct cdx_controller *cdx) 520{ 521 int ret; 522 523 ret = xa_alloc(&cdx_controllers, &cdx->id, cdx, 524 XA_LIMIT(0, MAX_CDX_CONTROLLERS - 1), GFP_KERNEL); 525 if (ret) { 526 dev_err(cdx->dev, 527 "No free index available. Maximum controllers already registered\n"); 528 cdx->id = (u8)MAX_CDX_CONTROLLERS; 529 return ret; 530 } 531 532 /* Scan all the devices */ 533 cdx->ops->scan(cdx); 534 535 return 0; 536} 537EXPORT_SYMBOL_GPL(cdx_register_controller); 538 539void cdx_unregister_controller(struct cdx_controller *cdx) 540{ 541 if (cdx->id >= MAX_CDX_CONTROLLERS) 542 return; 543 544 device_for_each_child(cdx->dev, NULL, cdx_unregister_device); 545 xa_erase(&cdx_controllers, cdx->id); 546} 547EXPORT_SYMBOL_GPL(cdx_unregister_controller); 548 549static int __init cdx_bus_init(void) 550{ 551 return bus_register(&cdx_bus_type); 552} 553postcore_initcall(cdx_bus_init); 554