1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright(c) 2019 HiSilicon Limited. */ 3#include <linux/bitfield.h> 4#include <linux/dmaengine.h> 5#include <linux/init.h> 6#include <linux/iopoll.h> 7#include <linux/module.h> 8#include <linux/pci.h> 9#include <linux/spinlock.h> 10#include "virt-dma.h" 11 12#define HISI_DMA_SQ_BASE_L 0x0 13#define HISI_DMA_SQ_BASE_H 0x4 14#define HISI_DMA_SQ_DEPTH 0x8 15#define HISI_DMA_SQ_TAIL_PTR 0xc 16#define HISI_DMA_CQ_BASE_L 0x10 17#define HISI_DMA_CQ_BASE_H 0x14 18#define HISI_DMA_CQ_DEPTH 0x18 19#define HISI_DMA_CQ_HEAD_PTR 0x1c 20#define HISI_DMA_CTRL0 0x20 21#define HISI_DMA_CTRL0_QUEUE_EN_S 0 22#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4 23#define HISI_DMA_CTRL1 0x24 24#define HISI_DMA_CTRL1_QUEUE_RESET_S 0 25#define HISI_DMA_Q_FSM_STS 0x30 26#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0) 27#define HISI_DMA_INT_STS 0x40 28#define HISI_DMA_INT_STS_MASK GENMASK(12, 0) 29#define HISI_DMA_INT_MSK 0x44 30#define HISI_DMA_MODE 0x217c 31#define HISI_DMA_OFFSET 0x100 32 33#define HISI_DMA_MSI_NUM 32 34#define HISI_DMA_CHAN_NUM 30 35#define HISI_DMA_Q_DEPTH_VAL 1024 36 37#define PCI_BAR_2 2 38 39enum hisi_dma_mode { 40 EP = 0, 41 RC, 42}; 43 44enum hisi_dma_chan_status { 45 DISABLE = -1, 46 IDLE = 0, 47 RUN, 48 CPL, 49 PAUSE, 50 HALT, 51 ABORT, 52 WAIT, 53 BUFFCLR, 54}; 55 56struct hisi_dma_sqe { 57 __le32 dw0; 58#define OPCODE_MASK GENMASK(3, 0) 59#define OPCODE_SMALL_PACKAGE 0x1 60#define OPCODE_M2M 0x4 61#define LOCAL_IRQ_EN BIT(8) 62#define ATTR_SRC_MASK GENMASK(14, 12) 63 __le32 dw1; 64 __le32 dw2; 65#define ATTR_DST_MASK GENMASK(26, 24) 66 __le32 length; 67 __le64 src_addr; 68 __le64 dst_addr; 69}; 70 71struct hisi_dma_cqe { 72 __le32 rsv0; 73 __le32 rsv1; 74 __le16 sq_head; 75 __le16 rsv2; 76 __le16 rsv3; 77 __le16 w0; 78#define STATUS_MASK GENMASK(15, 1) 79#define STATUS_SUCC 0x0 80#define VALID_BIT BIT(0) 81}; 82 83struct hisi_dma_desc { 84 struct virt_dma_desc vd; 85 struct hisi_dma_sqe sqe; 86}; 87 88struct hisi_dma_chan { 89 struct virt_dma_chan vc; 90 struct hisi_dma_dev *hdma_dev; 91 struct hisi_dma_sqe *sq; 92 struct hisi_dma_cqe *cq; 93 dma_addr_t sq_dma; 94 dma_addr_t cq_dma; 95 u32 sq_tail; 96 u32 cq_head; 97 u32 qp_num; 98 enum hisi_dma_chan_status status; 99 struct hisi_dma_desc *desc; 100}; 101 102struct hisi_dma_dev { 103 struct pci_dev *pdev; 104 void __iomem *base; 105 struct dma_device dma_dev; 106 u32 chan_num; 107 u32 chan_depth; 108 struct hisi_dma_chan chan[]; 109}; 110 111static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c) 112{ 113 return container_of(c, struct hisi_dma_chan, vc.chan); 114} 115 116static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd) 117{ 118 return container_of(vd, struct hisi_dma_desc, vd); 119} 120 121static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index, 122 u32 val) 123{ 124 writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET); 125} 126 127static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val) 128{ 129 u32 tmp; 130 131 tmp = readl_relaxed(addr); 132 tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos); 133 writel_relaxed(tmp, addr); 134} 135 136static void hisi_dma_free_irq_vectors(void *data) 137{ 138 pci_free_irq_vectors(data); 139} 140 141static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index, 142 bool pause) 143{ 144 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index * 145 HISI_DMA_OFFSET; 146 147 hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause); 148} 149 150static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index, 151 bool enable) 152{ 153 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index * 154 HISI_DMA_OFFSET; 155 156 hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable); 157} 158 159static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) 160{ 161 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index, 162 HISI_DMA_INT_STS_MASK); 163} 164 165static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) 166{ 167 void __iomem *base = hdma_dev->base; 168 169 hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index, 170 HISI_DMA_INT_STS_MASK); 171 hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0); 172} 173 174static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index) 175{ 176 void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index * 177 HISI_DMA_OFFSET; 178 179 hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1); 180} 181 182static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index) 183{ 184 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0); 185 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0); 186} 187 188static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan, 189 bool disable) 190{ 191 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 192 u32 index = chan->qp_num, tmp; 193 int ret; 194 195 hisi_dma_pause_dma(hdma_dev, index, true); 196 hisi_dma_enable_dma(hdma_dev, index, false); 197 hisi_dma_mask_irq(hdma_dev, index); 198 199 ret = readl_relaxed_poll_timeout(hdma_dev->base + 200 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp, 201 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000); 202 if (ret) { 203 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n"); 204 WARN_ON(1); 205 } 206 207 hisi_dma_do_reset(hdma_dev, index); 208 hisi_dma_reset_qp_point(hdma_dev, index); 209 hisi_dma_pause_dma(hdma_dev, index, false); 210 211 if (!disable) { 212 hisi_dma_enable_dma(hdma_dev, index, true); 213 hisi_dma_unmask_irq(hdma_dev, index); 214 } 215 216 ret = readl_relaxed_poll_timeout(hdma_dev->base + 217 HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp, 218 FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000); 219 if (ret) { 220 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n"); 221 WARN_ON(1); 222 } 223} 224 225static void hisi_dma_free_chan_resources(struct dma_chan *c) 226{ 227 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 228 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 229 230 hisi_dma_reset_or_disable_hw_chan(chan, false); 231 vchan_free_chan_resources(&chan->vc); 232 233 memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth); 234 memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth); 235 chan->sq_tail = 0; 236 chan->cq_head = 0; 237 chan->status = DISABLE; 238} 239 240static void hisi_dma_desc_free(struct virt_dma_desc *vd) 241{ 242 kfree(to_hisi_dma_desc(vd)); 243} 244 245static struct dma_async_tx_descriptor * 246hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src, 247 size_t len, unsigned long flags) 248{ 249 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 250 struct hisi_dma_desc *desc; 251 252 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 253 if (!desc) 254 return NULL; 255 256 desc->sqe.length = cpu_to_le32(len); 257 desc->sqe.src_addr = cpu_to_le64(src); 258 desc->sqe.dst_addr = cpu_to_le64(dst); 259 260 return vchan_tx_prep(&chan->vc, &desc->vd, flags); 261} 262 263static enum dma_status 264hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, 265 struct dma_tx_state *txstate) 266{ 267 return dma_cookie_status(c, cookie, txstate); 268} 269 270static void hisi_dma_start_transfer(struct hisi_dma_chan *chan) 271{ 272 struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail; 273 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 274 struct hisi_dma_desc *desc; 275 struct virt_dma_desc *vd; 276 277 vd = vchan_next_desc(&chan->vc); 278 if (!vd) { 279 chan->desc = NULL; 280 return; 281 } 282 list_del(&vd->node); 283 desc = to_hisi_dma_desc(vd); 284 chan->desc = desc; 285 286 memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe)); 287 288 /* update other field in sqe */ 289 sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M)); 290 sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN); 291 292 /* make sure data has been updated in sqe */ 293 wmb(); 294 295 /* update sq tail, point to new sqe position */ 296 chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth; 297 298 /* update sq_tail to trigger a new task */ 299 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num, 300 chan->sq_tail); 301} 302 303static void hisi_dma_issue_pending(struct dma_chan *c) 304{ 305 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 306 unsigned long flags; 307 308 spin_lock_irqsave(&chan->vc.lock, flags); 309 310 if (vchan_issue_pending(&chan->vc) && !chan->desc) 311 hisi_dma_start_transfer(chan); 312 313 spin_unlock_irqrestore(&chan->vc.lock, flags); 314} 315 316static int hisi_dma_terminate_all(struct dma_chan *c) 317{ 318 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 319 unsigned long flags; 320 LIST_HEAD(head); 321 322 spin_lock_irqsave(&chan->vc.lock, flags); 323 324 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true); 325 if (chan->desc) { 326 vchan_terminate_vdesc(&chan->desc->vd); 327 chan->desc = NULL; 328 } 329 330 vchan_get_all_descriptors(&chan->vc, &head); 331 332 spin_unlock_irqrestore(&chan->vc.lock, flags); 333 334 vchan_dma_desc_free_list(&chan->vc, &head); 335 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false); 336 337 return 0; 338} 339 340static void hisi_dma_synchronize(struct dma_chan *c) 341{ 342 struct hisi_dma_chan *chan = to_hisi_dma_chan(c); 343 344 vchan_synchronize(&chan->vc); 345} 346 347static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev) 348{ 349 size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth; 350 size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth; 351 struct device *dev = &hdma_dev->pdev->dev; 352 struct hisi_dma_chan *chan; 353 int i; 354 355 for (i = 0; i < hdma_dev->chan_num; i++) { 356 chan = &hdma_dev->chan[i]; 357 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma, 358 GFP_KERNEL); 359 if (!chan->sq) 360 return -ENOMEM; 361 362 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma, 363 GFP_KERNEL); 364 if (!chan->cq) 365 return -ENOMEM; 366 } 367 368 return 0; 369} 370 371static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index) 372{ 373 struct hisi_dma_chan *chan = &hdma_dev->chan[index]; 374 u32 hw_depth = hdma_dev->chan_depth - 1; 375 void __iomem *base = hdma_dev->base; 376 377 /* set sq, cq base */ 378 hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index, 379 lower_32_bits(chan->sq_dma)); 380 hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index, 381 upper_32_bits(chan->sq_dma)); 382 hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index, 383 lower_32_bits(chan->cq_dma)); 384 hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index, 385 upper_32_bits(chan->cq_dma)); 386 387 /* set sq, cq depth */ 388 hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth); 389 hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth); 390 391 /* init sq tail and cq head */ 392 hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0); 393 hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0); 394} 395 396static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) 397{ 398 hisi_dma_init_hw_qp(hdma_dev, qp_index); 399 hisi_dma_unmask_irq(hdma_dev, qp_index); 400 hisi_dma_enable_dma(hdma_dev, qp_index, true); 401} 402 403static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) 404{ 405 hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true); 406} 407 408static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev) 409{ 410 int i; 411 412 for (i = 0; i < hdma_dev->chan_num; i++) { 413 hdma_dev->chan[i].qp_num = i; 414 hdma_dev->chan[i].hdma_dev = hdma_dev; 415 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free; 416 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev); 417 hisi_dma_enable_qp(hdma_dev, i); 418 } 419} 420 421static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev) 422{ 423 int i; 424 425 for (i = 0; i < hdma_dev->chan_num; i++) { 426 hisi_dma_disable_qp(hdma_dev, i); 427 tasklet_kill(&hdma_dev->chan[i].vc.task); 428 } 429} 430 431static irqreturn_t hisi_dma_irq(int irq, void *data) 432{ 433 struct hisi_dma_chan *chan = data; 434 struct hisi_dma_dev *hdma_dev = chan->hdma_dev; 435 struct hisi_dma_desc *desc; 436 struct hisi_dma_cqe *cqe; 437 unsigned long flags; 438 439 spin_lock_irqsave(&chan->vc.lock, flags); 440 441 desc = chan->desc; 442 cqe = chan->cq + chan->cq_head; 443 if (desc) { 444 chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth; 445 hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, 446 chan->qp_num, chan->cq_head); 447 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) { 448 vchan_cookie_complete(&desc->vd); 449 hisi_dma_start_transfer(chan); 450 } else { 451 dev_err(&hdma_dev->pdev->dev, "task error!\n"); 452 } 453 } 454 455 spin_unlock_irqrestore(&chan->vc.lock, flags); 456 457 return IRQ_HANDLED; 458} 459 460static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev) 461{ 462 struct pci_dev *pdev = hdma_dev->pdev; 463 int i, ret; 464 465 for (i = 0; i < hdma_dev->chan_num; i++) { 466 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i), 467 hisi_dma_irq, IRQF_SHARED, "hisi_dma", 468 &hdma_dev->chan[i]); 469 if (ret) 470 return ret; 471 } 472 473 return 0; 474} 475 476/* This function enables all hw channels in a device */ 477static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev) 478{ 479 int ret; 480 481 ret = hisi_dma_alloc_qps_mem(hdma_dev); 482 if (ret) { 483 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n"); 484 return ret; 485 } 486 487 ret = hisi_dma_request_qps_irq(hdma_dev); 488 if (ret) { 489 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n"); 490 return ret; 491 } 492 493 hisi_dma_enable_qps(hdma_dev); 494 495 return 0; 496} 497 498static void hisi_dma_disable_hw_channels(void *data) 499{ 500 hisi_dma_disable_qps(data); 501} 502 503static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev, 504 enum hisi_dma_mode mode) 505{ 506 writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE); 507} 508 509static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) 510{ 511 struct device *dev = &pdev->dev; 512 struct hisi_dma_dev *hdma_dev; 513 struct dma_device *dma_dev; 514 int ret; 515 516 ret = pcim_enable_device(pdev); 517 if (ret) { 518 dev_err(dev, "failed to enable device mem!\n"); 519 return ret; 520 } 521 522 ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev)); 523 if (ret) { 524 dev_err(dev, "failed to remap I/O region!\n"); 525 return ret; 526 } 527 528 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 529 if (ret) 530 return ret; 531 532 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 533 if (ret) 534 return ret; 535 536 hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL); 537 if (!hdma_dev) 538 return -EINVAL; 539 540 hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; 541 hdma_dev->pdev = pdev; 542 hdma_dev->chan_num = HISI_DMA_CHAN_NUM; 543 hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL; 544 545 pci_set_drvdata(pdev, hdma_dev); 546 pci_set_master(pdev); 547 548 ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM, 549 PCI_IRQ_MSI); 550 if (ret < 0) { 551 dev_err(dev, "Failed to allocate MSI vectors!\n"); 552 return ret; 553 } 554 555 ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev); 556 if (ret) 557 return ret; 558 559 dma_dev = &hdma_dev->dma_dev; 560 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 561 dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources; 562 dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy; 563 dma_dev->device_tx_status = hisi_dma_tx_status; 564 dma_dev->device_issue_pending = hisi_dma_issue_pending; 565 dma_dev->device_terminate_all = hisi_dma_terminate_all; 566 dma_dev->device_synchronize = hisi_dma_synchronize; 567 dma_dev->directions = BIT(DMA_MEM_TO_MEM); 568 dma_dev->dev = dev; 569 INIT_LIST_HEAD(&dma_dev->channels); 570 571 hisi_dma_set_mode(hdma_dev, RC); 572 573 ret = hisi_dma_enable_hw_channels(hdma_dev); 574 if (ret < 0) { 575 dev_err(dev, "failed to enable hw channel!\n"); 576 return ret; 577 } 578 579 ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels, 580 hdma_dev); 581 if (ret) 582 return ret; 583 584 ret = dmaenginem_async_device_register(dma_dev); 585 if (ret < 0) 586 dev_err(dev, "failed to register device!\n"); 587 588 return ret; 589} 590 591static const struct pci_device_id hisi_dma_pci_tbl[] = { 592 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) }, 593 { 0, } 594}; 595 596static struct pci_driver hisi_dma_pci_driver = { 597 .name = "hisi_dma", 598 .id_table = hisi_dma_pci_tbl, 599 .probe = hisi_dma_probe, 600}; 601 602module_pci_driver(hisi_dma_pci_driver); 603 604MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 605MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>"); 606MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver"); 607MODULE_LICENSE("GPL v2"); 608MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl); 609