Lines Matching defs:ecc
228 struct edma_cc *ecc;
309 static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
311 return (unsigned int)__raw_readl(ecc->base + offset);
314 static inline void edma_write(struct edma_cc *ecc, int offset, int val)
316 __raw_writel(val, ecc->base + offset);
319 static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
322 unsigned val = edma_read(ecc, offset);
326 edma_write(ecc, offset, val);
329 static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
331 unsigned val = edma_read(ecc, offset);
334 edma_write(ecc, offset, val);
337 static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
339 unsigned val = edma_read(ecc, offset);
342 edma_write(ecc, offset, val);
345 static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
348 return edma_read(ecc, offset + (i << 2));
351 static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
354 edma_write(ecc, offset + (i << 2), val);
357 static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
360 edma_modify(ecc, offset + (i << 2), and, or);
363 static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
366 edma_or(ecc, offset + (i << 2), or);
369 static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
372 edma_or(ecc, offset + ((i * 2 + j) << 2), or);
375 static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
378 edma_write(ecc, offset + ((i * 2 + j) << 2), val);
381 static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
383 return edma_read(ecc, EDMA_SHADOW0 + offset);
386 static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
389 return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
392 static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
395 edma_write(ecc, EDMA_SHADOW0 + offset, val);
398 static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
401 edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
404 static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
407 return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
410 static inline void edma_param_write(struct edma_cc *ecc, int offset,
413 edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
416 static inline void edma_param_modify(struct edma_cc *ecc, int offset,
419 edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
422 static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
425 edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
428 static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
431 edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
434 static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
439 edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
444 struct edma_cc *ecc = echan->ecc;
447 if (ecc->chmap_exist) {
449 edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
455 struct edma_cc *ecc = echan->ecc;
461 edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
462 edma_shadow0_write_array(ecc, SH_IESR, idx, ch_bit);
464 edma_shadow0_write_array(ecc, SH_IECR, idx, ch_bit);
471 static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
475 if (slot >= ecc->num_slots)
477 memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
480 static int edma_read_slot(struct edma_cc *ecc, unsigned slot,
484 if (slot >= ecc->num_slots)
486 memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
493 * @ecc: pointer to edma_cc struct
506 static int edma_alloc_slot(struct edma_cc *ecc, int slot)
511 if (ecc->chmap_exist && slot < ecc->num_channels)
516 if (ecc->chmap_exist)
519 slot = ecc->num_channels;
521 slot = find_next_zero_bit(ecc->slot_inuse,
522 ecc->num_slots,
524 if (slot == ecc->num_slots)
526 if (!test_and_set_bit(slot, ecc->slot_inuse))
529 } else if (slot >= ecc->num_slots) {
531 } else if (test_and_set_bit(slot, ecc->slot_inuse)) {
535 edma_write_slot(ecc, slot, &dummy_paramset);
537 return EDMA_CTLR_CHAN(ecc->id, slot);
540 static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
543 if (slot >= ecc->num_slots)
546 edma_write_slot(ecc, slot, &dummy_paramset);
547 clear_bit(slot, ecc->slot_inuse);
552 * @ecc: pointer to edma_cc struct
558 static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
561 dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
565 if (from >= ecc->num_slots || to >= ecc->num_slots)
568 edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
574 * @ecc: pointer to edma_cc struct
580 static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
589 return edma_read(ecc, offs);
600 struct edma_cc *ecc = echan->ecc;
607 dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
608 edma_shadow0_read_array(ecc, SH_ESR, idx));
609 edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
612 dev_dbg(ecc->dev, "ER%d %08x\n", idx,
613 edma_shadow0_read_array(ecc, SH_ER, idx));
615 edma_write_array(ecc, EDMA_ECR, idx, ch_bit);
616 edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
618 edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
619 edma_shadow0_write_array(ecc, SH_EESR, idx, ch_bit);
620 dev_dbg(ecc->dev, "EER%d %08x\n", idx,
621 edma_shadow0_read_array(ecc, SH_EER, idx));
627 struct edma_cc *ecc = echan->ecc;
632 edma_shadow0_write_array(ecc, SH_EECR, idx, ch_bit);
633 edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
634 edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
635 edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
638 edma_shadow0_write_array(ecc, SH_ICR, idx, ch_bit);
640 dev_dbg(ecc->dev, "EER%d %08x\n", idx,
641 edma_shadow0_read_array(ecc, SH_EER, idx));
656 edma_shadow0_write_array(echan->ecc, SH_EECR,
666 edma_shadow0_write_array(echan->ecc, SH_EESR,
673 struct edma_cc *ecc = echan->ecc;
678 edma_shadow0_write_array(ecc, SH_ESR, idx, ch_bit);
680 dev_dbg(ecc->dev, "ESR%d %08x\n", idx,
681 edma_shadow0_read_array(ecc, SH_ESR, idx));
686 struct edma_cc *ecc = echan->ecc;
691 dev_dbg(ecc->dev, "EMR%d %08x\n", idx,
692 edma_read_array(ecc, EDMA_EMR, idx));
693 edma_shadow0_write_array(ecc, SH_ECR, idx, ch_bit);
695 edma_write_array(ecc, EDMA_EMCR, idx, ch_bit);
697 edma_shadow0_write_array(ecc, SH_SECR, idx, ch_bit);
698 edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
705 struct edma_cc *ecc = echan->ecc;
711 eventq_no = ecc->default_queue;
712 if (eventq_no >= ecc->num_tc)
716 edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
723 struct edma_cc *ecc = echan->ecc;
726 if (!test_bit(echan->ch_num, ecc->channels_mask)) {
727 dev_err(ecc->dev, "Channel%d is reserved, can not be used!\n",
733 edma_or_array2(ecc, EDMA_DRAE, 0, EDMA_REG_ARRAY_INDEX(channel),
777 struct edma_cc *ecc = echan->ecc;
802 edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
827 edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
839 edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
841 edma_link(ecc, echan->slot[nslots - 1],
842 echan->ecc->dummy_slot);
1116 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1246 echan->slot[1] = edma_alloc_slot(echan->ecc,
1438 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
1550 struct edma_cc *ecc = data;
1556 ctlr = ecc->id;
1560 dev_vdbg(ecc->dev, "dma_irq_handler\n");
1562 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
1564 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
1567 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
1570 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
1584 edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
1585 edma_completion_handler(&ecc->slave_chans[channel]);
1589 edma_shadow0_write(ecc, SH_IEVAL, 1);
1595 struct edma_cc *ecc = echan->ecc;
1605 err = edma_read_slot(ecc, echan->slot[0], &p);
1636 static inline bool edma_error_pending(struct edma_cc *ecc)
1638 if (edma_read_array(ecc, EDMA_EMR, 0) ||
1639 edma_read_array(ecc, EDMA_EMR, 1) ||
1640 edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
1649 struct edma_cc *ecc = data;
1655 ctlr = ecc->id;
1659 dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
1661 if (!edma_error_pending(ecc)) {
1667 dev_err(ecc->dev, "%s: Error interrupt without error event!\n",
1669 edma_write(ecc, EDMA_EEVAL, 1);
1678 val = edma_read_array(ecc, EDMA_EMR, j);
1682 dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
1689 edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
1691 edma_shadow0_write_array(ecc, SH_SECR, j,
1693 edma_error_handler(&ecc->slave_chans[k]);
1697 val = edma_read(ecc, EDMA_QEMR);
1699 dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
1701 edma_write(ecc, EDMA_QEMCR, val);
1702 edma_shadow0_write(ecc, SH_QSECR, val);
1705 val = edma_read(ecc, EDMA_CCERR);
1707 dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
1709 edma_write(ecc, EDMA_CCERRCLR, val);
1712 if (!edma_error_pending(ecc))
1718 edma_write(ecc, EDMA_EEVAL, 1);
1726 struct edma_cc *ecc = echan->ecc;
1727 struct device *dev = ecc->dev;
1733 } else if (ecc->tc_list) {
1735 echan->tc = &ecc->tc_list[ecc->info->default_queue];
1743 echan->slot[0] = edma_alloc_slot(ecc, echan->ch_num);
1770 struct device *dev = echan->ecc->dev;
1781 edma_free_slot(echan->ecc, echan->slot[i]);
1787 edma_set_chmap(echan, echan->ecc->dummy_slot);
1840 pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1856 while (edma_shadow0_read_array(echan->ecc, event_reg, idx) & ch_bit) {
1857 pos = edma_get_position(echan->ecc, echan->slot[0], dst);
1980 static void edma_dma_init(struct edma_cc *ecc, bool legacy_mode)
1982 struct dma_device *s_ddev = &ecc->dma_slave;
1984 s32 *memcpy_channels = ecc->info->memcpy_channels;
1990 if (ecc->legacy_mode && !memcpy_channels) {
1991 dev_warn(ecc->dev,
2019 s_ddev->dev = ecc->dev;
2023 m_ddev = devm_kzalloc(ecc->dev, sizeof(*m_ddev), GFP_KERNEL);
2025 dev_warn(ecc->dev, "memcpy is disabled due to OoM\n");
2029 ecc->dma_memcpy = m_ddev;
2052 m_ddev->dev = ecc->dev;
2054 } else if (!ecc->legacy_mode) {
2055 dev_info(ecc->dev, "memcpy is disabled\n");
2059 for (i = 0; i < ecc->num_channels; i++) {
2060 struct edma_chan *echan = &ecc->slave_chans[i];
2061 echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
2062 echan->ecc = ecc;
2077 struct edma_cc *ecc)
2084 cccfg = edma_read(ecc, EDMA_CCCFG);
2087 ecc->num_region = BIT(value);
2090 ecc->num_channels = BIT(value + 1);
2093 ecc->num_qchannels = value * 2;
2096 ecc->num_slots = BIT(value + 4);
2099 ecc->num_tc = value + 1;
2101 ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
2104 dev_dbg(dev, "num_region: %u\n", ecc->num_region);
2105 dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
2106 dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels);
2107 dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
2108 dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
2109 dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
2125 queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
2130 for (i = 0; i < ecc->num_tc; i++) {
2290 struct edma_cc *ecc = ofdma->of_dma_data;
2295 if (!ecc || dma_spec->args_count < 1)
2298 for (i = 0; i < ecc->num_channels; i++) {
2299 echan = &ecc->slave_chans[i];
2309 if (echan->ecc->legacy_mode && dma_spec->args_count == 1)
2312 if (!echan->ecc->legacy_mode && dma_spec->args_count == 2 &&
2313 dma_spec->args[1] < echan->ecc->num_tc) {
2314 echan->tc = &echan->ecc->tc_list[dma_spec->args[1]];
2350 struct edma_cc *ecc;
2375 ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
2376 if (!ecc)
2379 ecc->dev = dev;
2380 ecc->id = pdev->id;
2381 ecc->legacy_mode = legacy_mode;
2383 if (ecc->id < 0)
2384 ecc->id = 0;
2395 ecc->base = devm_ioremap_resource(dev, mem);
2396 if (IS_ERR(ecc->base))
2397 return PTR_ERR(ecc->base);
2399 platform_set_drvdata(pdev, ecc);
2410 ret = edma_setup_from_hw(dev, info, ecc);
2415 ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
2416 sizeof(*ecc->slave_chans), GFP_KERNEL);
2418 ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
2421 ecc->channels_mask = devm_kcalloc(dev,
2422 BITS_TO_LONGS(ecc->num_channels),
2424 if (!ecc->slave_chans || !ecc->slot_inuse || !ecc->channels_mask) {
2430 bitmap_fill(ecc->channels_mask, ecc->num_channels);
2432 ecc->default_queue = info->default_queue;
2439 bitmap_set(ecc->slot_inuse, reserved[i][0],
2447 bitmap_clear(ecc->channels_mask, reserved[i][0],
2452 for (i = 0; i < ecc->num_slots; i++) {
2454 if (!test_bit(i, ecc->slot_inuse))
2455 edma_write_slot(ecc, i, &dummy_paramset);
2471 ecc);
2476 ecc->ccint = irq;
2492 ecc);
2497 ecc->ccerrint = irq;
2500 ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
2501 if (ecc->dummy_slot < 0) {
2503 ret = ecc->dummy_slot;
2509 if (!ecc->legacy_mode) {
2514 ecc->tc_list = devm_kcalloc(dev, ecc->num_tc,
2515 sizeof(*ecc->tc_list), GFP_KERNEL);
2516 if (!ecc->tc_list) {
2524 if (ret || i == ecc->num_tc)
2527 ecc->tc_list[i].node = tc_args.np;
2528 ecc->tc_list[i].id = i;
2537 array_max = DIV_ROUND_UP(ecc->num_channels, BITS_PER_TYPE(u32));
2540 (u32 *)ecc->channels_mask,
2551 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2554 edma_write_array2(ecc, EDMA_DRAE, 0, 0, 0x0);
2555 edma_write_array2(ecc, EDMA_DRAE, 0, 1, 0x0);
2556 edma_write_array(ecc, EDMA_QRAE, 0, 0x0);
2558 ecc->info = info;
2561 edma_dma_init(ecc, legacy_mode);
2563 for (i = 0; i < ecc->num_channels; i++) {
2565 if (!test_bit(i, ecc->channels_mask))
2569 edma_assign_channel_eventq(&ecc->slave_chans[i],
2572 edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot);
2575 ecc->dma_slave.filter.map = info->slave_map;
2576 ecc->dma_slave.filter.mapcnt = info->slavecnt;
2577 ecc->dma_slave.filter.fn = edma_filter_fn;
2579 ret = dma_async_device_register(&ecc->dma_slave);
2585 if (ecc->dma_memcpy) {
2586 ret = dma_async_device_register(ecc->dma_memcpy);
2590 dma_async_device_unregister(&ecc->dma_slave);
2596 of_dma_controller_register(node, of_edma_xlate, ecc);
2603 edma_free_slot(ecc, ecc->dummy_slot);
2624 struct edma_cc *ecc = dev_get_drvdata(dev);
2626 devm_free_irq(dev, ecc->ccint, ecc);
2627 devm_free_irq(dev, ecc->ccerrint, ecc);
2629 edma_cleanupp_vchan(&ecc->dma_slave);
2633 dma_async_device_unregister(&ecc->dma_slave);
2634 if (ecc->dma_memcpy)
2635 dma_async_device_unregister(ecc->dma_memcpy);
2636 edma_free_slot(ecc, ecc->dummy_slot);
2646 struct edma_cc *ecc = dev_get_drvdata(dev);
2647 struct edma_chan *echan = ecc->slave_chans;
2650 for (i = 0; i < ecc->num_channels; i++) {
2660 struct edma_cc *ecc = dev_get_drvdata(dev);
2661 struct edma_chan *echan = ecc->slave_chans;
2666 edma_write_slot(ecc, ecc->dummy_slot, &dummy_paramset);
2668 queue_priority_mapping = ecc->info->queue_priority_mapping;
2672 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2675 for (i = 0; i < ecc->num_channels; i++) {
2678 edma_or_array2(ecc, EDMA_DRAE, 0,