| /kernel/linux/linux-5.10/drivers/dma/ioat/ |
| H A D | dma.c | 72 static void ioat_eh(struct ioatdma_chan *ioat_chan);
74 static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr) in ioat_print_chanerrs() argument
80 dev_err(to_dev(ioat_chan), "Err(%d): %s\n", in ioat_print_chanerrs()
94 struct ioatdma_chan *ioat_chan; in ioat_dma_do_interrupt() local
111 ioat_chan = ioat_chan_by_index(instance, bit); in ioat_dma_do_interrupt()
112 if (test_bit(IOAT_RUN, &ioat_chan->state)) in ioat_dma_do_interrupt()
113 tasklet_schedule(&ioat_chan->cleanup_task); in ioat_dma_do_interrupt()
127 struct ioatdma_chan *ioat_chan = data; in ioat_dma_do_interrupt_msix() local
129 if (test_bit(IOAT_RUN, &ioat_chan->state)) in ioat_dma_do_interrupt_msix()
130 tasklet_schedule(&ioat_chan in ioat_dma_do_interrupt_msix()
135 ioat_stop(struct ioatdma_chan *ioat_chan) ioat_stop() argument
171 __ioat_issue_pending(struct ioatdma_chan *ioat_chan) __ioat_issue_pending() argument
185 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_issue_pending() local
201 ioat_update_pending(struct ioatdma_chan *ioat_chan) ioat_update_pending() argument
207 __ioat_start_null_desc(struct ioatdma_chan *ioat_chan) __ioat_start_null_desc() argument
241 ioat_start_null_desc(struct ioatdma_chan *ioat_chan) ioat_start_null_desc() argument
249 __ioat_restart_chan(struct ioatdma_chan *ioat_chan) __ioat_restart_chan() argument
271 ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo) ioat_quiesce() argument
292 ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo) ioat_reset_sync() argument
313 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); global() variable
330 ioat_update_pending(ioat_chan); global() variable
341 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_alloc_ring_ent() local
374 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_alloc_ring() local
529 ioat_get_current_completion(struct ioatdma_chan *ioat_chan) ioat_get_current_completion() argument
543 ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, u64 *phys_complete) ioat_cleanup_preamble() argument
557 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) desc_get_errstat() argument
591 __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) __cleanup() argument
671 ioat_cleanup(struct ioatdma_chan *ioat_chan) ioat_cleanup() argument
695 struct ioatdma_chan *ioat_chan = from_tasklet(ioat_chan, t, cleanup_task); ioat_cleanup_event() local
703 ioat_restart_channel(struct ioatdma_chan *ioat_chan) ioat_restart_channel() argument
721 ioat_abort_descs(struct ioatdma_chan *ioat_chan) ioat_abort_descs() argument
774 ioat_eh(struct ioatdma_chan *ioat_chan) ioat_eh() argument
874 check_active(struct ioatdma_chan *ioat_chan) check_active() argument
885 ioat_reboot_chan(struct ioatdma_chan *ioat_chan) ioat_reboot_chan() argument
904 struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer); ioat_timer_event() local
990 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_tx_status() local
1002 ioat_reset_hw(struct ioatdma_chan *ioat_chan) ioat_reset_hw() argument
[all...] |
| H A D | dma.h | 23 #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
24 #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
220 __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw, in __dump_desc_dbg() argument
223 struct device *dev = to_dev(ioat_chan); in __dump_desc_dbg()
241 static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan) in ioat_chansts() argument
243 return readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET); in ioat_chansts()
251 static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan) in ioat_chanerr() argument
253 return readl(ioat_chan in ioat_chanerr()
256 ioat_suspend(struct ioatdma_chan *ioat_chan) ioat_suspend() argument
264 ioat_reset(struct ioatdma_chan *ioat_chan) ioat_reset() argument
272 ioat_reset_pending(struct ioatdma_chan *ioat_chan) ioat_reset_pending() argument
308 ioat_ring_size(struct ioatdma_chan *ioat_chan) ioat_ring_size() argument
314 ioat_ring_active(struct ioatdma_chan *ioat_chan) ioat_ring_active() argument
321 ioat_ring_pending(struct ioatdma_chan *ioat_chan) ioat_ring_pending() argument
327 ioat_ring_space(struct ioatdma_chan *ioat_chan) ioat_ring_space() argument
333 ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len) ioat_xferlen_to_descs() argument
342 ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx) ioat_get_ring_ent() argument
348 ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr) ioat_set_chainaddr() argument
[all...] |
| H A D | init.c | 121 struct ioatdma_chan *ioat_chan, int idx);
405 struct ioatdma_chan *ioat_chan; in ioat_dma_setup_interrupts() local
434 ioat_chan = ioat_chan_by_index(ioat_dma, i); in ioat_dma_setup_interrupts()
437 "ioat-msix", ioat_chan); in ioat_dma_setup_interrupts()
441 ioat_chan = ioat_chan_by_index(ioat_dma, j); in ioat_dma_setup_interrupts()
442 devm_free_irq(dev, msix->vector, ioat_chan); in ioat_dma_setup_interrupts()
567 struct ioatdma_chan *ioat_chan; in ioat_enumerate_channels() local
588 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); in ioat_enumerate_channels()
589 if (!ioat_chan) in ioat_enumerate_channels()
609 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_free_chan_resources() local
680 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_alloc_chan_resources() local
766 ioat_init_channel(struct ioatdma_device *ioat_dma, struct ioatdma_chan *ioat_chan, int idx) ioat_init_channel() argument
1065 struct ioatdma_chan *ioat_chan; ioat_intr_quirk() local
1093 struct ioatdma_chan *ioat_chan; ioat3_dma_probe() local
1214 struct ioatdma_chan *ioat_chan; ioat_shutdown() local
1245 struct ioatdma_chan *ioat_chan; ioat_resume() local
[all...] |
| H A D | sysfs.c | 45 struct ioatdma_chan *ioat_chan; in ioat_attr_show() local
48 ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); in ioat_attr_show()
52 return entry->show(&ioat_chan->dma_chan, page); in ioat_attr_show()
60 struct ioatdma_chan *ioat_chan; in ioat_attr_store() local
63 ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); in ioat_attr_store()
67 return entry->store(&ioat_chan->dma_chan, page, count); in ioat_attr_store()
81 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); in ioat_kobject_add() local
85 err = kobject_init_and_add(&ioat_chan->kobj, type, in ioat_kobject_add()
88 dev_warn(to_dev(ioat_chan), in ioat_kobject_add()
90 kobject_put(&ioat_chan in ioat_kobject_add()
102 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_kobject_del() local
113 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ring_size_show() local
121 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ring_active_show() local
130 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); intr_coalesce_show() local
139 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); intr_coalesce_store() local
[all...] |
| H A D | prep.c | 104 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); in ioat_dma_prep_memcpy_lock() local
112 if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) in ioat_dma_prep_memcpy_lock()
115 num_descs = ioat_xferlen_to_descs(ioat_chan, len); in ioat_dma_prep_memcpy_lock()
117 ioat_check_space_lock(ioat_chan, num_descs) == 0) in ioat_dma_prep_memcpy_lock()
118 idx = ioat_chan->head; in ioat_dma_prep_memcpy_lock()
123 size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log); in ioat_dma_prep_memcpy_lock()
125 desc = ioat_get_ring_ent(ioat_chan, idx + i); in ioat_dma_prep_memcpy_lock()
136 dump_desc_dbg(ioat_chan, desc); in ioat_dma_prep_memcpy_lock()
144 dump_desc_dbg(ioat_chan, desc); in ioat_dma_prep_memcpy_lock()
156 struct ioatdma_chan *ioat_chan in __ioat_prep_xor_lock() local
248 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_xor() local
261 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_xor_val() local
276 dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext) dump_pq_desc_dbg() argument
303 dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) dump_pq16_desc_dbg() argument
344 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); __ioat_prep_pq_lock() local
468 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); __ioat_prep_pq16_lock() local
578 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pq() local
623 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pq_val() local
652 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pqxor() local
679 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pqxor_val() local
707 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_prep_interrupt_lock() local
[all...] |
| /kernel/linux/linux-6.6/drivers/dma/ioat/ |
| H A D | dma.c | 72 static void ioat_eh(struct ioatdma_chan *ioat_chan);
74 static void ioat_print_chanerrs(struct ioatdma_chan *ioat_chan, u32 chanerr) in ioat_print_chanerrs() argument
80 dev_err(to_dev(ioat_chan), "Err(%d): %s\n", in ioat_print_chanerrs()
94 struct ioatdma_chan *ioat_chan; in ioat_dma_do_interrupt() local
111 ioat_chan = ioat_chan_by_index(instance, bit); in ioat_dma_do_interrupt()
112 if (test_bit(IOAT_RUN, &ioat_chan->state)) in ioat_dma_do_interrupt()
113 tasklet_schedule(&ioat_chan->cleanup_task); in ioat_dma_do_interrupt()
127 struct ioatdma_chan *ioat_chan = data; in ioat_dma_do_interrupt_msix() local
129 if (test_bit(IOAT_RUN, &ioat_chan->state)) in ioat_dma_do_interrupt_msix()
130 tasklet_schedule(&ioat_chan in ioat_dma_do_interrupt_msix()
135 ioat_stop(struct ioatdma_chan *ioat_chan) ioat_stop() argument
171 __ioat_issue_pending(struct ioatdma_chan *ioat_chan) __ioat_issue_pending() argument
185 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_issue_pending() local
201 ioat_update_pending(struct ioatdma_chan *ioat_chan) ioat_update_pending() argument
207 __ioat_start_null_desc(struct ioatdma_chan *ioat_chan) __ioat_start_null_desc() argument
241 ioat_start_null_desc(struct ioatdma_chan *ioat_chan) ioat_start_null_desc() argument
249 __ioat_restart_chan(struct ioatdma_chan *ioat_chan) __ioat_restart_chan() argument
271 ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo) ioat_quiesce() argument
292 ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo) ioat_reset_sync() argument
313 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); global() variable
330 ioat_update_pending(ioat_chan); global() variable
341 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_alloc_ring_ent() local
374 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_alloc_ring() local
529 ioat_get_current_completion(struct ioatdma_chan *ioat_chan) ioat_get_current_completion() argument
543 ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, u64 *phys_complete) ioat_cleanup_preamble() argument
557 desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) desc_get_errstat() argument
591 __ioat_cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) __ioat_cleanup() argument
671 ioat_cleanup(struct ioatdma_chan *ioat_chan) ioat_cleanup() argument
695 struct ioatdma_chan *ioat_chan = from_tasklet(ioat_chan, t, cleanup_task); ioat_cleanup_event() local
703 ioat_restart_channel(struct ioatdma_chan *ioat_chan) ioat_restart_channel() argument
721 ioat_abort_descs(struct ioatdma_chan *ioat_chan) ioat_abort_descs() argument
774 ioat_eh(struct ioatdma_chan *ioat_chan) ioat_eh() argument
874 check_active(struct ioatdma_chan *ioat_chan) check_active() argument
885 ioat_reboot_chan(struct ioatdma_chan *ioat_chan) ioat_reboot_chan() argument
904 struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer); ioat_timer_event() local
990 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_tx_status() local
1002 ioat_reset_hw(struct ioatdma_chan *ioat_chan) ioat_reset_hw() argument
[all...] |
| H A D | dma.h | 23 #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
24 #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
219 __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw, in __dump_desc_dbg() argument
222 struct device *dev = to_dev(ioat_chan); in __dump_desc_dbg()
240 static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan) in ioat_chansts() argument
242 return readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET); in ioat_chansts()
250 static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan) in ioat_chanerr() argument
252 return readl(ioat_chan in ioat_chanerr()
255 ioat_suspend(struct ioatdma_chan *ioat_chan) ioat_suspend() argument
263 ioat_reset(struct ioatdma_chan *ioat_chan) ioat_reset() argument
271 ioat_reset_pending(struct ioatdma_chan *ioat_chan) ioat_reset_pending() argument
307 ioat_ring_size(struct ioatdma_chan *ioat_chan) ioat_ring_size() argument
313 ioat_ring_active(struct ioatdma_chan *ioat_chan) ioat_ring_active() argument
320 ioat_ring_pending(struct ioatdma_chan *ioat_chan) ioat_ring_pending() argument
326 ioat_ring_space(struct ioatdma_chan *ioat_chan) ioat_ring_space() argument
332 ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len) ioat_xferlen_to_descs() argument
341 ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx) ioat_get_ring_ent() argument
347 ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr) ioat_set_chainaddr() argument
[all...] |
| H A D | init.c | 120 struct ioatdma_chan *ioat_chan, int idx);
404 struct ioatdma_chan *ioat_chan; in ioat_dma_setup_interrupts() local
433 ioat_chan = ioat_chan_by_index(ioat_dma, i); in ioat_dma_setup_interrupts()
436 "ioat-msix", ioat_chan); in ioat_dma_setup_interrupts()
440 ioat_chan = ioat_chan_by_index(ioat_dma, j); in ioat_dma_setup_interrupts()
441 devm_free_irq(dev, msix->vector, ioat_chan); in ioat_dma_setup_interrupts()
566 struct ioatdma_chan *ioat_chan; in ioat_enumerate_channels() local
588 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); in ioat_enumerate_channels()
589 if (!ioat_chan) in ioat_enumerate_channels()
609 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_free_chan_resources() local
680 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_alloc_chan_resources() local
766 ioat_init_channel(struct ioatdma_device *ioat_dma, struct ioatdma_chan *ioat_chan, int idx) ioat_init_channel() argument
1065 struct ioatdma_chan *ioat_chan; ioat_intr_quirk() local
1093 struct ioatdma_chan *ioat_chan; ioat3_dma_probe() local
1214 struct ioatdma_chan *ioat_chan; ioat_shutdown() local
1245 struct ioatdma_chan *ioat_chan; ioat_resume() local
[all...] |
| H A D | sysfs.c | 45 struct ioatdma_chan *ioat_chan; in ioat_attr_show() local
48 ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); in ioat_attr_show()
52 return entry->show(&ioat_chan->dma_chan, page); in ioat_attr_show()
60 struct ioatdma_chan *ioat_chan; in ioat_attr_store() local
63 ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); in ioat_attr_store()
67 return entry->store(&ioat_chan->dma_chan, page, count); in ioat_attr_store()
81 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); in ioat_kobject_add() local
85 err = kobject_init_and_add(&ioat_chan->kobj, type, in ioat_kobject_add()
88 dev_warn(to_dev(ioat_chan), in ioat_kobject_add()
90 kobject_put(&ioat_chan in ioat_kobject_add()
102 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_kobject_del() local
113 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ring_size_show() local
121 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ring_active_show() local
130 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); intr_coalesce_show() local
139 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); intr_coalesce_store() local
[all...] |
| H A D | prep.c | 104 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); in ioat_dma_prep_memcpy_lock() local
112 if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state)) in ioat_dma_prep_memcpy_lock()
115 num_descs = ioat_xferlen_to_descs(ioat_chan, len); in ioat_dma_prep_memcpy_lock()
117 ioat_check_space_lock(ioat_chan, num_descs) == 0) in ioat_dma_prep_memcpy_lock()
118 idx = ioat_chan->head; in ioat_dma_prep_memcpy_lock()
123 size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log); in ioat_dma_prep_memcpy_lock()
125 desc = ioat_get_ring_ent(ioat_chan, idx + i); in ioat_dma_prep_memcpy_lock()
136 dump_desc_dbg(ioat_chan, desc); in ioat_dma_prep_memcpy_lock()
144 dump_desc_dbg(ioat_chan, desc); in ioat_dma_prep_memcpy_lock()
156 struct ioatdma_chan *ioat_chan in __ioat_prep_xor_lock() local
248 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_xor() local
261 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_xor_val() local
276 dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext) dump_pq_desc_dbg() argument
303 dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) dump_pq16_desc_dbg() argument
344 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); __ioat_prep_pq_lock() local
468 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); __ioat_prep_pq16_lock() local
578 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pq() local
623 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pq_val() local
652 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pqxor() local
679 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan); ioat_prep_pqxor_val() local
707 struct ioatdma_chan *ioat_chan = to_ioat_chan(c); ioat_prep_interrupt_lock() local
[all...] |