1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * offload engine driver for the Intel Xscale series of i/o processors 4 * Copyright © 2006, Intel Corporation. 5 */ 6 7/* 8 * This driver supports the asynchrounous DMA copy and RAID engines available 9 * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 10 */ 11 12#include <linux/init.h> 13#include <linux/module.h> 14#include <linux/delay.h> 15#include <linux/dma-mapping.h> 16#include <linux/spinlock.h> 17#include <linux/interrupt.h> 18#include <linux/platform_device.h> 19#include <linux/prefetch.h> 20#include <linux/memory.h> 21#include <linux/ioport.h> 22#include <linux/raid/pq.h> 23#include <linux/slab.h> 24 25#include "iop-adma.h" 26#include "dmaengine.h" 27 28#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) 29#define to_iop_adma_device(dev) \ 30 container_of(dev, struct iop_adma_device, common) 31#define tx_to_iop_adma_slot(tx) \ 32 container_of(tx, struct iop_adma_desc_slot, async_tx) 33 34/** 35 * iop_adma_free_slots - flags descriptor slots for reuse 36 * @slot: Slot to free 37 * Caller must hold &iop_chan->lock while calling this function 38 */ 39static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) 40{ 41 int stride = slot->slots_per_op; 42 43 while (stride--) { 44 slot->slots_per_op = 0; 45 slot = list_entry(slot->slot_node.next, 46 struct iop_adma_desc_slot, 47 slot_node); 48 } 49} 50 51static dma_cookie_t 52iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 53 struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 54{ 55 struct dma_async_tx_descriptor *tx = &desc->async_tx; 56 57 BUG_ON(tx->cookie < 0); 58 if (tx->cookie > 0) { 59 cookie = tx->cookie; 60 tx->cookie = 0; 61 62 /* call the callback (must not sleep or submit new 63 * operations to this channel) 64 */ 65 dmaengine_desc_get_callback_invoke(tx, NULL); 66 67 dma_descriptor_unmap(tx); 68 if (desc->group_head) 69 desc->group_head = NULL; 70 } 71 72 /* run dependent operations */ 73 dma_run_dependencies(tx); 74 75 return cookie; 76} 77 78static int 79iop_adma_clean_slot(struct iop_adma_desc_slot *desc, 80 struct iop_adma_chan *iop_chan) 81{ 82 /* the client is allowed to attach dependent operations 83 * until 'ack' is set 84 */ 85 if (!async_tx_test_ack(&desc->async_tx)) 86 return 0; 87 88 /* leave the last descriptor in the chain 89 * so we can append to it 90 */ 91 if (desc->chain_node.next == &iop_chan->chain) 92 return 1; 93 94 dev_dbg(iop_chan->device->common.dev, 95 "\tfree slot: %d slots_per_op: %d\n", 96 desc->idx, desc->slots_per_op); 97 98 list_del(&desc->chain_node); 99 iop_adma_free_slots(desc); 100 101 return 0; 102} 103 104static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 105{ 106 struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; 107 dma_cookie_t cookie = 0; 108 u32 current_desc = iop_chan_get_current_descriptor(iop_chan); 109 int busy = iop_chan_is_busy(iop_chan); 110 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 111 112 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 113 /* free completed slots from the chain starting with 114 * the oldest descriptor 115 */ 116 list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 117 chain_node) { 118 pr_debug("\tcookie: %d slot: %d busy: %d " 119 "this_desc: %pad next_desc: %#llx ack: %d\n", 120 iter->async_tx.cookie, iter->idx, busy, 121 &iter->async_tx.phys, (u64)iop_desc_get_next_desc(iter), 122 async_tx_test_ack(&iter->async_tx)); 123 prefetch(_iter); 124 prefetch(&_iter->async_tx); 125 126 /* do not advance past the current descriptor loaded into the 127 * hardware channel, subsequent descriptors are either in 128 * process or have not been submitted 129 */ 130 if (seen_current) 131 break; 132 133 /* stop the search if we reach the current descriptor and the 134 * channel is busy, or if it appears that the current descriptor 135 * needs to be re-read (i.e. has been appended to) 136 */ 137 if (iter->async_tx.phys == current_desc) { 138 BUG_ON(seen_current++); 139 if (busy || iop_desc_get_next_desc(iter)) 140 break; 141 } 142 143 /* detect the start of a group transaction */ 144 if (!slot_cnt && !slots_per_op) { 145 slot_cnt = iter->slot_cnt; 146 slots_per_op = iter->slots_per_op; 147 if (slot_cnt <= slots_per_op) { 148 slot_cnt = 0; 149 slots_per_op = 0; 150 } 151 } 152 153 if (slot_cnt) { 154 pr_debug("\tgroup++\n"); 155 if (!grp_start) 156 grp_start = iter; 157 slot_cnt -= slots_per_op; 158 } 159 160 /* all the members of a group are complete */ 161 if (slots_per_op != 0 && slot_cnt == 0) { 162 struct iop_adma_desc_slot *grp_iter, *_grp_iter; 163 int end_of_chain = 0; 164 pr_debug("\tgroup end\n"); 165 166 /* collect the total results */ 167 if (grp_start->xor_check_result) { 168 u32 zero_sum_result = 0; 169 slot_cnt = grp_start->slot_cnt; 170 grp_iter = grp_start; 171 172 list_for_each_entry_from(grp_iter, 173 &iop_chan->chain, chain_node) { 174 zero_sum_result |= 175 iop_desc_get_zero_result(grp_iter); 176 pr_debug("\titer%d result: %d\n", 177 grp_iter->idx, zero_sum_result); 178 slot_cnt -= slots_per_op; 179 if (slot_cnt == 0) 180 break; 181 } 182 pr_debug("\tgrp_start->xor_check_result: %p\n", 183 grp_start->xor_check_result); 184 *grp_start->xor_check_result = zero_sum_result; 185 } 186 187 /* clean up the group */ 188 slot_cnt = grp_start->slot_cnt; 189 grp_iter = grp_start; 190 list_for_each_entry_safe_from(grp_iter, _grp_iter, 191 &iop_chan->chain, chain_node) { 192 cookie = iop_adma_run_tx_complete_actions( 193 grp_iter, iop_chan, cookie); 194 195 slot_cnt -= slots_per_op; 196 end_of_chain = iop_adma_clean_slot(grp_iter, 197 iop_chan); 198 199 if (slot_cnt == 0 || end_of_chain) 200 break; 201 } 202 203 /* the group should be complete at this point */ 204 BUG_ON(slot_cnt); 205 206 slots_per_op = 0; 207 grp_start = NULL; 208 if (end_of_chain) 209 break; 210 else 211 continue; 212 } else if (slots_per_op) /* wait for group completion */ 213 continue; 214 215 /* write back zero sum results (single descriptor case) */ 216 if (iter->xor_check_result && iter->async_tx.cookie) 217 *iter->xor_check_result = 218 iop_desc_get_zero_result(iter); 219 220 cookie = iop_adma_run_tx_complete_actions( 221 iter, iop_chan, cookie); 222 223 if (iop_adma_clean_slot(iter, iop_chan)) 224 break; 225 } 226 227 if (cookie > 0) { 228 iop_chan->common.completed_cookie = cookie; 229 pr_debug("\tcompleted cookie %d\n", cookie); 230 } 231} 232 233static void 234iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) 235{ 236 spin_lock_bh(&iop_chan->lock); 237 __iop_adma_slot_cleanup(iop_chan); 238 spin_unlock_bh(&iop_chan->lock); 239} 240 241static void iop_adma_tasklet(struct tasklet_struct *t) 242{ 243 struct iop_adma_chan *iop_chan = from_tasklet(iop_chan, t, 244 irq_tasklet); 245 246 /* lockdep will flag depedency submissions as potentially 247 * recursive locking, this is not the case as a dependency 248 * submission will never recurse a channels submit routine. 249 * There are checks in async_tx.c to prevent this. 250 */ 251 spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); 252 __iop_adma_slot_cleanup(iop_chan); 253 spin_unlock(&iop_chan->lock); 254} 255 256static struct iop_adma_desc_slot * 257iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, 258 int slots_per_op) 259{ 260 struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; 261 LIST_HEAD(chain); 262 int slots_found, retry = 0; 263 264 /* start search from the last allocated descrtiptor 265 * if a contiguous allocation can not be found start searching 266 * from the beginning of the list 267 */ 268retry: 269 slots_found = 0; 270 if (retry == 0) 271 iter = iop_chan->last_used; 272 else 273 iter = list_entry(&iop_chan->all_slots, 274 struct iop_adma_desc_slot, 275 slot_node); 276 277 list_for_each_entry_safe_continue( 278 iter, _iter, &iop_chan->all_slots, slot_node) { 279 prefetch(_iter); 280 prefetch(&_iter->async_tx); 281 if (iter->slots_per_op) { 282 /* give up after finding the first busy slot 283 * on the second pass through the list 284 */ 285 if (retry) 286 break; 287 288 slots_found = 0; 289 continue; 290 } 291 292 /* start the allocation if the slot is correctly aligned */ 293 if (!slots_found++) { 294 if (iop_desc_is_aligned(iter, slots_per_op)) 295 alloc_start = iter; 296 else { 297 slots_found = 0; 298 continue; 299 } 300 } 301 302 if (slots_found == num_slots) { 303 struct iop_adma_desc_slot *alloc_tail = NULL; 304 struct iop_adma_desc_slot *last_used = NULL; 305 iter = alloc_start; 306 while (num_slots) { 307 int i; 308 dev_dbg(iop_chan->device->common.dev, 309 "allocated slot: %d " 310 "(desc %p phys: %#llx) slots_per_op %d\n", 311 iter->idx, iter->hw_desc, 312 (u64)iter->async_tx.phys, slots_per_op); 313 314 /* pre-ack all but the last descriptor */ 315 if (num_slots != slots_per_op) 316 async_tx_ack(&iter->async_tx); 317 318 list_add_tail(&iter->chain_node, &chain); 319 alloc_tail = iter; 320 iter->async_tx.cookie = 0; 321 iter->slot_cnt = num_slots; 322 iter->xor_check_result = NULL; 323 for (i = 0; i < slots_per_op; i++) { 324 iter->slots_per_op = slots_per_op - i; 325 last_used = iter; 326 iter = list_entry(iter->slot_node.next, 327 struct iop_adma_desc_slot, 328 slot_node); 329 } 330 num_slots -= slots_per_op; 331 } 332 alloc_tail->group_head = alloc_start; 333 alloc_tail->async_tx.cookie = -EBUSY; 334 list_splice(&chain, &alloc_tail->tx_list); 335 iop_chan->last_used = last_used; 336 iop_desc_clear_next_desc(alloc_start); 337 iop_desc_clear_next_desc(alloc_tail); 338 return alloc_tail; 339 } 340 } 341 if (!retry++) 342 goto retry; 343 344 /* perform direct reclaim if the allocation fails */ 345 __iop_adma_slot_cleanup(iop_chan); 346 347 return NULL; 348} 349 350static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) 351{ 352 dev_dbg(iop_chan->device->common.dev, "pending: %d\n", 353 iop_chan->pending); 354 355 if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { 356 iop_chan->pending = 0; 357 iop_chan_append(iop_chan); 358 } 359} 360 361static dma_cookie_t 362iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) 363{ 364 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); 365 struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); 366 struct iop_adma_desc_slot *grp_start, *old_chain_tail; 367 int slot_cnt; 368 dma_cookie_t cookie; 369 dma_addr_t next_dma; 370 371 grp_start = sw_desc->group_head; 372 slot_cnt = grp_start->slot_cnt; 373 374 spin_lock_bh(&iop_chan->lock); 375 cookie = dma_cookie_assign(tx); 376 377 old_chain_tail = list_entry(iop_chan->chain.prev, 378 struct iop_adma_desc_slot, chain_node); 379 list_splice_init(&sw_desc->tx_list, 380 &old_chain_tail->chain_node); 381 382 /* fix up the hardware chain */ 383 next_dma = grp_start->async_tx.phys; 384 iop_desc_set_next_desc(old_chain_tail, next_dma); 385 BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ 386 387 /* check for pre-chained descriptors */ 388 iop_paranoia(iop_desc_get_next_desc(sw_desc)); 389 390 /* increment the pending count by the number of slots 391 * memcpy operations have a 1:1 (slot:operation) relation 392 * other operations are heavier and will pop the threshold 393 * more often. 394 */ 395 iop_chan->pending += slot_cnt; 396 iop_adma_check_threshold(iop_chan); 397 spin_unlock_bh(&iop_chan->lock); 398 399 dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", 400 __func__, sw_desc->async_tx.cookie, sw_desc->idx); 401 402 return cookie; 403} 404 405static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); 406static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); 407 408/** 409 * iop_adma_alloc_chan_resources - returns the number of allocated descriptors 410 * @chan: allocate descriptor resources for this channel 411 * 412 * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To 413 * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be 414 * greater than 2x the number slots needed to satisfy a device->max_xor 415 * request. 416 * */ 417static int iop_adma_alloc_chan_resources(struct dma_chan *chan) 418{ 419 char *hw_desc; 420 dma_addr_t dma_desc; 421 int idx; 422 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 423 struct iop_adma_desc_slot *slot = NULL; 424 int init = iop_chan->slots_allocated ? 0 : 1; 425 struct iop_adma_platform_data *plat_data = 426 dev_get_platdata(&iop_chan->device->pdev->dev); 427 int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; 428 429 /* Allocate descriptor slots */ 430 do { 431 idx = iop_chan->slots_allocated; 432 if (idx == num_descs_in_pool) 433 break; 434 435 slot = kzalloc(sizeof(*slot), GFP_KERNEL); 436 if (!slot) { 437 printk(KERN_INFO "IOP ADMA Channel only initialized" 438 " %d descriptor slots", idx); 439 break; 440 } 441 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; 442 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; 443 444 dma_async_tx_descriptor_init(&slot->async_tx, chan); 445 slot->async_tx.tx_submit = iop_adma_tx_submit; 446 INIT_LIST_HEAD(&slot->tx_list); 447 INIT_LIST_HEAD(&slot->chain_node); 448 INIT_LIST_HEAD(&slot->slot_node); 449 dma_desc = iop_chan->device->dma_desc_pool; 450 slot->async_tx.phys = dma_desc + idx * IOP_ADMA_SLOT_SIZE; 451 slot->idx = idx; 452 453 spin_lock_bh(&iop_chan->lock); 454 iop_chan->slots_allocated++; 455 list_add_tail(&slot->slot_node, &iop_chan->all_slots); 456 spin_unlock_bh(&iop_chan->lock); 457 } while (iop_chan->slots_allocated < num_descs_in_pool); 458 459 if (idx && !iop_chan->last_used) 460 iop_chan->last_used = list_entry(iop_chan->all_slots.next, 461 struct iop_adma_desc_slot, 462 slot_node); 463 464 dev_dbg(iop_chan->device->common.dev, 465 "allocated %d descriptor slots last_used: %p\n", 466 iop_chan->slots_allocated, iop_chan->last_used); 467 468 /* initialize the channel and the chain with a null operation */ 469 if (init) { 470 if (dma_has_cap(DMA_MEMCPY, 471 iop_chan->device->common.cap_mask)) 472 iop_chan_start_null_memcpy(iop_chan); 473 else if (dma_has_cap(DMA_XOR, 474 iop_chan->device->common.cap_mask)) 475 iop_chan_start_null_xor(iop_chan); 476 else 477 BUG(); 478 } 479 480 return (idx > 0) ? idx : -ENOMEM; 481} 482 483static struct dma_async_tx_descriptor * 484iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 485{ 486 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 487 struct iop_adma_desc_slot *sw_desc, *grp_start; 488 int slot_cnt, slots_per_op; 489 490 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 491 492 spin_lock_bh(&iop_chan->lock); 493 slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); 494 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 495 if (sw_desc) { 496 grp_start = sw_desc->group_head; 497 iop_desc_init_interrupt(grp_start, iop_chan); 498 sw_desc->async_tx.flags = flags; 499 } 500 spin_unlock_bh(&iop_chan->lock); 501 502 return sw_desc ? &sw_desc->async_tx : NULL; 503} 504 505static struct dma_async_tx_descriptor * 506iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, 507 dma_addr_t dma_src, size_t len, unsigned long flags) 508{ 509 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 510 struct iop_adma_desc_slot *sw_desc, *grp_start; 511 int slot_cnt, slots_per_op; 512 513 if (unlikely(!len)) 514 return NULL; 515 BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); 516 517 dev_dbg(iop_chan->device->common.dev, "%s len: %zu\n", 518 __func__, len); 519 520 spin_lock_bh(&iop_chan->lock); 521 slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); 522 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 523 if (sw_desc) { 524 grp_start = sw_desc->group_head; 525 iop_desc_init_memcpy(grp_start, flags); 526 iop_desc_set_byte_count(grp_start, iop_chan, len); 527 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 528 iop_desc_set_memcpy_src_addr(grp_start, dma_src); 529 sw_desc->async_tx.flags = flags; 530 } 531 spin_unlock_bh(&iop_chan->lock); 532 533 return sw_desc ? &sw_desc->async_tx : NULL; 534} 535 536static struct dma_async_tx_descriptor * 537iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, 538 dma_addr_t *dma_src, unsigned int src_cnt, size_t len, 539 unsigned long flags) 540{ 541 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 542 struct iop_adma_desc_slot *sw_desc, *grp_start; 543 int slot_cnt, slots_per_op; 544 545 if (unlikely(!len)) 546 return NULL; 547 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 548 549 dev_dbg(iop_chan->device->common.dev, 550 "%s src_cnt: %d len: %zu flags: %lx\n", 551 __func__, src_cnt, len, flags); 552 553 spin_lock_bh(&iop_chan->lock); 554 slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); 555 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 556 if (sw_desc) { 557 grp_start = sw_desc->group_head; 558 iop_desc_init_xor(grp_start, src_cnt, flags); 559 iop_desc_set_byte_count(grp_start, iop_chan, len); 560 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 561 sw_desc->async_tx.flags = flags; 562 while (src_cnt--) 563 iop_desc_set_xor_src_addr(grp_start, src_cnt, 564 dma_src[src_cnt]); 565 } 566 spin_unlock_bh(&iop_chan->lock); 567 568 return sw_desc ? &sw_desc->async_tx : NULL; 569} 570 571static struct dma_async_tx_descriptor * 572iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, 573 unsigned int src_cnt, size_t len, u32 *result, 574 unsigned long flags) 575{ 576 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 577 struct iop_adma_desc_slot *sw_desc, *grp_start; 578 int slot_cnt, slots_per_op; 579 580 if (unlikely(!len)) 581 return NULL; 582 583 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", 584 __func__, src_cnt, len); 585 586 spin_lock_bh(&iop_chan->lock); 587 slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); 588 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 589 if (sw_desc) { 590 grp_start = sw_desc->group_head; 591 iop_desc_init_zero_sum(grp_start, src_cnt, flags); 592 iop_desc_set_zero_sum_byte_count(grp_start, len); 593 grp_start->xor_check_result = result; 594 pr_debug("\t%s: grp_start->xor_check_result: %p\n", 595 __func__, grp_start->xor_check_result); 596 sw_desc->async_tx.flags = flags; 597 while (src_cnt--) 598 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, 599 dma_src[src_cnt]); 600 } 601 spin_unlock_bh(&iop_chan->lock); 602 603 return sw_desc ? &sw_desc->async_tx : NULL; 604} 605 606static struct dma_async_tx_descriptor * 607iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 608 unsigned int src_cnt, const unsigned char *scf, size_t len, 609 unsigned long flags) 610{ 611 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 612 struct iop_adma_desc_slot *sw_desc, *g; 613 int slot_cnt, slots_per_op; 614 int continue_srcs; 615 616 if (unlikely(!len)) 617 return NULL; 618 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 619 620 dev_dbg(iop_chan->device->common.dev, 621 "%s src_cnt: %d len: %zu flags: %lx\n", 622 __func__, src_cnt, len, flags); 623 624 if (dmaf_p_disabled_continue(flags)) 625 continue_srcs = 1+src_cnt; 626 else if (dmaf_continue(flags)) 627 continue_srcs = 3+src_cnt; 628 else 629 continue_srcs = 0+src_cnt; 630 631 spin_lock_bh(&iop_chan->lock); 632 slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); 633 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 634 if (sw_desc) { 635 int i; 636 637 g = sw_desc->group_head; 638 iop_desc_set_byte_count(g, iop_chan, len); 639 640 /* even if P is disabled its destination address (bits 641 * [3:0]) must match Q. It is ok if P points to an 642 * invalid address, it won't be written. 643 */ 644 if (flags & DMA_PREP_PQ_DISABLE_P) 645 dst[0] = dst[1] & 0x7; 646 647 iop_desc_set_pq_addr(g, dst); 648 sw_desc->async_tx.flags = flags; 649 for (i = 0; i < src_cnt; i++) 650 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); 651 652 /* if we are continuing a previous operation factor in 653 * the old p and q values, see the comment for dma_maxpq 654 * in include/linux/dmaengine.h 655 */ 656 if (dmaf_p_disabled_continue(flags)) 657 iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 658 else if (dmaf_continue(flags)) { 659 iop_desc_set_pq_src_addr(g, i++, dst[0], 0); 660 iop_desc_set_pq_src_addr(g, i++, dst[1], 1); 661 iop_desc_set_pq_src_addr(g, i++, dst[1], 0); 662 } 663 iop_desc_init_pq(g, i, flags); 664 } 665 spin_unlock_bh(&iop_chan->lock); 666 667 return sw_desc ? &sw_desc->async_tx : NULL; 668} 669 670static struct dma_async_tx_descriptor * 671iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 672 unsigned int src_cnt, const unsigned char *scf, 673 size_t len, enum sum_check_flags *pqres, 674 unsigned long flags) 675{ 676 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 677 struct iop_adma_desc_slot *sw_desc, *g; 678 int slot_cnt, slots_per_op; 679 680 if (unlikely(!len)) 681 return NULL; 682 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); 683 684 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", 685 __func__, src_cnt, len); 686 687 spin_lock_bh(&iop_chan->lock); 688 slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); 689 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 690 if (sw_desc) { 691 /* for validate operations p and q are tagged onto the 692 * end of the source list 693 */ 694 int pq_idx = src_cnt; 695 696 g = sw_desc->group_head; 697 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); 698 iop_desc_set_pq_zero_sum_byte_count(g, len); 699 g->pq_check_result = pqres; 700 pr_debug("\t%s: g->pq_check_result: %p\n", 701 __func__, g->pq_check_result); 702 sw_desc->async_tx.flags = flags; 703 while (src_cnt--) 704 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, 705 src[src_cnt], 706 scf[src_cnt]); 707 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); 708 } 709 spin_unlock_bh(&iop_chan->lock); 710 711 return sw_desc ? &sw_desc->async_tx : NULL; 712} 713 714static void iop_adma_free_chan_resources(struct dma_chan *chan) 715{ 716 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 717 struct iop_adma_desc_slot *iter, *_iter; 718 int in_use_descs = 0; 719 720 iop_adma_slot_cleanup(iop_chan); 721 722 spin_lock_bh(&iop_chan->lock); 723 list_for_each_entry_safe(iter, _iter, &iop_chan->chain, 724 chain_node) { 725 in_use_descs++; 726 list_del(&iter->chain_node); 727 } 728 list_for_each_entry_safe_reverse( 729 iter, _iter, &iop_chan->all_slots, slot_node) { 730 list_del(&iter->slot_node); 731 kfree(iter); 732 iop_chan->slots_allocated--; 733 } 734 iop_chan->last_used = NULL; 735 736 dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", 737 __func__, iop_chan->slots_allocated); 738 spin_unlock_bh(&iop_chan->lock); 739 740 /* one is ok since we left it on there on purpose */ 741 if (in_use_descs > 1) 742 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", 743 in_use_descs - 1); 744} 745 746/** 747 * iop_adma_status - poll the status of an ADMA transaction 748 * @chan: ADMA channel handle 749 * @cookie: ADMA transaction identifier 750 * @txstate: a holder for the current state of the channel or NULL 751 */ 752static enum dma_status iop_adma_status(struct dma_chan *chan, 753 dma_cookie_t cookie, 754 struct dma_tx_state *txstate) 755{ 756 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 757 int ret; 758 759 ret = dma_cookie_status(chan, cookie, txstate); 760 if (ret == DMA_COMPLETE) 761 return ret; 762 763 iop_adma_slot_cleanup(iop_chan); 764 765 return dma_cookie_status(chan, cookie, txstate); 766} 767 768static irqreturn_t iop_adma_eot_handler(int irq, void *data) 769{ 770 struct iop_adma_chan *chan = data; 771 772 dev_dbg(chan->device->common.dev, "%s\n", __func__); 773 774 tasklet_schedule(&chan->irq_tasklet); 775 776 iop_adma_device_clear_eot_status(chan); 777 778 return IRQ_HANDLED; 779} 780 781static irqreturn_t iop_adma_eoc_handler(int irq, void *data) 782{ 783 struct iop_adma_chan *chan = data; 784 785 dev_dbg(chan->device->common.dev, "%s\n", __func__); 786 787 tasklet_schedule(&chan->irq_tasklet); 788 789 iop_adma_device_clear_eoc_status(chan); 790 791 return IRQ_HANDLED; 792} 793 794static irqreturn_t iop_adma_err_handler(int irq, void *data) 795{ 796 struct iop_adma_chan *chan = data; 797 unsigned long status = iop_chan_get_status(chan); 798 799 dev_err(chan->device->common.dev, 800 "error ( %s%s%s%s%s%s%s)\n", 801 iop_is_err_int_parity(status, chan) ? "int_parity " : "", 802 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", 803 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", 804 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", 805 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", 806 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", 807 iop_is_err_split_tx(status, chan) ? "split_tx " : ""); 808 809 iop_adma_device_clear_err_status(chan); 810 811 BUG(); 812 813 return IRQ_HANDLED; 814} 815 816static void iop_adma_issue_pending(struct dma_chan *chan) 817{ 818 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 819 820 if (iop_chan->pending) { 821 iop_chan->pending = 0; 822 iop_chan_append(iop_chan); 823 } 824} 825 826/* 827 * Perform a transaction to verify the HW works. 828 */ 829#define IOP_ADMA_TEST_SIZE 2000 830 831static int iop_adma_memcpy_self_test(struct iop_adma_device *device) 832{ 833 int i; 834 void *src, *dest; 835 dma_addr_t src_dma, dest_dma; 836 struct dma_chan *dma_chan; 837 dma_cookie_t cookie; 838 struct dma_async_tx_descriptor *tx; 839 int err = 0; 840 struct iop_adma_chan *iop_chan; 841 842 dev_dbg(device->common.dev, "%s\n", __func__); 843 844 src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 845 if (!src) 846 return -ENOMEM; 847 dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); 848 if (!dest) { 849 kfree(src); 850 return -ENOMEM; 851 } 852 853 /* Fill in src buffer */ 854 for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) 855 ((u8 *) src)[i] = (u8)i; 856 857 /* Start copy, using first DMA channel */ 858 dma_chan = container_of(device->common.channels.next, 859 struct dma_chan, 860 device_node); 861 if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 862 err = -ENODEV; 863 goto out; 864 } 865 866 dest_dma = dma_map_single(dma_chan->device->dev, dest, 867 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 868 src_dma = dma_map_single(dma_chan->device->dev, src, 869 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); 870 tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 871 IOP_ADMA_TEST_SIZE, 872 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 873 874 cookie = iop_adma_tx_submit(tx); 875 iop_adma_issue_pending(dma_chan); 876 msleep(1); 877 878 if (iop_adma_status(dma_chan, cookie, NULL) != 879 DMA_COMPLETE) { 880 dev_err(dma_chan->device->dev, 881 "Self-test copy timed out, disabling\n"); 882 err = -ENODEV; 883 goto free_resources; 884 } 885 886 iop_chan = to_iop_adma_chan(dma_chan); 887 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 888 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); 889 if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { 890 dev_err(dma_chan->device->dev, 891 "Self-test copy failed compare, disabling\n"); 892 err = -ENODEV; 893 goto free_resources; 894 } 895 896free_resources: 897 iop_adma_free_chan_resources(dma_chan); 898out: 899 kfree(src); 900 kfree(dest); 901 return err; 902} 903 904#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ 905static int 906iop_adma_xor_val_self_test(struct iop_adma_device *device) 907{ 908 int i, src_idx; 909 struct page *dest; 910 struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; 911 struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 912 dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; 913 dma_addr_t dest_dma; 914 struct dma_async_tx_descriptor *tx; 915 struct dma_chan *dma_chan; 916 dma_cookie_t cookie; 917 u8 cmp_byte = 0; 918 u32 cmp_word; 919 u32 zero_sum_result; 920 int err = 0; 921 struct iop_adma_chan *iop_chan; 922 923 dev_dbg(device->common.dev, "%s\n", __func__); 924 925 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 926 xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 927 if (!xor_srcs[src_idx]) { 928 while (src_idx--) 929 __free_page(xor_srcs[src_idx]); 930 return -ENOMEM; 931 } 932 } 933 934 dest = alloc_page(GFP_KERNEL); 935 if (!dest) { 936 while (src_idx--) 937 __free_page(xor_srcs[src_idx]); 938 return -ENOMEM; 939 } 940 941 /* Fill in src buffers */ 942 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { 943 u8 *ptr = page_address(xor_srcs[src_idx]); 944 for (i = 0; i < PAGE_SIZE; i++) 945 ptr[i] = (1 << src_idx); 946 } 947 948 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) 949 cmp_byte ^= (u8) (1 << src_idx); 950 951 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 952 (cmp_byte << 8) | cmp_byte; 953 954 memset(page_address(dest), 0, PAGE_SIZE); 955 956 dma_chan = container_of(device->common.channels.next, 957 struct dma_chan, 958 device_node); 959 if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 960 err = -ENODEV; 961 goto out; 962 } 963 964 /* test xor */ 965 dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, 966 PAGE_SIZE, DMA_FROM_DEVICE); 967 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 968 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 969 0, PAGE_SIZE, DMA_TO_DEVICE); 970 tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 971 IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, 972 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 973 974 cookie = iop_adma_tx_submit(tx); 975 iop_adma_issue_pending(dma_chan); 976 msleep(8); 977 978 if (iop_adma_status(dma_chan, cookie, NULL) != 979 DMA_COMPLETE) { 980 dev_err(dma_chan->device->dev, 981 "Self-test xor timed out, disabling\n"); 982 err = -ENODEV; 983 goto free_resources; 984 } 985 986 iop_chan = to_iop_adma_chan(dma_chan); 987 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, 988 PAGE_SIZE, DMA_FROM_DEVICE); 989 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 990 u32 *ptr = page_address(dest); 991 if (ptr[i] != cmp_word) { 992 dev_err(dma_chan->device->dev, 993 "Self-test xor failed compare, disabling\n"); 994 err = -ENODEV; 995 goto free_resources; 996 } 997 } 998 dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, 999 PAGE_SIZE, DMA_TO_DEVICE); 1000 1001 /* skip zero sum if the capability is not present */ 1002 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) 1003 goto free_resources; 1004 1005 /* zero sum the sources with the destintation page */ 1006 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1007 zero_sum_srcs[i] = xor_srcs[i]; 1008 zero_sum_srcs[i] = dest; 1009 1010 zero_sum_result = 1; 1011 1012 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 1013 dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1014 zero_sum_srcs[i], 0, PAGE_SIZE, 1015 DMA_TO_DEVICE); 1016 tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1017 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1018 &zero_sum_result, 1019 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1020 1021 cookie = iop_adma_tx_submit(tx); 1022 iop_adma_issue_pending(dma_chan); 1023 msleep(8); 1024 1025 if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1026 dev_err(dma_chan->device->dev, 1027 "Self-test zero sum timed out, disabling\n"); 1028 err = -ENODEV; 1029 goto free_resources; 1030 } 1031 1032 if (zero_sum_result != 0) { 1033 dev_err(dma_chan->device->dev, 1034 "Self-test zero sum failed compare, disabling\n"); 1035 err = -ENODEV; 1036 goto free_resources; 1037 } 1038 1039 /* test for non-zero parity sum */ 1040 zero_sum_result = 0; 1041 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) 1042 dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1043 zero_sum_srcs[i], 0, PAGE_SIZE, 1044 DMA_TO_DEVICE); 1045 tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, 1046 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1047 &zero_sum_result, 1048 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1049 1050 cookie = iop_adma_tx_submit(tx); 1051 iop_adma_issue_pending(dma_chan); 1052 msleep(8); 1053 1054 if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { 1055 dev_err(dma_chan->device->dev, 1056 "Self-test non-zero sum timed out, disabling\n"); 1057 err = -ENODEV; 1058 goto free_resources; 1059 } 1060 1061 if (zero_sum_result != 1) { 1062 dev_err(dma_chan->device->dev, 1063 "Self-test non-zero sum failed compare, disabling\n"); 1064 err = -ENODEV; 1065 goto free_resources; 1066 } 1067 1068free_resources: 1069 iop_adma_free_chan_resources(dma_chan); 1070out: 1071 src_idx = IOP_ADMA_NUM_SRC_TEST; 1072 while (src_idx--) 1073 __free_page(xor_srcs[src_idx]); 1074 __free_page(dest); 1075 return err; 1076} 1077 1078#ifdef CONFIG_RAID6_PQ 1079static int 1080iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) 1081{ 1082 /* combined sources, software pq results, and extra hw pq results */ 1083 struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; 1084 /* ptr to the extra hw pq buffers defined above */ 1085 struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; 1086 /* address conversion buffers (dma_map / page_address) */ 1087 void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; 1088 dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; 1089 dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; 1090 1091 int i; 1092 struct dma_async_tx_descriptor *tx; 1093 struct dma_chan *dma_chan; 1094 dma_cookie_t cookie; 1095 u32 zero_sum_result; 1096 int err = 0; 1097 struct device *dev; 1098 1099 dev_dbg(device->common.dev, "%s\n", __func__); 1100 1101 for (i = 0; i < ARRAY_SIZE(pq); i++) { 1102 pq[i] = alloc_page(GFP_KERNEL); 1103 if (!pq[i]) { 1104 while (i--) 1105 __free_page(pq[i]); 1106 return -ENOMEM; 1107 } 1108 } 1109 1110 /* Fill in src buffers */ 1111 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { 1112 pq_sw[i] = page_address(pq[i]); 1113 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); 1114 } 1115 pq_sw[i] = page_address(pq[i]); 1116 pq_sw[i+1] = page_address(pq[i+1]); 1117 1118 dma_chan = container_of(device->common.channels.next, 1119 struct dma_chan, 1120 device_node); 1121 if (iop_adma_alloc_chan_resources(dma_chan) < 1) { 1122 err = -ENODEV; 1123 goto out; 1124 } 1125 1126 dev = dma_chan->device->dev; 1127 1128 /* initialize the dests */ 1129 memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); 1130 memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); 1131 1132 /* test pq */ 1133 pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1134 pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); 1135 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) 1136 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1137 DMA_TO_DEVICE); 1138 1139 tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, 1140 IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, 1141 PAGE_SIZE, 1142 DMA_PREP_INTERRUPT | 1143 DMA_CTRL_ACK); 1144 1145 cookie = iop_adma_tx_submit(tx); 1146 iop_adma_issue_pending(dma_chan); 1147 msleep(8); 1148 1149 if (iop_adma_status(dma_chan, cookie, NULL) != 1150 DMA_COMPLETE) { 1151 dev_err(dev, "Self-test pq timed out, disabling\n"); 1152 err = -ENODEV; 1153 goto free_resources; 1154 } 1155 1156 raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); 1157 1158 if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], 1159 page_address(pq_hw[0]), PAGE_SIZE) != 0) { 1160 dev_err(dev, "Self-test p failed compare, disabling\n"); 1161 err = -ENODEV; 1162 goto free_resources; 1163 } 1164 if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], 1165 page_address(pq_hw[1]), PAGE_SIZE) != 0) { 1166 dev_err(dev, "Self-test q failed compare, disabling\n"); 1167 err = -ENODEV; 1168 goto free_resources; 1169 } 1170 1171 /* test correct zero sum using the software generated pq values */ 1172 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1173 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1174 DMA_TO_DEVICE); 1175 1176 zero_sum_result = ~0; 1177 tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1178 pq_src, IOP_ADMA_NUM_SRC_TEST, 1179 raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1180 DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1181 1182 cookie = iop_adma_tx_submit(tx); 1183 iop_adma_issue_pending(dma_chan); 1184 msleep(8); 1185 1186 if (iop_adma_status(dma_chan, cookie, NULL) != 1187 DMA_COMPLETE) { 1188 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); 1189 err = -ENODEV; 1190 goto free_resources; 1191 } 1192 1193 if (zero_sum_result != 0) { 1194 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", 1195 zero_sum_result); 1196 err = -ENODEV; 1197 goto free_resources; 1198 } 1199 1200 /* test incorrect zero sum */ 1201 i = IOP_ADMA_NUM_SRC_TEST; 1202 memset(pq_sw[i] + 100, 0, 100); 1203 memset(pq_sw[i+1] + 200, 0, 200); 1204 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) 1205 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, 1206 DMA_TO_DEVICE); 1207 1208 zero_sum_result = 0; 1209 tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], 1210 pq_src, IOP_ADMA_NUM_SRC_TEST, 1211 raid6_gfexp, PAGE_SIZE, &zero_sum_result, 1212 DMA_PREP_INTERRUPT|DMA_CTRL_ACK); 1213 1214 cookie = iop_adma_tx_submit(tx); 1215 iop_adma_issue_pending(dma_chan); 1216 msleep(8); 1217 1218 if (iop_adma_status(dma_chan, cookie, NULL) != 1219 DMA_COMPLETE) { 1220 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); 1221 err = -ENODEV; 1222 goto free_resources; 1223 } 1224 1225 if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { 1226 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", 1227 zero_sum_result); 1228 err = -ENODEV; 1229 goto free_resources; 1230 } 1231 1232free_resources: 1233 iop_adma_free_chan_resources(dma_chan); 1234out: 1235 i = ARRAY_SIZE(pq); 1236 while (i--) 1237 __free_page(pq[i]); 1238 return err; 1239} 1240#endif 1241 1242static int iop_adma_remove(struct platform_device *dev) 1243{ 1244 struct iop_adma_device *device = platform_get_drvdata(dev); 1245 struct dma_chan *chan, *_chan; 1246 struct iop_adma_chan *iop_chan; 1247 struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev); 1248 1249 dma_async_device_unregister(&device->common); 1250 1251 dma_free_coherent(&dev->dev, plat_data->pool_size, 1252 device->dma_desc_pool_virt, device->dma_desc_pool); 1253 1254 list_for_each_entry_safe(chan, _chan, &device->common.channels, 1255 device_node) { 1256 iop_chan = to_iop_adma_chan(chan); 1257 list_del(&chan->device_node); 1258 kfree(iop_chan); 1259 } 1260 kfree(device); 1261 1262 return 0; 1263} 1264 1265static int iop_adma_probe(struct platform_device *pdev) 1266{ 1267 struct resource *res; 1268 int ret = 0, i; 1269 struct iop_adma_device *adev; 1270 struct iop_adma_chan *iop_chan; 1271 struct dma_device *dma_dev; 1272 struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev); 1273 1274 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1275 if (!res) 1276 return -ENODEV; 1277 1278 if (!devm_request_mem_region(&pdev->dev, res->start, 1279 resource_size(res), pdev->name)) 1280 return -EBUSY; 1281 1282 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 1283 if (!adev) 1284 return -ENOMEM; 1285 dma_dev = &adev->common; 1286 1287 /* allocate coherent memory for hardware descriptors 1288 * note: writecombine gives slightly better performance, but 1289 * requires that we explicitly flush the writes 1290 */ 1291 adev->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, 1292 plat_data->pool_size, 1293 &adev->dma_desc_pool, 1294 GFP_KERNEL); 1295 if (!adev->dma_desc_pool_virt) { 1296 ret = -ENOMEM; 1297 goto err_free_adev; 1298 } 1299 1300 dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %pad\n", 1301 __func__, adev->dma_desc_pool_virt, &adev->dma_desc_pool); 1302 1303 adev->id = plat_data->hw_id; 1304 1305 /* discover transaction capabilites from the platform data */ 1306 dma_dev->cap_mask = plat_data->cap_mask; 1307 1308 adev->pdev = pdev; 1309 platform_set_drvdata(pdev, adev); 1310 1311 INIT_LIST_HEAD(&dma_dev->channels); 1312 1313 /* set base routines */ 1314 dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; 1315 dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; 1316 dma_dev->device_tx_status = iop_adma_status; 1317 dma_dev->device_issue_pending = iop_adma_issue_pending; 1318 dma_dev->dev = &pdev->dev; 1319 1320 /* set prep routines based on capability */ 1321 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1322 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; 1323 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1324 dma_dev->max_xor = iop_adma_get_max_xor(); 1325 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; 1326 } 1327 if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) 1328 dma_dev->device_prep_dma_xor_val = 1329 iop_adma_prep_dma_xor_val; 1330 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1331 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); 1332 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; 1333 } 1334 if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) 1335 dma_dev->device_prep_dma_pq_val = 1336 iop_adma_prep_dma_pq_val; 1337 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1338 dma_dev->device_prep_dma_interrupt = 1339 iop_adma_prep_dma_interrupt; 1340 1341 iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); 1342 if (!iop_chan) { 1343 ret = -ENOMEM; 1344 goto err_free_dma; 1345 } 1346 iop_chan->device = adev; 1347 1348 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, 1349 resource_size(res)); 1350 if (!iop_chan->mmr_base) { 1351 ret = -ENOMEM; 1352 goto err_free_iop_chan; 1353 } 1354 tasklet_setup(&iop_chan->irq_tasklet, iop_adma_tasklet); 1355 1356 /* clear errors before enabling interrupts */ 1357 iop_adma_device_clear_err_status(iop_chan); 1358 1359 for (i = 0; i < 3; i++) { 1360 static const irq_handler_t handler[] = { 1361 iop_adma_eot_handler, 1362 iop_adma_eoc_handler, 1363 iop_adma_err_handler 1364 }; 1365 int irq = platform_get_irq(pdev, i); 1366 if (irq < 0) { 1367 ret = -ENXIO; 1368 goto err_free_iop_chan; 1369 } else { 1370 ret = devm_request_irq(&pdev->dev, irq, 1371 handler[i], 0, pdev->name, iop_chan); 1372 if (ret) 1373 goto err_free_iop_chan; 1374 } 1375 } 1376 1377 spin_lock_init(&iop_chan->lock); 1378 INIT_LIST_HEAD(&iop_chan->chain); 1379 INIT_LIST_HEAD(&iop_chan->all_slots); 1380 iop_chan->common.device = dma_dev; 1381 dma_cookie_init(&iop_chan->common); 1382 list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); 1383 1384 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1385 ret = iop_adma_memcpy_self_test(adev); 1386 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1387 if (ret) 1388 goto err_free_iop_chan; 1389 } 1390 1391 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1392 ret = iop_adma_xor_val_self_test(adev); 1393 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1394 if (ret) 1395 goto err_free_iop_chan; 1396 } 1397 1398 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && 1399 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { 1400 #ifdef CONFIG_RAID6_PQ 1401 ret = iop_adma_pq_zero_sum_self_test(adev); 1402 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); 1403 #else 1404 /* can not test raid6, so do not publish capability */ 1405 dma_cap_clear(DMA_PQ, dma_dev->cap_mask); 1406 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); 1407 ret = 0; 1408 #endif 1409 if (ret) 1410 goto err_free_iop_chan; 1411 } 1412 1413 dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n", 1414 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", 1415 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", 1416 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1417 dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", 1418 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1419 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1420 1421 dma_async_device_register(dma_dev); 1422 goto out; 1423 1424 err_free_iop_chan: 1425 kfree(iop_chan); 1426 err_free_dma: 1427 dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, 1428 adev->dma_desc_pool_virt, adev->dma_desc_pool); 1429 err_free_adev: 1430 kfree(adev); 1431 out: 1432 return ret; 1433} 1434 1435static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) 1436{ 1437 struct iop_adma_desc_slot *sw_desc, *grp_start; 1438 dma_cookie_t cookie; 1439 int slot_cnt, slots_per_op; 1440 1441 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1442 1443 spin_lock_bh(&iop_chan->lock); 1444 slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); 1445 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1446 if (sw_desc) { 1447 grp_start = sw_desc->group_head; 1448 1449 list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1450 async_tx_ack(&sw_desc->async_tx); 1451 iop_desc_init_memcpy(grp_start, 0); 1452 iop_desc_set_byte_count(grp_start, iop_chan, 0); 1453 iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1454 iop_desc_set_memcpy_src_addr(grp_start, 0); 1455 1456 cookie = dma_cookie_assign(&sw_desc->async_tx); 1457 1458 /* initialize the completed cookie to be less than 1459 * the most recently used cookie 1460 */ 1461 iop_chan->common.completed_cookie = cookie - 1; 1462 1463 /* channel should not be busy */ 1464 BUG_ON(iop_chan_is_busy(iop_chan)); 1465 1466 /* clear any prior error-status bits */ 1467 iop_adma_device_clear_err_status(iop_chan); 1468 1469 /* disable operation */ 1470 iop_chan_disable(iop_chan); 1471 1472 /* set the descriptor address */ 1473 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1474 1475 /* 1/ don't add pre-chained descriptors 1476 * 2/ dummy read to flush next_desc write 1477 */ 1478 BUG_ON(iop_desc_get_next_desc(sw_desc)); 1479 1480 /* run the descriptor */ 1481 iop_chan_enable(iop_chan); 1482 } else 1483 dev_err(iop_chan->device->common.dev, 1484 "failed to allocate null descriptor\n"); 1485 spin_unlock_bh(&iop_chan->lock); 1486} 1487 1488static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) 1489{ 1490 struct iop_adma_desc_slot *sw_desc, *grp_start; 1491 dma_cookie_t cookie; 1492 int slot_cnt, slots_per_op; 1493 1494 dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); 1495 1496 spin_lock_bh(&iop_chan->lock); 1497 slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); 1498 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); 1499 if (sw_desc) { 1500 grp_start = sw_desc->group_head; 1501 list_splice_init(&sw_desc->tx_list, &iop_chan->chain); 1502 async_tx_ack(&sw_desc->async_tx); 1503 iop_desc_init_null_xor(grp_start, 2, 0); 1504 iop_desc_set_byte_count(grp_start, iop_chan, 0); 1505 iop_desc_set_dest_addr(grp_start, iop_chan, 0); 1506 iop_desc_set_xor_src_addr(grp_start, 0, 0); 1507 iop_desc_set_xor_src_addr(grp_start, 1, 0); 1508 1509 cookie = dma_cookie_assign(&sw_desc->async_tx); 1510 1511 /* initialize the completed cookie to be less than 1512 * the most recently used cookie 1513 */ 1514 iop_chan->common.completed_cookie = cookie - 1; 1515 1516 /* channel should not be busy */ 1517 BUG_ON(iop_chan_is_busy(iop_chan)); 1518 1519 /* clear any prior error-status bits */ 1520 iop_adma_device_clear_err_status(iop_chan); 1521 1522 /* disable operation */ 1523 iop_chan_disable(iop_chan); 1524 1525 /* set the descriptor address */ 1526 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); 1527 1528 /* 1/ don't add pre-chained descriptors 1529 * 2/ dummy read to flush next_desc write 1530 */ 1531 BUG_ON(iop_desc_get_next_desc(sw_desc)); 1532 1533 /* run the descriptor */ 1534 iop_chan_enable(iop_chan); 1535 } else 1536 dev_err(iop_chan->device->common.dev, 1537 "failed to allocate null descriptor\n"); 1538 spin_unlock_bh(&iop_chan->lock); 1539} 1540 1541static struct platform_driver iop_adma_driver = { 1542 .probe = iop_adma_probe, 1543 .remove = iop_adma_remove, 1544 .driver = { 1545 .name = "iop-adma", 1546 }, 1547}; 1548 1549module_platform_driver(iop_adma_driver); 1550 1551MODULE_AUTHOR("Intel Corporation"); 1552MODULE_DESCRIPTION("IOP ADMA Engine Driver"); 1553MODULE_LICENSE("GPL"); 1554MODULE_ALIAS("platform:iop-adma"); 1555