Lines Matching refs:pbuf
17 * @pbuf: a number of blocks allocated within a PIO send context
27 * o pbuf->start always starts on a block boundary
28 * o pbuf can wrap only at a block boundary
30 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
33 void __iomem *dest = pbuf->start + SOP_DISTANCE;
80 * the buffer wraps, then pbuf->end == dend == dest
84 if (pbuf->end <= dend) {
85 while (dest < pbuf->end) {
91 dest -= pbuf->sc->size;
92 dend -= pbuf->sc->size;
114 * fill in rest of block, no need to check pbuf->end
123 this_cpu_dec(*pbuf->sc->buffers_allocated);
176 * of pbuf->carry. Other bytes are left as-is. Any previous
177 * value in pbuf->carry is lost.
183 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
186 pbuf->carry.val64 = 0;
187 jcopy(&pbuf->carry.val8[0], from, nbytes);
188 pbuf->carry_bytes = nbytes;
192 * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
199 static inline void read_extra_bytes(struct pio_buf *pbuf,
202 jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
203 pbuf->carry_bytes += nbytes;
207 * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
209 * pbuf->carry with the upper bytes zeroed..
216 struct pio_buf *pbuf,
223 temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
225 pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
241 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
243 if (pbuf->carry_bytes) {
245 writeq(pbuf->carry.val64, dest);
257 * @pbuf: destination buffer
262 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
265 void __iomem *dest = pbuf->start + SOP_DISTANCE;
311 * the buffer wraps, then pbuf->end == dend == dest
315 if (pbuf->end <= dend) {
316 while (dest < pbuf->end) {
322 dest -= pbuf->sc->size;
323 dend -= pbuf->sc->size;
338 read_low_bytes(pbuf, from, nbytes & 0x7);
340 pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
349 * @pbuf: destination buffer
355 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
357 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
365 if (pbuf->qw_written < PIO_BLOCK_QWS) {
378 send = pbuf->start + PIO_BLOCK_SIZE;
387 merge_write8(pbuf, dest, from);
409 * the buffer wraps, then pbuf->end == dend == dest
412 if (pbuf->end <= dend) {
413 while (dest < pbuf->end) {
414 merge_write8(pbuf, dest, from);
419 dest -= pbuf->sc->size;
420 dend -= pbuf->sc->size;
425 merge_write8(pbuf, dest, from);
430 pbuf->qw_written += qw_to_write;
433 if (pbuf->carry_bytes + bytes_left >= 8) {
437 nread = 8 - pbuf->carry_bytes;
438 read_extra_bytes(pbuf, from, nread);
452 if (dest >= pbuf->end)
453 dest -= pbuf->sc->size;
455 else if (pbuf->qw_written < PIO_BLOCK_QWS)
459 carry8_write8(pbuf->carry, dest);
460 pbuf->qw_written++;
465 read_low_bytes(pbuf, from, bytes_left);
468 read_extra_bytes(pbuf, from, bytes_left);
476 * @pbuf: destination buffer
482 static void mid_copy_straight(struct pio_buf *pbuf,
485 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
491 if (pbuf->qw_written < PIO_BLOCK_QWS) {
504 send = pbuf->start + PIO_BLOCK_SIZE;
535 * the buffer wraps, then pbuf->end == dend == dest
538 if (pbuf->end <= dend) {
539 while (dest < pbuf->end) {
545 dest -= pbuf->sc->size;
546 dend -= pbuf->sc->size;
557 read_low_bytes(pbuf, from, nbytes & 0x7);
559 pbuf->qw_written += nbytes >> 3;
567 * @pbuf: a number of blocks allocated within a PIO send context
571 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
575 if (pbuf->carry_bytes + nbytes < 8) {
577 read_extra_bytes(pbuf, from, nbytes);
594 if (pbuf->carry_bytes + to_align < 8) {
596 read_extra_bytes(pbuf, from, to_align);
601 unsigned long to_fill = 8 - pbuf->carry_bytes;
607 read_extra_bytes(pbuf, from, to_fill);
615 dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
626 if (dest >= pbuf->end)
627 dest -= pbuf->sc->size;
629 else if (pbuf->qw_written < PIO_BLOCK_QWS)
632 carry8_write8(pbuf->carry, dest);
633 pbuf->qw_written++;
636 /* this will overwrite anything in pbuf->carry */
637 read_low_bytes(pbuf, from, extra);
654 if (pbuf->carry_bytes)
655 mid_copy_mix(pbuf, from, nbytes);
657 mid_copy_straight(pbuf, from, nbytes);
663 * Write any remainder (in pbuf->carry) and finish writing the whole block.
665 * @pbuf: a number of blocks allocated within a PIO send context
667 void seg_pio_copy_end(struct pio_buf *pbuf)
669 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
679 if (dest >= pbuf->end)
680 dest -= pbuf->sc->size;
682 else if (pbuf->qw_written < PIO_BLOCK_QWS)
686 if (carry_write8(pbuf, dest)) {
713 this_cpu_dec(*pbuf->sc->buffers_allocated);