Lines Matching refs:pbuf
58 * @pbuf: a number of blocks allocated within a PIO send context
68 * o pbuf->start always starts on a block boundary
69 * o pbuf can wrap only at a block boundary
71 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
74 void __iomem *dest = pbuf->start + SOP_DISTANCE;
121 * the buffer wraps, then pbuf->end == dend == dest
125 if (pbuf->end <= dend) {
126 while (dest < pbuf->end) {
132 dest -= pbuf->sc->size;
133 dend -= pbuf->sc->size;
155 * fill in rest of block, no need to check pbuf->end
164 this_cpu_dec(*pbuf->sc->buffers_allocated);
217 * of pbuf->carry. Other bytes are left as-is. Any previous
218 * value in pbuf->carry is lost.
224 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
227 pbuf->carry.val64 = 0;
228 jcopy(&pbuf->carry.val8[0], from, nbytes);
229 pbuf->carry_bytes = nbytes;
233 * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
240 static inline void read_extra_bytes(struct pio_buf *pbuf,
243 jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
244 pbuf->carry_bytes += nbytes;
248 * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
250 * pbuf->carry with the upper bytes zeroed..
257 struct pio_buf *pbuf,
264 temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
266 pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
282 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
284 if (pbuf->carry_bytes) {
286 writeq(pbuf->carry.val64, dest);
298 * @pbuf: destination buffer
303 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
306 void __iomem *dest = pbuf->start + SOP_DISTANCE;
352 * the buffer wraps, then pbuf->end == dend == dest
356 if (pbuf->end <= dend) {
357 while (dest < pbuf->end) {
363 dest -= pbuf->sc->size;
364 dend -= pbuf->sc->size;
379 read_low_bytes(pbuf, from, nbytes & 0x7);
381 pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
390 * @pbuf: destination buffer
396 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
398 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
406 if (pbuf->qw_written < PIO_BLOCK_QWS) {
419 send = pbuf->start + PIO_BLOCK_SIZE;
428 merge_write8(pbuf, dest, from);
450 * the buffer wraps, then pbuf->end == dend == dest
453 if (pbuf->end <= dend) {
454 while (dest < pbuf->end) {
455 merge_write8(pbuf, dest, from);
460 dest -= pbuf->sc->size;
461 dend -= pbuf->sc->size;
466 merge_write8(pbuf, dest, from);
471 pbuf->qw_written += qw_to_write;
474 if (pbuf->carry_bytes + bytes_left >= 8) {
478 nread = 8 - pbuf->carry_bytes;
479 read_extra_bytes(pbuf, from, nread);
493 if (dest >= pbuf->end)
494 dest -= pbuf->sc->size;
496 else if (pbuf->qw_written < PIO_BLOCK_QWS)
500 carry8_write8(pbuf->carry, dest);
501 pbuf->qw_written++;
506 read_low_bytes(pbuf, from, bytes_left);
509 read_extra_bytes(pbuf, from, bytes_left);
517 * @pbuf: destination buffer
523 static void mid_copy_straight(struct pio_buf *pbuf,
526 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
532 if (pbuf->qw_written < PIO_BLOCK_QWS) {
545 send = pbuf->start + PIO_BLOCK_SIZE;
576 * the buffer wraps, then pbuf->end == dend == dest
579 if (pbuf->end <= dend) {
580 while (dest < pbuf->end) {
586 dest -= pbuf->sc->size;
587 dend -= pbuf->sc->size;
598 read_low_bytes(pbuf, from, nbytes & 0x7);
600 pbuf->qw_written += nbytes >> 3;
608 * @pbuf: a number of blocks allocated within a PIO send context
612 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
616 if (pbuf->carry_bytes + nbytes < 8) {
618 read_extra_bytes(pbuf, from, nbytes);
635 if (pbuf->carry_bytes + to_align < 8) {
637 read_extra_bytes(pbuf, from, to_align);
642 unsigned long to_fill = 8 - pbuf->carry_bytes;
648 read_extra_bytes(pbuf, from, to_fill);
656 dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
667 if (dest >= pbuf->end)
668 dest -= pbuf->sc->size;
670 else if (pbuf->qw_written < PIO_BLOCK_QWS)
673 carry8_write8(pbuf->carry, dest);
674 pbuf->qw_written++;
677 /* this will overwrite anything in pbuf->carry */
678 read_low_bytes(pbuf, from, extra);
695 if (pbuf->carry_bytes)
696 mid_copy_mix(pbuf, from, nbytes);
698 mid_copy_straight(pbuf, from, nbytes);
704 * Write any remainder (in pbuf->carry) and finish writing the whole block.
706 * @pbuf: a number of blocks allocated within a PIO send context
708 void seg_pio_copy_end(struct pio_buf *pbuf)
710 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
720 if (dest >= pbuf->end)
721 dest -= pbuf->sc->size;
723 else if (pbuf->qw_written < PIO_BLOCK_QWS)
727 if (carry_write8(pbuf, dest)) {
754 this_cpu_dec(*pbuf->sc->buffers_allocated);