Lines Matching defs:uarg
1542 struct ubuf_info_msgzc *uarg;
1551 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1552 uarg = (void *)skb->cb;
1553 uarg->mmp.user = NULL;
1555 if (mm_account_pinned_pages(&uarg->mmp, size)) {
1560 uarg->ubuf.callback = msg_zerocopy_callback;
1561 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1562 uarg->len = 1;
1563 uarg->bytelen = size;
1564 uarg->zerocopy = 1;
1565 uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
1566 refcount_set(&uarg->ubuf.refcnt, 1);
1569 return &uarg->ubuf;
1572 static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
1574 return container_of((void *)uarg, struct sk_buff, cb);
1578 struct ubuf_info *uarg)
1580 if (uarg) {
1586 if (uarg->callback != msg_zerocopy_callback)
1590 * so uarg->len and sk_zckey access is serialized
1597 uarg_zc = uarg_to_msgzc(uarg);
1600 /* TCP can create new skb to attach new uarg */
1616 net_zcopy_get(uarg);
1618 return uarg;
1647 static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
1649 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1658 mm_unaccount_pinned_pages(&uarg->mmp);
1663 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1666 len = uarg->len;
1667 lo = uarg->id;
1668 hi = uarg->id + len - 1;
1669 is_zerocopy = uarg->zerocopy;
1697 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1700 struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
1704 if (refcount_dec_and_test(&uarg->refcnt))
1709 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1711 struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
1714 uarg_to_msgzc(uarg)->len--;
1717 msg_zerocopy_callback(NULL, uarg, true);
1723 struct ubuf_info *uarg)
1728 /* An skb can only point to one uarg. This edge case happens when
1731 if (orig_uarg && uarg != orig_uarg)
1746 skb_zcopy_set(skb, uarg, NULL);