Lines Matching defs:uarg
1105 struct ubuf_info *uarg;
1114 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1115 uarg = (void *)skb->cb;
1116 uarg->mmp.user = NULL;
1118 if (mm_account_pinned_pages(&uarg->mmp, size)) {
1123 uarg->callback = sock_zerocopy_callback;
1124 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
1125 uarg->len = 1;
1126 uarg->bytelen = size;
1127 uarg->zerocopy = 1;
1128 refcount_set(&uarg->refcnt, 1);
1131 return uarg;
1135 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
1137 return container_of((void *)uarg, struct sk_buff, cb);
1141 struct ubuf_info *uarg)
1143 if (uarg) {
1148 * so uarg->len and sk_zckey access is serialized
1155 bytelen = uarg->bytelen + size;
1156 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
1157 /* TCP can create new skb to attach new uarg */
1164 if ((u32)(uarg->id + uarg->len) == next) {
1165 if (mm_account_pinned_pages(&uarg->mmp, size))
1167 uarg->len++;
1168 uarg->bytelen = bytelen;
1173 sock_zerocopy_get(uarg);
1175 return uarg;
1204 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1206 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1214 mm_unaccount_pinned_pages(&uarg->mmp);
1219 if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1222 len = uarg->len;
1223 lo = uarg->id;
1224 hi = uarg->id + len - 1;
1253 void sock_zerocopy_put(struct ubuf_info *uarg)
1255 if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1256 if (uarg->callback)
1257 uarg->callback(uarg, uarg->zerocopy);
1259 consume_skb(skb_from_uarg(uarg));
1264 void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1266 if (uarg) {
1267 struct sock *sk = skb_from_uarg(uarg)->sk;
1270 uarg->len--;
1273 sock_zerocopy_put(uarg);
1286 struct ubuf_info *uarg)
1292 /* An skb can only point to one uarg. This edge case happens when
1295 if (orig_uarg && uarg != orig_uarg)
1310 skb_zcopy_set(skb, uarg, NULL);