Lines Matching refs:skb
19 * Ray VanTassle : Fixed --skb->lock in free
74 #include <trace/events/skb.h>
92 * @skb: buffer
102 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
106 msg, addr, skb->len, sz, skb->head, skb->data,
107 (unsigned long)skb->tail, (unsigned long)skb->end,
108 skb->dev ? skb->dev->name : "<NULL>");
112 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
114 skb_panic(skb, sz, addr, __func__);
117 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
119 skb_panic(skb, sz, addr, __func__);
170 * instead of head cache and allocate a cloned (child) skb.
187 struct sk_buff *skb;
198 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
199 if (!skb)
201 prefetchw(skb);
206 * Both skb->head and skb_shared_info are cache line aligned.
225 memset(skb, 0, offsetof(struct sk_buff, tail));
226 /* Account for allocated memory : skb + skb->head */
227 skb->truesize = SKB_TRUESIZE(size);
228 skb->pfmemalloc = pfmemalloc;
229 refcount_set(&skb->users, 1);
230 skb->head = data;
231 skb->data = data;
232 skb_reset_tail_pointer(skb);
233 skb->end = skb->tail + size;
234 skb->mac_header = (typeof(skb->mac_header))~0U;
235 skb->transport_header = (typeof(skb->transport_header))~0U;
238 shinfo = skb_shinfo(skb);
245 fclones = container_of(skb, struct sk_buff_fclones, skb1);
247 skb->fclone = SKB_FCLONE_ORIG;
253 skb_set_kcov_handle(skb, kcov_common_handle());
256 return skb;
258 kmem_cache_free(cache, skb);
259 skb = NULL;
265 static struct sk_buff *__build_skb_around(struct sk_buff *skb,
274 skb->truesize = SKB_TRUESIZE(size);
275 refcount_set(&skb->users, 1);
276 skb->head = data;
277 skb->data = data;
278 skb_reset_tail_pointer(skb);
279 skb->end = skb->tail + size;
280 skb->mac_header = (typeof(skb->mac_header))~0U;
281 skb->transport_header = (typeof(skb->transport_header))~0U;
284 shinfo = skb_shinfo(skb);
288 skb_set_kcov_handle(skb, kcov_common_handle());
290 return skb;
302 * The return is the new skb buffer.
314 struct sk_buff *skb;
316 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
317 if (unlikely(!skb))
320 memset(skb, 0, offsetof(struct sk_buff, tail));
322 return __build_skb_around(skb, data, frag_size);
326 * takes care of skb->head and skb->pfmemalloc
332 struct sk_buff *skb = __build_skb(data, frag_size);
334 if (skb && frag_size) {
335 skb->head_frag = 1;
337 skb->pfmemalloc = 1;
339 return skb;
344 * build_skb_around - build a network buffer around provided skb
345 * @skb: sk_buff provide by caller, must be memset cleared
349 struct sk_buff *build_skb_around(struct sk_buff *skb,
352 if (unlikely(!skb))
355 skb = __build_skb_around(skb, data, frag_size);
357 if (skb && frag_size) {
358 skb->head_frag = 1;
360 skb->pfmemalloc = 1;
362 return skb;
434 struct sk_buff *skb;
441 * we use kmalloc() for skb->head allocation.
446 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
447 if (!skb)
473 skb = __build_skb(data, len);
474 if (unlikely(!skb)) {
480 skb->pfmemalloc = 1;
481 skb->head_frag = 1;
484 skb_reserve(skb, NET_SKB_PAD);
485 skb->dev = dev;
488 return skb;
509 struct sk_buff *skb;
515 * we use kmalloc() for skb->head allocation.
520 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
521 if (!skb)
537 skb = __build_skb(data, len);
538 if (unlikely(!skb)) {
544 skb->pfmemalloc = 1;
545 skb->head_frag = 1;
548 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
549 skb->dev = napi->dev;
552 return skb;
556 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
559 skb_fill_page_desc(skb, i, page, off, size);
560 skb->len += size;
561 skb->data_len += size;
562 skb->truesize += truesize;
566 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
569 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
572 skb->len += size;
573 skb->data_len += size;
574 skb->truesize += truesize;
584 static inline void skb_drop_fraglist(struct sk_buff *skb)
586 skb_drop_list(&skb_shinfo(skb)->frag_list);
589 static void skb_clone_fraglist(struct sk_buff *skb)
593 skb_walk_frags(skb, list)
597 static void skb_free_head(struct sk_buff *skb)
599 unsigned char *head = skb->head;
601 if (skb->head_frag)
607 static void skb_release_data(struct sk_buff *skb)
609 struct skb_shared_info *shinfo = skb_shinfo(skb);
612 if (skb->cloned &&
613 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
623 skb_zcopy_clear(skb, true);
624 skb_free_head(skb);
630 static void kfree_skbmem(struct sk_buff *skb)
634 switch (skb->fclone) {
636 kmem_cache_free(skbuff_head_cache, skb);
640 fclones = container_of(skb, struct sk_buff_fclones, skb1);
642 /* We usually free the clone (TX completion) before original skb
651 fclones = container_of(skb, struct sk_buff_fclones, skb2);
660 void skb_release_head_state(struct sk_buff *skb)
662 skb_dst_drop(skb);
663 if (skb->destructor) {
665 skb->destructor(skb);
668 nf_conntrack_put(skb_nfct(skb));
670 skb_ext_put(skb);
674 static void skb_release_all(struct sk_buff *skb)
676 skb_release_head_state(skb);
677 if (likely(skb->head))
678 skb_release_data(skb);
683 * @skb: buffer
690 void __kfree_skb(struct sk_buff *skb)
692 skb_release_all(skb);
693 kfree_skbmem(skb);
699 * @skb: buffer to free
704 void kfree_skb(struct sk_buff *skb)
706 if (!skb_unref(skb))
709 trace_kfree_skb(skb, __builtin_return_address(0));
710 __kfree_skb(skb);
725 /* Dump skb information and contents.
731 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
733 struct skb_shared_info *sh = skb_shinfo(skb);
734 struct net_device *dev = skb->dev;
735 struct sock *sk = skb->sk;
742 len = skb->len;
744 len = min_t(int, skb->len, MAX_HEADER + 128);
746 headroom = skb_headroom(skb);
747 tailroom = skb_tailroom(skb);
749 has_mac = skb_mac_header_was_set(skb);
750 has_trans = skb_transport_header_was_set(skb);
757 level, skb->len, headroom, skb_headlen(skb), tailroom,
758 has_mac ? skb->mac_header : -1,
759 has_mac ? skb_mac_header_len(skb) : -1,
760 skb->network_header,
761 has_trans ? skb_network_header_len(skb) : -1,
762 has_trans ? skb->transport_header : -1,
765 skb->csum, skb->ip_summed, skb->csum_complete_sw,
766 skb->csum_valid, skb->csum_level,
767 skb->hash, skb->sw_hash, skb->l4_hash,
768 ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
778 print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
779 16, 1, skb->head, headroom, false);
781 seg_len = min_t(int, skb_headlen(skb), len);
783 print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
784 16, 1, skb->data, seg_len, false);
788 print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
789 16, 1, skb_tail_pointer(skb), tailroom, false);
791 for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
792 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
802 print_hex_dump(level, "skb frag: ",
812 if (full_pkt && skb_has_frag_list(skb)) {
813 printk("skb fraglist:\n");
814 skb_walk_frags(skb, list_skb)
822 * @skb: buffer that triggered an error
824 * Report xmit error if a device callback is tracking this skb.
825 * skb must be freed afterwards.
827 void skb_tx_error(struct sk_buff *skb)
829 skb_zcopy_clear(skb, true);
836 * @skb: buffer to free
842 void consume_skb(struct sk_buff *skb)
844 if (!skb_unref(skb))
847 trace_consume_skb(skb);
848 __kfree_skb(skb);
855 * @skb: buffer to free
858 * skb reference and all the head states have been already dropped
860 void __consume_stateless_skb(struct sk_buff *skb)
862 trace_consume_skb(skb);
863 skb_release_data(skb);
864 kfree_skbmem(skb);
879 static inline void _kfree_skb_defer(struct sk_buff *skb)
883 /* drop skb->head and call any destructors for packet */
884 skb_release_all(skb);
886 /* record skb to CPU local list */
887 nc->skb_cache[nc->skb_count++] = skb;
891 prefetchw(skb);
901 void __kfree_skb_defer(struct sk_buff *skb)
903 _kfree_skb_defer(skb);
906 void napi_consume_skb(struct sk_buff *skb, int budget)
910 dev_consume_skb_any(skb);
914 if (!skb_unref(skb))
918 trace_consume_skb(skb);
921 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
922 __kfree_skb(skb);
926 _kfree_skb_defer(skb);
989 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
991 #define C(x) n->x = skb->x
995 __copy_skb_header(n, skb);
1000 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
1014 atomic_inc(&(skb_shinfo(skb)->dataref));
1015 skb->cloned = 1;
1047 * skb_morph - morph one skb into another
1048 * @dst: the skb to receive the contents
1049 * @src: the skb to supply the contents
1051 * This is identical to skb_clone except that the target skb is
1054 * The target skb is returned upon exit.
1106 struct sk_buff *skb;
1110 skb = sock_omalloc(sk, 0, GFP_KERNEL);
1111 if (!skb)
1114 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
1115 uarg = (void *)skb->cb;
1119 kfree_skb(skb);
1157 /* TCP can create new skb to attach new uarg */
1184 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1186 struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1206 struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1208 struct sock *sk = skb->sk;
1226 serr = SKB_EXT_ERR(skb);
1240 __skb_queue_tail(q, skb);
1241 skb = NULL;
1248 consume_skb(skb);
1278 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)
1280 return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
1284 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1288 struct ubuf_info *orig_uarg = skb_zcopy(skb);
1290 int err, orig_len = skb->len;
1292 /* An skb can only point to one uarg. This edge case happens when
1293 * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1298 err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1299 if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1300 struct sock *save_sk = skb->sk;
1302 /* Streams do not free skb on error. Reset to prev state. */
1304 skb->sk = sk;
1305 ___pskb_trim(skb, orig_len);
1306 skb->sk = save_sk;
1310 skb_zcopy_set(skb, uarg, NULL);
1311 return skb->len - orig_len;
1336 * skb_copy_ubufs - copy userspace skb frags buffers to kernel
1337 * @skb: the skb to modify
1340 * This must be called on SKBTX_DEV_ZEROCOPY skb.
1350 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1352 int num_frags = skb_shinfo(skb)->nr_frags;
1357 if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1363 new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1381 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1406 /* skb frags release userspace buffers */
1408 skb_frag_unref(skb, i);
1410 /* skb frags point to kernel buffers */
1412 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1415 __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1416 skb_shinfo(skb)->nr_frags = new_frags;
1419 skb_zcopy_clear(skb, false);
1426 * @skb: buffer to clone
1438 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1440 struct sk_buff_fclones *fclones = container_of(skb,
1445 if (skb_orphan_frags(skb, gfp_mask))
1448 if (skb->fclone == SKB_FCLONE_ORIG &&
1453 if (skb_pfmemalloc(skb))
1463 return __skb_clone(n, skb);
1467 void skb_headers_offset_update(struct sk_buff *skb, int off)
1470 if (skb->ip_summed == CHECKSUM_PARTIAL)
1471 skb->csum_start += off;
1472 /* {transport,network,mac}_header and tail are relative to skb->head */
1473 skb->transport_header += off;
1474 skb->network_header += off;
1475 if (skb_mac_header_was_set(skb))
1476 skb->mac_header += off;
1477 skb->inner_transport_header += off;
1478 skb->inner_network_header += off;
1479 skb->inner_mac_header += off;
1493 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1495 if (skb_pfmemalloc(skb))
1502 * @skb: buffer to copy
1517 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
1523 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
1526 headerlen = skb_headroom(skb);
1527 size = skb_end_offset(skb) + skb->data_len;
1529 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1536 skb_put(n, skb->len);
1538 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
1540 skb_copy_header(n, skb);
1547 * @skb: buffer to copy
1548 * @headroom: headroom of new skb
1550 * @fclone: if true allocate the copy of the skb from the fclone
1562 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1565 unsigned int size = skb_headlen(skb) + headroom;
1566 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1575 skb_put(n, skb_headlen(skb));
1577 skb_copy_from_linear_data(skb, n->data, n->len);
1579 n->truesize += skb->data_len;
1580 n->data_len = skb->data_len;
1581 n->len = skb->len;
1583 if (skb_shinfo(skb)->nr_frags) {
1586 if (skb_orphan_frags(skb, gfp_mask) ||
1587 skb_zerocopy_clone(n, skb, gfp_mask)) {
1592 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1593 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1594 skb_frag_ref(skb, i);
1599 if (skb_has_frag_list(skb)) {
1600 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1604 skb_copy_header(n, skb);
1612 * @skb: buffer to reallocate
1618 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1622 * All the pointers pointing into skb header may change and must be
1626 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1629 int i, osize = skb_end_offset(skb);
1636 BUG_ON(skb_shared(skb));
1640 if (skb_pfmemalloc(skb))
1651 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1654 skb_shinfo(skb),
1655 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1662 if (skb_cloned(skb)) {
1663 if (skb_orphan_frags(skb, gfp_mask))
1665 if (skb_zcopy(skb))
1666 refcount_inc(&skb_uarg(skb)->refcnt);
1667 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1668 skb_frag_ref(skb, i);
1670 if (skb_has_frag_list(skb))
1671 skb_clone_fraglist(skb);
1673 skb_release_data(skb);
1675 skb_free_head(skb);
1677 off = (data + nhead) - skb->head;
1679 skb->head = data;
1680 skb->head_frag = 0;
1681 skb->data += off;
1683 skb->end = size;
1686 skb->end = skb->head + size;
1688 skb->tail += off;
1689 skb_headers_offset_update(skb, nhead);
1690 skb->cloned = 0;
1691 skb->hdr_len = 0;
1692 skb->nohdr = 0;
1693 atomic_set(&skb_shinfo(skb)->dataref, 1);
1695 skb_metadata_clear(skb);
1697 /* It is not generally safe to change skb->truesize.
1699 * when skb is orphaned (not attached to a socket).
1701 if (!skb->sk || skb->destructor == sock_edemux)
1702 skb->truesize += size - osize;
1713 /* Make private copy of skb with writable head and some headroom */
1715 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1718 int delta = headroom - skb_headroom(skb);
1721 skb2 = pskb_copy(skb, GFP_ATOMIC);
1723 skb2 = skb_clone(skb, GFP_ATOMIC);
1736 * @skb: buffer to copy
1752 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1763 if (WARN_ON_ONCE(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))
1766 oldheadroom = skb_headroom(skb);
1767 n = __alloc_skb(newheadroom + skb->len + newtailroom,
1768 gfp_mask, skb_alloc_rx_flag(skb),
1776 skb_put(n, skb->len);
1786 BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1787 skb->len + head_copy_len));
1789 skb_copy_header(n, skb);
1798 * __skb_pad - zero pad the tail of an skb
1799 * @skb: buffer to pad
1807 * May return error in out of memory cases. The skb is freed on error
1811 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
1817 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1818 memset(skb->data+skb->len, 0, pad);
1822 ntail = skb->data_len + pad - (skb->end - skb->tail);
1823 if (likely(skb_cloned(skb) || ntail > 0)) {
1824 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1829 /* FIXME: The use of this function with non-linear skb's really needs
1832 err = skb_linearize(skb);
1836 memset(skb->data + skb->len, 0, pad);
1841 kfree_skb(skb);
1848 * @skb: start of the buffer to use
1853 * fragmented buffer. @tail must be the last fragment of @skb -- or
1854 * @skb itself. If this would exceed the total buffer size the kernel
1859 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1861 if (tail != skb) {
1862 skb->data_len += len;
1863 skb->len += len;
1871 * @skb: buffer to use
1878 void *skb_put(struct sk_buff *skb, unsigned int len)
1880 void *tmp = skb_tail_pointer(skb);
1881 SKB_LINEAR_ASSERT(skb);
1882 skb->tail += len;
1883 skb->len += len;
1884 if (unlikely(skb->tail > skb->end))
1885 skb_over_panic(skb, len, __builtin_return_address(0));
1892 * @skb: buffer to use
1899 void *skb_push(struct sk_buff *skb, unsigned int len)
1901 skb->data -= len;
1902 skb->len += len;
1903 if (unlikely(skb->data < skb->head))
1904 skb_under_panic(skb, len, __builtin_return_address(0));
1905 return skb->data;
1911 * @skb: buffer to use
1919 void *skb_pull(struct sk_buff *skb, unsigned int len)
1921 return skb_pull_inline(skb, len);
1927 * @skb: buffer to alter
1932 * The skb must be linear.
1934 void skb_trim(struct sk_buff *skb, unsigned int len)
1936 if (skb->len > len)
1937 __skb_trim(skb, len);
1941 /* Trims skb to length len. It can change skb pointers.
1944 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1948 int offset = skb_headlen(skb);
1949 int nfrags = skb_shinfo(skb)->nr_frags;
1953 if (skb_cloned(skb) &&
1954 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1962 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1969 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1972 skb_shinfo(skb)->nr_frags = i;
1975 skb_frag_unref(skb, i);
1977 if (skb_has_frag_list(skb))
1978 skb_drop_fraglist(skb);
1982 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
2014 if (len > skb_headlen(skb)) {
2015 skb->data_len -= skb->len - len;
2016 skb->len = len;
2018 skb->len = len;
2019 skb->data_len = 0;
2020 skb_set_tail_pointer(skb, len);
2023 if (!skb->sk || skb->destructor == sock_edemux)
2024 skb_condense(skb);
2031 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len)
2033 if (skb->ip_summed == CHECKSUM_COMPLETE) {
2034 int delta = skb->len - len;
2036 skb->csum = csum_block_sub(skb->csum,
2037 skb_checksum(skb, len, delta, 0),
2039 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
2040 int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len;
2041 int offset = skb_checksum_start_offset(skb) + skb->csum_offset;
2046 return __pskb_trim(skb, len);
2051 * __pskb_pull_tail - advance tail of skb header
2052 * @skb: buffer to reallocate
2062 * or value of new tail of skb in the case of success.
2064 * All the pointers pointing into skb header may change and must be
2068 /* Moves tail of skb head forward, copying data from fragmented part,
2071 * 2. It may change skb pointers.
2075 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
2077 /* If skb has not enough free space at tail, get new one
2079 * room at tail, reallocate without expansion only if skb is cloned.
2081 int i, k, eat = (skb->tail + delta) - skb->end;
2083 if (eat > 0 || skb_cloned(skb)) {
2084 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
2089 BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
2090 skb_tail_pointer(skb), delta));
2095 if (!skb_has_frag_list(skb))
2100 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2101 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2109 * Certainly, it is possible to add an offset to skb data,
2112 * further bloating skb head and crucify ourselves here instead.
2116 struct sk_buff *list = skb_shinfo(skb)->frag_list;
2128 if (skb_is_gso(skb) && !list->head_frag &&
2130 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
2153 while ((list = skb_shinfo(skb)->frag_list) != insp) {
2154 skb_shinfo(skb)->frag_list = list->next;
2160 skb_shinfo(skb)->frag_list = clone;
2163 /* Success! Now we may commit changes to skb data. */
2168 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2169 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
2172 skb_frag_unref(skb, i);
2175 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2177 *frag = skb_shinfo(skb)->frags[i];
2188 skb_shinfo(skb)->nr_frags = k;
2191 skb->tail += delta;
2192 skb->data_len -= delta;
2194 if (!skb->data_len)
2195 skb_zcopy_clear(skb, false);
2197 return skb_tail_pointer(skb);
2202 * skb_copy_bits - copy bits from skb to kernel buffer
2203 * @skb: source skb
2208 * Copy the specified number of bytes from the source skb to the
2216 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2218 int start = skb_headlen(skb);
2222 if (offset > (int)skb->len - len)
2229 skb_copy_from_linear_data_offset(skb, offset, to, copy);
2236 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2238 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2267 skb_walk_frags(skb, frag_iter) {
2398 * Map linear and fragment data from the skb to spd. It reports true if the
2401 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2409 * If skb->head_frag is set, this 'linear' part is backed by a
2413 if (__splice_segment(virt_to_page(skb->data),
2414 (unsigned long) skb->data & (PAGE_SIZE - 1),
2415 skb_headlen(skb),
2417 skb_head_is_locked(skb),
2424 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2425 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2433 skb_walk_frags(skb, iter) {
2450 * Map data from the skb to a pipe. Should handle both the linear part,
2453 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2468 __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2477 /* Send skb data on a socket. Socket must be locked. */
2478 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2482 struct sk_buff *head = skb;
2489 while (offset < skb_headlen(skb) && len) {
2493 slen = min_t(int, len, skb_headlen(skb) - offset);
2494 kv.iov_base = skb->data + offset;
2507 /* All the data was skb head? */
2512 offset -= skb_headlen(skb);
2515 for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2516 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2524 for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2525 skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2547 if (skb == head) {
2548 if (skb_has_frag_list(skb)) {
2549 skb = skb_shinfo(skb)->frag_list;
2552 } else if (skb->next) {
2553 skb = skb->next;
2567 * skb_store_bits - store bits from kernel buffer to skb
2568 * @skb: destination buffer
2574 * destination skb. This function handles all the messy bits of
2578 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2580 int start = skb_headlen(skb);
2584 if (offset > (int)skb->len - len)
2590 skb_copy_to_linear_data_offset(skb, offset, from, copy);
2597 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2598 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2628 skb_walk_frags(skb, frag_iter) {
2655 /* Checksum skb data. */
2656 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2659 int start = skb_headlen(skb);
2669 skb->data + offset, copy, csum);
2676 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2678 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2713 skb_walk_frags(skb, frag_iter) {
2740 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2748 return __skb_checksum(skb, offset, len, csum, &ops);
2754 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2757 int start = skb_headlen(skb);
2767 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2776 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2781 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2783 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2812 skb_walk_frags(skb, frag_iter) {
2839 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
2843 sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
2846 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2847 !skb->csum_complete_sw)
2848 netdev_rx_csum_fault(skb->dev, skb);
2850 if (!skb_shared(skb))
2851 skb->csum_valid = !sum;
2856 /* This function assumes skb->csum already holds pseudo header's checksum,
2858 * __skb_checksum_validate_complete(). And, the original skb->csum must
2862 * zero. The new checksum is stored back into skb->csum unless the skb is
2865 __sum16 __skb_checksum_complete(struct sk_buff *skb)
2870 csum = skb_checksum(skb, 0, skb->len, 0);
2872 sum = csum_fold(csum_add(skb->csum, csum));
2876 * between the original skb->csum and skb_checksum(). This means either
2877 * the original hardware checksum is incorrect or we screw up skb->csum
2878 * when moving skb->data around.
2881 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
2882 !skb->csum_complete_sw)
2883 netdev_rx_csum_fault(skb->dev, skb);
2886 if (!skb_shared(skb)) {
2888 skb->csum = csum;
2889 skb->ip_summed = CHECKSUM_COMPLETE;
2890 skb->csum_complete_sw = 1;
2891 skb->csum_valid = !sum;
2928 * Calculates the amount of linear headroom needed in the 'to' skb passed
2952 * skb_zerocopy - Zero copy skb to skb
2967 * -EFAULT: skb_copy_bits() found some problem with skb geometry
2973 int plen = 0; /* length of skb->head fragment */
3030 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
3035 if (skb->ip_summed == CHECKSUM_PARTIAL)
3036 csstart = skb_checksum_start_offset(skb);
3038 csstart = skb_headlen(skb);
3040 BUG_ON(csstart > skb_headlen(skb));
3042 skb_copy_from_linear_data(skb, to, csstart);
3045 if (csstart != skb->len)
3046 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
3047 skb->len - csstart);
3049 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3050 long csstuff = csstart + skb->csum_offset;
3108 struct sk_buff *skb;
3109 while ((skb = skb_dequeue(list)) != NULL)
3110 kfree_skb(skb);
3115 * skb_rbtree_purge - empty a skb rbtree
3130 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
3133 rb_erase(&skb->rbnode, root);
3134 sum += skb->truesize;
3135 kfree_skb(skb);
3184 * @skb: buffer to remove
3192 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
3197 __skb_unlink(skb, list);
3222 static inline void skb_split_inside_header(struct sk_buff *skb,
3228 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
3231 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
3232 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
3234 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
3235 skb_shinfo(skb)->nr_frags = 0;
3236 skb1->data_len = skb->data_len;
3238 skb->data_len = 0;
3239 skb->len = len;
3240 skb_set_tail_pointer(skb, len);
3243 static inline void skb_split_no_header(struct sk_buff *skb,
3248 const int nfrags = skb_shinfo(skb)->nr_frags;
3250 skb_shinfo(skb)->nr_frags = 0;
3251 skb1->len = skb1->data_len = skb->len - len;
3252 skb->len = len;
3253 skb->data_len = len - pos;
3256 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3259 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3270 skb_frag_ref(skb, i);
3273 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3274 skb_shinfo(skb)->nr_frags++;
3278 skb_shinfo(skb)->nr_frags++;
3285 * skb_split - Split fragmented skb to two parts at length len.
3286 * @skb: the buffer to split
3288 * @len: new length for skb
3290 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3292 int pos = skb_headlen(skb);
3294 skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3296 skb_zerocopy_clone(skb1, skb, 0);
3298 skb_split_inside_header(skb, skb1, len, pos);
3300 skb_split_no_header(skb, skb1, len, pos);
3304 /* Shifting from/to a cloned skb is a no-go.
3308 static int skb_prepare_for_shift(struct sk_buff *skb)
3312 if (skb_cloned(skb)) {
3317 unsigned int save_truesize = skb->truesize;
3319 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3320 skb->truesize = save_truesize;
3326 * skb_shift - Shifts paged data partially from skb to another
3328 * @skb: buffer from which the paged data comes from
3332 * the length of the skb, from skb to tgt. Returns number bytes shifted.
3333 * It's up to caller to free skb if everything was shifted.
3341 * specialized skb free'er to handle frags without up-to-date nr_frags.
3343 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3348 BUG_ON(shiftlen > skb->len);
3350 if (skb_headlen(skb))
3352 if (skb_zcopy(tgt) || skb_zcopy(skb))
3358 fragfrom = &skb_shinfo(skb)->frags[from];
3372 if (skb_prepare_for_shift(skb) ||
3377 fragfrom = &skb_shinfo(skb)->frags[from];
3390 /* Skip full, not-fitting skb to avoid expensive operations */
3391 if ((shiftlen == skb->len) &&
3392 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3395 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3398 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3402 fragfrom = &skb_shinfo(skb)->frags[from];
3430 fragfrom = &skb_shinfo(skb)->frags[0];
3437 /* Reposition in the original skb */
3439 while (from < skb_shinfo(skb)->nr_frags)
3440 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3441 skb_shinfo(skb)->nr_frags = to;
3443 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3446 /* Most likely the tgt won't ever need its checksum anymore, skb on
3450 skb->ip_summed = CHECKSUM_PARTIAL;
3453 skb->len -= shiftlen;
3454 skb->data_len -= shiftlen;
3455 skb->truesize -= shiftlen;
3464 * skb_prepare_seq_read - Prepare a sequential read of skb data
3465 * @skb: the buffer to read
3473 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3478 st->root_skb = st->cur_skb = skb;
3485 * skb_seq_read - Sequentially read skb data
3490 * Reads a block of skb data at @consumed relative to the
3493 * of the block or 0 if the end of the skb data or the upper
3577 * skb_abort_seq_read - Abort a sequential read of skb data
3605 * skb_find_text - Find a text pattern in skb data
3606 * @skb: the buffer to look in
3611 * Finds a pattern in the skb data according to the specified
3616 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3625 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3632 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3635 int i = skb_shinfo(skb)->nr_frags;
3637 if (skb_can_coalesce(skb, i, page, offset)) {
3638 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3641 skb_fill_page_desc(skb, i, page, offset, size);
3651 * skb_pull_rcsum - pull skb and update receive checksum
3652 * @skb: buffer to update
3661 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3663 unsigned char *data = skb->data;
3665 BUG_ON(len > skb->len);
3666 __skb_pull(skb, len);
3667 skb_postpull_rcsum(skb, data, len);
3668 return skb->data;
3685 struct sk_buff *skb_segment_list(struct sk_buff *skb,
3689 struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
3690 unsigned int tnl_hlen = skb_tnl_header_len(skb);
3697 skb_push(skb, -skb_network_offset(skb) + offset);
3700 err = skb_unclone(skb, GFP_ATOMIC);
3704 skb_shinfo(skb)->frag_list = NULL;
3724 skb->next = nskb;
3740 __copy_skb_header(nskb, skb);
3742 skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
3743 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
3752 skb->truesize = skb->truesize - delta_truesize;
3753 skb->data_len = skb->data_len - delta_len;
3754 skb->len = skb->len - delta_len;
3756 skb_gso_reset(skb);
3758 skb->prev = tail;
3760 if (skb_needs_linearize(skb, features) &&
3761 __skb_linearize(skb))
3764 skb_get(skb);
3766 return skb;
3769 kfree_skb_list(skb->next);
3770 skb->next = NULL;
3775 int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
3777 if (unlikely(p->len + skb->len >= 65536))
3781 skb_shinfo(p)->frag_list = skb;
3783 NAPI_GRO_CB(p)->last->next = skb;
3785 skb_pull(skb, skb_gro_offset(skb));
3787 NAPI_GRO_CB(p)->last = skb;
3789 p->data_len += skb->len;
3790 p->truesize += skb->truesize;
3791 p->len += skb->len;
3793 NAPI_GRO_CB(skb)->same_flow = 1;
3799 * skb_segment - Perform protocol segmentation on skb.
3803 * This function performs segmentation on the given skb. It returns
4157 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
4159 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
4160 unsigned int offset = skb_gro_offset(skb);
4161 unsigned int headlen = skb_headlen(skb);
4162 unsigned int len = skb_gro_len(skb);
4166 if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush))
4195 delta_truesize = skb->truesize -
4196 SKB_TRUESIZE(skb_end_offset(skb));
4198 skb->truesize -= skb->data_len;
4199 skb->len -= skb->data_len;
4200 skb->data_len = 0;
4202 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
4204 } else if (skb->head_frag) {
4207 struct page *page = virt_to_head_page(skb->head);
4214 first_offset = skb->data -
4227 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4228 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
4233 delta_truesize = skb->truesize;
4239 skb->data_len -= eat;
4240 skb->len -= eat;
4244 __skb_pull(skb, offset);
4247 skb_shinfo(p)->frag_list = skb;
4249 NAPI_GRO_CB(p)->last->next = skb;
4250 NAPI_GRO_CB(p)->last = skb;
4251 __skb_header_release(skb);
4264 NAPI_GRO_CB(skb)->same_flow = 1;
4338 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
4341 int start = skb_headlen(skb);
4352 sg_set_buf(sg, skb->data + offset, copy);
4359 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4364 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
4366 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4382 skb_walk_frags(skb, frag_iter) {
4411 * @skb: Socket buffer containing the buffers to be mapped
4421 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4423 int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
4434 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4435 * sglist without mark the sg which contain last skb data as the end.
4453 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4456 return __skb_to_sgvec(skb, sg, offset, len, 0);
4464 * @skb: The socket buffer to check.
4466 * @trailer: Returned pointer to the skb where the @tailbits space begins
4474 * set to point to the skb in which this space begins.
4479 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4485 /* If skb is cloned or its head is paged, reallocate
4489 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4490 !__pskb_pull_tail(skb, __skb_pagelen(skb)))
4494 if (!skb_has_frag_list(skb)) {
4500 if (skb_tailroom(skb) < tailbits &&
4501 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4505 *trailer = skb;
4512 skb_p = &skb_shinfo(skb)->frag_list;
4525 /* If the skb is the last, worry about trailer. */
4556 * OK, link new skb, drop old one */
4572 static void sock_rmem_free(struct sk_buff *skb)
4574 struct sock *sk = skb->sk;
4576 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4579 static void skb_set_err_queue(struct sk_buff *skb)
4584 skb->pkt_type = PACKET_OUTGOING;
4591 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4593 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4597 skb_orphan(skb);
4598 skb->sk = sk;
4599 skb->destructor = sock_rmem_free;
4600 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4601 skb_set_err_queue(skb);
4604 skb_dst_force(skb);
4606 skb_queue_tail(&sk->sk_error_queue, skb);
4613 static bool is_icmp_err_skb(const struct sk_buff *skb)
4615 return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4616 SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4622 struct sk_buff *skb, *skb_next = NULL;
4627 skb = __skb_dequeue(q);
4628 if (skb && (skb_next = skb_peek(q))) {
4635 if (is_icmp_err_skb(skb) && !icmp_next)
4641 return skb;
4646 * skb_clone_sk - create clone of skb, and take reference to socket
4647 * @skb: the skb to clone
4658 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4660 struct sock *sk = skb->sk;
4666 clone = skb_clone(skb, GFP_ATOMIC);
4679 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4687 BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4689 serr = SKB_EXT_ERR(skb);
4695 serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4697 serr->ee.ee_data = skb_shinfo(skb)->tskey;
4703 err = sock_queue_err_skb(sk, skb);
4706 kfree_skb(skb);
4723 void skb_complete_tx_timestamp(struct sk_buff *skb,
4726 struct sock *sk = skb->sk;
4735 *skb_hwtstamps(skb) = *hwtstamps;
4736 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4742 kfree_skb(skb);
4750 struct sk_buff *skb;
4769 skb = tcp_get_timestamping_opt_stats(sk, orig_skb);
4773 skb = alloc_skb(0, GFP_ATOMIC);
4775 skb = skb_clone(orig_skb, GFP_ATOMIC);
4777 if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
4778 kfree_skb(skb);
4782 if (!skb)
4786 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4788 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4792 *skb_hwtstamps(skb) = *hwtstamps;
4794 skb->tstamp = ktime_get_real();
4796 __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4808 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4810 struct sock *sk = skb->sk;
4814 skb->wifi_acked_valid = 1;
4815 skb->wifi_acked = acked;
4817 serr = SKB_EXT_ERR(skb);
4826 err = sock_queue_err_skb(sk, skb);
4830 kfree_skb(skb);
4836 * @skb: the skb to set
4837 * @start: the number of bytes after skb->data to start checksumming.
4841 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4843 * This function checks and sets those values and skb->ip_summed: if this
4846 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4849 u32 csum_start = skb_headroom(skb) + (u32)start;
4851 if (unlikely(csum_start > U16_MAX || csum_end > skb_headlen(skb))) {
4853 start, off, skb_headroom(skb), skb_headlen(skb));
4856 skb->ip_summed = CHECKSUM_PARTIAL;
4857 skb->csum_start = csum_start;
4858 skb->csum_offset = off;
4859 skb_set_transport_header(skb, start);
4864 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4867 if (skb_headlen(skb) >= len)
4873 if (max > skb->len)
4874 max = skb->len;
4876 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4879 if (skb_headlen(skb) < len)
4887 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4895 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4897 if (!err && !skb_partial_csum_set(skb, off,
4901 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4904 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4906 if (!err && !skb_partial_csum_set(skb, off,
4910 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4921 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
4930 err = skb_maybe_pull_tail(skb,
4936 if (ip_is_fragment(ip_hdr(skb)))
4939 off = ip_hdrlen(skb);
4946 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4951 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4952 ip_hdr(skb)->daddr,
4953 skb->len - off,
4954 ip_hdr(skb)->protocol, 0);
4966 #define OPT_HDR(type, skb, off) \
4967 (type *)(skb_network_header(skb) + (off))
4969 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4984 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
4988 nexthdr = ipv6_hdr(skb)->nexthdr;
4990 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
4998 err = skb_maybe_pull_tail(skb,
5005 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
5013 err = skb_maybe_pull_tail(skb,
5020 hp = OPT_HDR(struct ip_auth_hdr, skb, off);
5028 err = skb_maybe_pull_tail(skb,
5035 hp = OPT_HDR(struct frag_hdr, skb, off);
5055 csum = skb_checksum_setup_ip(skb, nexthdr, off);
5060 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
5061 &ipv6_hdr(skb)->daddr,
5062 skb->len - off, nexthdr, 0);
5071 * @skb: the skb to set up
5074 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
5078 switch (skb->protocol) {
5080 err = skb_checksum_setup_ipv4(skb, recalculate);
5084 err = skb_checksum_setup_ipv6(skb, recalculate);
5097 * skb_checksum_maybe_trim - maybe trims the given skb
5098 * @skb: the skb to check
5101 * Checks whether the given skb has data beyond the given transport length.
5102 * If so, returns a cloned skb trimmed to this transport length.
5103 * Otherwise returns the provided skb. Returns NULL in error cases
5104 * (e.g. transport_len exceeds skb length or out-of-memory).
5106 * Caller needs to set the skb transport header and free any returned skb if it
5107 * differs from the provided skb.
5109 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
5113 unsigned int len = skb_transport_offset(skb) + transport_len;
5116 if (skb->len < len)
5118 else if (skb->len == len)
5119 return skb;
5121 skb_chk = skb_clone(skb, GFP_ATOMIC);
5135 * skb_checksum_trimmed - validate checksum of an skb
5136 * @skb: the skb to check
5140 * Applies the given checksum function skb_chkf to the provided skb.
5141 * Returns a checked and maybe trimmed skb. Returns NULL on error.
5143 * If the skb has data beyond the given transport length, then a
5144 * trimmed & cloned skb is checked and returned.
5146 * Caller needs to set the skb transport header and free any returned skb if it
5147 * differs from the provided skb.
5149 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
5151 __sum16(*skb_chkf)(struct sk_buff *skb))
5154 unsigned int offset = skb_transport_offset(skb);
5157 skb_chk = skb_checksum_maybe_trim(skb, transport_len);
5174 if (skb_chk && skb_chk != skb)
5182 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
5185 skb->dev->name);
5189 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
5192 skb_release_head_state(skb);
5193 kmem_cache_free(skbuff_head_cache, skb);
5195 __kfree_skb(skb);
5201 * skb_try_coalesce - try to merge skb to prior one
5269 /* if the skb is not cloned this does nothing
5285 * skb_scrub_packet - scrub an skb
5287 * @skb: buffer to clean
5293 * skb_scrub_packet can also be used to clean a skb before injecting it in
5295 * skb that could impact namespace isolation.
5297 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
5299 skb->pkt_type = PACKET_HOST;
5300 skb->skb_iif = 0;
5301 skb->ignore_df = 0;
5302 skb_dst_drop(skb);
5303 skb_ext_reset(skb);
5304 nf_reset_ct(skb);
5305 nf_reset_trace(skb);
5308 skb->offload_fwd_mark = 0;
5309 skb->offload_l3_fwd_mark = 0;
5315 ipvs_reset(skb);
5316 skb->mark = 0;
5317 skb->tstamp = 0;
5324 * @skb: GSO skb
5331 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
5333 const struct skb_shared_info *shinfo = skb_shinfo(skb);
5336 if (skb->encapsulation) {
5337 thlen = skb_inner_transport_header(skb) -
5338 skb_transport_header(skb);
5341 thlen += inner_tcp_hdrlen(skb);
5343 thlen = tcp_hdrlen(skb);
5344 } else if (unlikely(skb_is_gso_sctp(skb))) {
5359 * @skb: GSO skb
5366 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
5368 unsigned int hdr_len = skb_transport_header(skb) -
5369 skb_network_header(skb);
5371 return hdr_len + skb_gso_transport_seglen(skb);
5377 * @skb: GSO skb
5383 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
5385 unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
5387 return hdr_len + skb_gso_transport_seglen(skb);
5391 * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
5393 * There are a couple of instances where we have a GSO skb, and we
5402 * @skb: GSO skb
5411 static inline bool skb_gso_size_check(const struct sk_buff *skb,
5414 const struct skb_shared_info *shinfo = skb_shinfo(skb);
5423 skb_walk_frags(skb, iter) {
5432 * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
5434 * @skb: GSO skb
5437 * skb_gso_validate_network_len validates if a given skb will fit a
5441 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
5443 return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
5448 * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
5450 * @skb: GSO skb
5453 * skb_gso_validate_mac_len validates if a given skb will fit a wanted
5456 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
5458 return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
5462 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5467 if (skb_cow(skb, skb_headroom(skb)) < 0) {
5468 kfree_skb(skb);
5472 mac_len = skb->data - skb_mac_header(skb);
5474 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5478 meta_len = skb_metadata_len(skb);
5480 meta = skb_metadata_end(skb) - meta_len;
5484 skb->mac_header += VLAN_HLEN;
5485 return skb;
5488 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5493 if (unlikely(skb_vlan_tag_present(skb))) {
5495 return skb;
5498 skb = skb_share_check(skb, GFP_ATOMIC);
5499 if (unlikely(!skb))
5502 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN + sizeof(unsigned short))))
5505 vhdr = (struct vlan_hdr *)skb->data;
5507 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5509 skb_pull_rcsum(skb, VLAN_HLEN);
5510 vlan_set_encap_proto(skb, vhdr);
5512 skb = skb_reorder_vlan_header(skb);
5513 if (unlikely(!skb))
5516 skb_reset_network_header(skb);
5517 skb_reset_transport_header(skb);
5518 skb_reset_mac_len(skb);
5520 return skb;
5523 kfree_skb(skb);
5528 int skb_ensure_writable(struct sk_buff *skb, int write_len)
5530 if (!pskb_may_pull(skb, write_len))
5533 if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5536 return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5541 * expects a non skb_vlan_tag_present skb with a vlan tag payload
5543 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5546 int offset = skb->data - skb_mac_header(skb);
5550 "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5555 err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5559 skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5561 vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5564 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5565 __skb_pull(skb, VLAN_HLEN);
5567 vlan_set_encap_proto(skb, vhdr);
5568 skb->mac_header += VLAN_HLEN;
5570 if (skb_network_offset(skb) < ETH_HLEN)
5571 skb_set_network_header(skb, ETH_HLEN);
5573 skb_reset_mac_len(skb);
5580 * Expects skb->data at mac header.
5582 int skb_vlan_pop(struct sk_buff *skb)
5588 if (likely(skb_vlan_tag_present(skb))) {
5589 __vlan_hwaccel_clear_tag(skb);
5591 if (unlikely(!eth_type_vlan(skb->protocol)))
5594 err = __skb_vlan_pop(skb, &vlan_tci);
5599 if (likely(!eth_type_vlan(skb->protocol)))
5602 vlan_proto = skb->protocol;
5603 err = __skb_vlan_pop(skb, &vlan_tci);
5607 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5613 * Expects skb->data at mac header.
5615 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5617 if (skb_vlan_tag_present(skb)) {
5618 int offset = skb->data - skb_mac_header(skb);
5622 "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5627 err = __vlan_insert_tag(skb, skb->vlan_proto,
5628 skb_vlan_tag_get(skb));
5632 skb->protocol = skb->vlan_proto;
5633 skb->mac_len += VLAN_HLEN;
5635 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5637 __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5645 * @skb: Socket buffer to modify
5647 * Drop the Ethernet header of @skb.
5649 * Expects that skb->data points to the mac header and that no VLAN tags are
5654 int skb_eth_pop(struct sk_buff *skb)
5656 if (!pskb_may_pull(skb, ETH_HLEN) || skb_vlan_tagged(skb) ||
5657 skb_network_offset(skb) < ETH_HLEN)
5660 skb_pull_rcsum(skb, ETH_HLEN);
5661 skb_reset_mac_header(skb);
5662 skb_reset_mac_len(skb);
5671 * @skb: Socket buffer to modify
5675 * Prepend @skb with a new Ethernet header.
5677 * Expects that skb->data points to the mac header, which must be empty.
5681 int skb_eth_push(struct sk_buff *skb, const unsigned char *dst,
5687 if (skb_network_offset(skb) || skb_vlan_tag_present(skb))
5690 err = skb_cow_head(skb, sizeof(*eth));
5694 skb_push(skb, sizeof(*eth));
5695 skb_reset_mac_header(skb);
5696 skb_reset_mac_len(skb);
5698 eth = eth_hdr(skb);
5701 eth->h_proto = skb->protocol;
5703 skb_postpush_rcsum(skb, eth, sizeof(*eth));
5709 /* Update the ethertype of hdr and the skb csum value if required. */
5710 static void skb_mod_eth_type(struct sk_buff *skb, struct ethhdr *hdr,
5713 if (skb->ip_summed == CHECKSUM_COMPLETE) {
5716 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5726 * @skb: buffer
5733 * Expects skb->data at mac header.
5737 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
5747 if (skb->encapsulation)
5750 err = skb_cow_head(skb, MPLS_HLEN);
5754 if (!skb->inner_protocol) {
5755 skb_set_inner_network_header(skb, skb_network_offset(skb));
5756 skb_set_inner_protocol(skb, skb->protocol);
5759 skb_push(skb, MPLS_HLEN);
5760 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
5762 skb_reset_mac_header(skb);
5763 skb_set_network_header(skb, mac_len);
5764 skb_reset_mac_len(skb);
5766 lse = mpls_hdr(skb);
5768 skb_postpush_rcsum(skb, lse, MPLS_HLEN);
5771 skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
5772 skb->protocol = mpls_proto;
5781 * @skb: buffer
5786 * Expects skb->data at mac header.
5790 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
5795 if (unlikely(!eth_p_mpls(skb->protocol)))
5798 err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
5802 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
5803 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
5806 __skb_pull(skb, MPLS_HLEN);
5807 skb_reset_mac_header(skb);
5808 skb_set_network_header(skb, mac_len);
5814 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
5815 skb_mod_eth_type(skb, hdr, next_proto);
5817 skb->protocol = next_proto;
5826 * @skb: buffer
5829 * Expects skb->data at mac header.
5833 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
5837 if (unlikely(!eth_p_mpls(skb->protocol)))
5840 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
5844 if (skb->ip_summed == CHECKSUM_COMPLETE) {
5845 __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
5847 skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
5850 mpls_hdr(skb)->label_stack_entry = mpls_lse;
5859 * @skb: buffer
5861 * Expects skb->data at mac header.
5865 int skb_mpls_dec_ttl(struct sk_buff *skb)
5870 if (unlikely(!eth_p_mpls(skb->protocol)))
5873 if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
5876 lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
5884 return skb_mpls_update_lse(skb, cpu_to_be32(lse));
5889 * alloc_skb_with_frags - allocate skb with page frags
5897 * This can be used to allocate a paged skb, given a maximal order for frags.
5907 struct sk_buff *skb;
5919 skb = alloc_skb(header_len, gfp_mask);
5920 if (!skb)
5923 skb->truesize += npages << PAGE_SHIFT;
5948 skb_fill_page_desc(skb, i, page, 0, chunk);
5952 return skb;
5955 kfree_skb(skb);
5960 /* carve out the first off bytes from skb when off < headlen */
5961 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5965 int size = skb_end_offset(skb);
5971 if (skb_pfmemalloc(skb))
5982 skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
5983 skb->len -= off;
5986 skb_shinfo(skb),
5988 frags[skb_shinfo(skb)->nr_frags]));
5989 if (skb_cloned(skb)) {
5991 if (skb_orphan_frags(skb, gfp_mask)) {
5995 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
5996 skb_frag_ref(skb, i);
5997 if (skb_has_frag_list(skb))
5998 skb_clone_fraglist(skb);
5999 skb_release_data(skb);
6004 skb_free_head(skb);
6007 skb->head = data;
6008 skb->data = data;
6009 skb->head_frag = 0;
6011 skb->end = size;
6013 skb->end = skb->head + size;
6015 skb_set_tail_pointer(skb, skb_headlen(skb));
6016 skb_headers_offset_update(skb, 0);
6017 skb->cloned = 0;
6018 skb->hdr_len = 0;
6019 skb->nohdr = 0;
6020 atomic_set(&skb_shinfo(skb)->dataref, 1);
6025 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
6027 /* carve out the first eat bytes from skb's frag_list. May recurse into
6030 static int pskb_carve_frag_list(struct sk_buff *skb,
6081 /* carve off first len bytes from skb. Split line (off) is in the
6082 * non-linear part of skb
6084 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
6088 int size = skb_end_offset(skb);
6090 const int nfrags = skb_shinfo(skb)->nr_frags;
6095 if (skb_pfmemalloc(skb))
6106 skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
6107 if (skb_orphan_frags(skb, gfp_mask)) {
6113 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
6116 shinfo->frags[k] = skb_shinfo(skb)->frags[i];
6130 skb_frag_ref(skb, i);
6136 if (skb_has_frag_list(skb))
6137 skb_clone_fraglist(skb);
6140 if (k == 0 && pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask)) {
6142 if (skb_has_frag_list(skb))
6143 kfree_skb_list(skb_shinfo(skb)->frag_list);
6147 skb_release_data(skb);
6149 skb->head = data;
6150 skb->head_frag = 0;
6151 skb->data = data;
6153 skb->end = size;
6155 skb->end = skb->head + size;
6157 skb_reset_tail_pointer(skb);
6158 skb_headers_offset_update(skb, 0);
6159 skb->cloned = 0;
6160 skb->hdr_len = 0;
6161 skb->nohdr = 0;
6162 skb->len -= off;
6163 skb->data_len = skb->len;
6164 atomic_set(&skb_shinfo(skb)->dataref, 1);
6168 /* remove len bytes from the beginning of the skb */
6169 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
6171 int headlen = skb_headlen(skb);
6174 return pskb_carve_inside_header(skb, len, headlen, gfp);
6176 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
6179 /* Extract to_copy bytes starting at off from skb, and return this in
6180 * a new skb
6182 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
6185 struct sk_buff *clone = skb_clone(skb, gfp);
6201 * @skb: buffer
6203 * Can be used to save memory before skb is added to a busy queue.
6204 * If packet has bytes in frags and enough tail room in skb->head,
6208 * We do not reallocate skb->head thus can not fail.
6209 * Caller must re-evaluate skb->truesize if needed.
6211 void skb_condense(struct sk_buff *skb)
6213 if (skb->data_len) {
6214 if (skb->data_len > skb->end - skb->tail ||
6215 skb_cloned(skb))
6219 __pskb_pull_tail(skb, skb->data_len);
6221 /* At this point, skb->truesize might be over estimated,
6222 * because skb had a fragment, and fragments do not tell
6224 * When we pulled its content into skb->head, fragment
6226 * adjust skb->truesize, not knowing the frag truesize.
6228 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
6238 * __skb_ext_alloc - allocate a new skb extensions storage
6243 * skb via __skb_ext_set().
6287 * __skb_ext_set - attach the specified extension storage to this skb
6288 * @skb: buffer
6296 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
6301 skb_ext_put(skb);
6305 skb->extensions = ext;
6306 skb->active_extensions = 1 << id;
6312 * @skb: buffer
6319 * If the skb was cloned, COW applies and the returned memory can be
6324 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)
6329 if (skb->active_extensions) {
6330 old = skb->extensions;
6332 new = skb_ext_maybe_cow(old, skb->active_extensions);
6352 skb->extensions = new;
6353 skb->active_extensions |= 1 << id;
6368 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
6370 struct skb_ext *ext = skb->extensions;
6372 skb->active_extensions &= ~(1 << id);
6373 if (skb->active_extensions == 0) {
6374 skb->extensions = NULL;