1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * IPV6 GSO/GRO offload support 4 * Linux INET6 implementation 5 */ 6 7#include <linux/kernel.h> 8#include <linux/socket.h> 9#include <linux/netdevice.h> 10#include <linux/skbuff.h> 11#include <linux/printk.h> 12 13#include <net/protocol.h> 14#include <net/ipv6.h> 15#include <net/inet_common.h> 16#include <net/tcp.h> 17#include <net/udp.h> 18 19#include "ip6_offload.h" 20 21/* All GRO functions are always builtin, except UDP over ipv6, which lays in 22 * ipv6 module, as it depends on UDPv6 lookup function, so we need special care 23 * when ipv6 is built as a module 24 */ 25#if IS_BUILTIN(CONFIG_IPV6) 26#define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__) 27#else 28#define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__) 29#endif 30 31#define indirect_call_gro_receive_l4(f2, f1, cb, head, skb) \ 32({ \ 33 unlikely(gro_recursion_inc_test(skb)) ? \ 34 NAPI_GRO_CB(skb)->flush |= 1, NULL : \ 35 INDIRECT_CALL_L4(cb, f2, f1, head, skb); \ 36}) 37 38static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto) 39{ 40 const struct net_offload *ops = NULL; 41 42 for (;;) { 43 struct ipv6_opt_hdr *opth; 44 int len; 45 46 if (proto != NEXTHDR_HOP) { 47 ops = rcu_dereference(inet6_offloads[proto]); 48 49 if (unlikely(!ops)) 50 break; 51 52 if (!(ops->flags & INET6_PROTO_GSO_EXTHDR)) 53 break; 54 } 55 56 if (unlikely(!pskb_may_pull(skb, 8))) 57 break; 58 59 opth = (void *)skb->data; 60 len = ipv6_optlen(opth); 61 62 if (unlikely(!pskb_may_pull(skb, len))) 63 break; 64 65 opth = (void *)skb->data; 66 proto = opth->nexthdr; 67 __skb_pull(skb, len); 68 } 69 70 return proto; 71} 72 73static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb, 74 netdev_features_t features) 75{ 76 struct sk_buff *segs = ERR_PTR(-EINVAL); 77 struct ipv6hdr *ipv6h; 78 const struct net_offload *ops; 79 int proto; 80 struct frag_hdr *fptr; 81 unsigned int payload_len; 82 u8 *prevhdr; 83 int offset = 0; 84 bool encap, udpfrag; 85 int nhoff; 86 bool gso_partial; 87 88 skb_reset_network_header(skb); 89 nhoff = skb_network_header(skb) - skb_mac_header(skb); 90 if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h)))) 91 goto out; 92 93 encap = SKB_GSO_CB(skb)->encap_level > 0; 94 if (encap) 95 features &= skb->dev->hw_enc_features; 96 SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h); 97 98 ipv6h = ipv6_hdr(skb); 99 __skb_pull(skb, sizeof(*ipv6h)); 100 segs = ERR_PTR(-EPROTONOSUPPORT); 101 102 proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr); 103 104 if (skb->encapsulation && 105 skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6)) 106 udpfrag = proto == IPPROTO_UDP && encap && 107 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP); 108 else 109 udpfrag = proto == IPPROTO_UDP && !skb->encapsulation && 110 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP); 111 112 ops = rcu_dereference(inet6_offloads[proto]); 113 if (likely(ops && ops->callbacks.gso_segment)) { 114 skb_reset_transport_header(skb); 115 segs = ops->callbacks.gso_segment(skb, features); 116 if (!segs) 117 skb->network_header = skb_mac_header(skb) + nhoff - skb->head; 118 } 119 120 if (IS_ERR_OR_NULL(segs)) 121 goto out; 122 123 gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL); 124 125 for (skb = segs; skb; skb = skb->next) { 126 ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff); 127 if (gso_partial && skb_is_gso(skb)) 128 payload_len = skb_shinfo(skb)->gso_size + 129 SKB_GSO_CB(skb)->data_offset + 130 skb->head - (unsigned char *)(ipv6h + 1); 131 else 132 payload_len = skb->len - nhoff - sizeof(*ipv6h); 133 ipv6h->payload_len = htons(payload_len); 134 skb->network_header = (u8 *)ipv6h - skb->head; 135 skb_reset_mac_len(skb); 136 137 if (udpfrag) { 138 int err = ip6_find_1stfragopt(skb, &prevhdr); 139 if (err < 0) { 140 kfree_skb_list(segs); 141 return ERR_PTR(err); 142 } 143 fptr = (struct frag_hdr *)((u8 *)ipv6h + err); 144 fptr->frag_off = htons(offset); 145 if (skb->next) 146 fptr->frag_off |= htons(IP6_MF); 147 offset += (ntohs(ipv6h->payload_len) - 148 sizeof(struct frag_hdr)); 149 } 150 if (encap) 151 skb_reset_inner_headers(skb); 152 } 153 154out: 155 return segs; 156} 157 158/* Return the total length of all the extension hdrs, following the same 159 * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs. 160 */ 161static int ipv6_exthdrs_len(struct ipv6hdr *iph, 162 const struct net_offload **opps) 163{ 164 struct ipv6_opt_hdr *opth = (void *)iph; 165 int len = 0, proto, optlen = sizeof(*iph); 166 167 proto = iph->nexthdr; 168 for (;;) { 169 if (proto != NEXTHDR_HOP) { 170 *opps = rcu_dereference(inet6_offloads[proto]); 171 if (unlikely(!(*opps))) 172 break; 173 if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR)) 174 break; 175 } 176 opth = (void *)opth + optlen; 177 optlen = ipv6_optlen(opth); 178 len += optlen; 179 proto = opth->nexthdr; 180 } 181 return len; 182} 183 184INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head, 185 struct sk_buff *skb) 186{ 187 const struct net_offload *ops; 188 struct sk_buff *pp = NULL; 189 struct sk_buff *p; 190 struct ipv6hdr *iph; 191 unsigned int nlen; 192 unsigned int hlen; 193 unsigned int off; 194 u16 flush = 1; 195 int proto; 196 197 off = skb_gro_offset(skb); 198 hlen = off + sizeof(*iph); 199 iph = skb_gro_header_fast(skb, off); 200 if (skb_gro_header_hard(skb, hlen)) { 201 iph = skb_gro_header_slow(skb, hlen, off); 202 if (unlikely(!iph)) 203 goto out; 204 } 205 206 skb_set_network_header(skb, off); 207 skb_gro_pull(skb, sizeof(*iph)); 208 skb_set_transport_header(skb, skb_gro_offset(skb)); 209 210 flush += ntohs(iph->payload_len) != skb_gro_len(skb); 211 212 rcu_read_lock(); 213 proto = iph->nexthdr; 214 ops = rcu_dereference(inet6_offloads[proto]); 215 if (!ops || !ops->callbacks.gro_receive) { 216 __pskb_pull(skb, skb_gro_offset(skb)); 217 skb_gro_frag0_invalidate(skb); 218 proto = ipv6_gso_pull_exthdrs(skb, proto); 219 skb_gro_pull(skb, -skb_transport_offset(skb)); 220 skb_reset_transport_header(skb); 221 __skb_push(skb, skb_gro_offset(skb)); 222 223 ops = rcu_dereference(inet6_offloads[proto]); 224 if (!ops || !ops->callbacks.gro_receive) 225 goto out_unlock; 226 227 iph = ipv6_hdr(skb); 228 } 229 230 NAPI_GRO_CB(skb)->proto = proto; 231 232 flush--; 233 nlen = skb_network_header_len(skb); 234 235 list_for_each_entry(p, head, list) { 236 const struct ipv6hdr *iph2; 237 __be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */ 238 239 if (!NAPI_GRO_CB(p)->same_flow) 240 continue; 241 242 iph2 = (struct ipv6hdr *)(p->data + off); 243 first_word = *(__be32 *)iph ^ *(__be32 *)iph2; 244 245 /* All fields must match except length and Traffic Class. 246 * XXX skbs on the gro_list have all been parsed and pulled 247 * already so we don't need to compare nlen 248 * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops))) 249 * memcmp() alone below is sufficient, right? 250 */ 251 if ((first_word & htonl(0xF00FFFFF)) || 252 !ipv6_addr_equal(&iph->saddr, &iph2->saddr) || 253 !ipv6_addr_equal(&iph->daddr, &iph2->daddr) || 254 *(u16 *)&iph->nexthdr != *(u16 *)&iph2->nexthdr) { 255not_same_flow: 256 NAPI_GRO_CB(p)->same_flow = 0; 257 continue; 258 } 259 if (unlikely(nlen > sizeof(struct ipv6hdr))) { 260 if (memcmp(iph + 1, iph2 + 1, 261 nlen - sizeof(struct ipv6hdr))) 262 goto not_same_flow; 263 } 264 /* flush if Traffic Class fields are different */ 265 NAPI_GRO_CB(p)->flush |= !!(first_word & htonl(0x0FF00000)); 266 NAPI_GRO_CB(p)->flush |= flush; 267 268 /* If the previous IP ID value was based on an atomic 269 * datagram we can overwrite the value and ignore it. 270 */ 271 if (NAPI_GRO_CB(skb)->is_atomic) 272 NAPI_GRO_CB(p)->flush_id = 0; 273 } 274 275 NAPI_GRO_CB(skb)->is_atomic = true; 276 NAPI_GRO_CB(skb)->flush |= flush; 277 278 skb_gro_postpull_rcsum(skb, iph, nlen); 279 280 pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive, 281 ops->callbacks.gro_receive, head, skb); 282 283out_unlock: 284 rcu_read_unlock(); 285 286out: 287 skb_gro_flush_final(skb, pp, flush); 288 289 return pp; 290} 291 292static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head, 293 struct sk_buff *skb) 294{ 295 /* Common GRO receive for SIT and IP6IP6 */ 296 297 if (NAPI_GRO_CB(skb)->encap_mark) { 298 NAPI_GRO_CB(skb)->flush = 1; 299 return NULL; 300 } 301 302 NAPI_GRO_CB(skb)->encap_mark = 1; 303 304 return ipv6_gro_receive(head, skb); 305} 306 307static struct sk_buff *ip4ip6_gro_receive(struct list_head *head, 308 struct sk_buff *skb) 309{ 310 /* Common GRO receive for SIT and IP6IP6 */ 311 312 if (NAPI_GRO_CB(skb)->encap_mark) { 313 NAPI_GRO_CB(skb)->flush = 1; 314 return NULL; 315 } 316 317 NAPI_GRO_CB(skb)->encap_mark = 1; 318 319 return inet_gro_receive(head, skb); 320} 321 322INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff) 323{ 324 const struct net_offload *ops; 325 struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + nhoff); 326 int err = -ENOSYS; 327 328 if (skb->encapsulation) { 329 skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6)); 330 skb_set_inner_network_header(skb, nhoff); 331 } 332 333 iph->payload_len = htons(skb->len - nhoff - sizeof(*iph)); 334 335 rcu_read_lock(); 336 337 nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops); 338 if (WARN_ON(!ops || !ops->callbacks.gro_complete)) 339 goto out_unlock; 340 341 err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete, 342 udp6_gro_complete, skb, nhoff); 343 344out_unlock: 345 rcu_read_unlock(); 346 347 return err; 348} 349 350static int sit_gro_complete(struct sk_buff *skb, int nhoff) 351{ 352 skb->encapsulation = 1; 353 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4; 354 return ipv6_gro_complete(skb, nhoff); 355} 356 357static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff) 358{ 359 skb->encapsulation = 1; 360 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6; 361 return ipv6_gro_complete(skb, nhoff); 362} 363 364static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff) 365{ 366 skb->encapsulation = 1; 367 skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6; 368 return inet_gro_complete(skb, nhoff); 369} 370 371static struct packet_offload ipv6_packet_offload __read_mostly = { 372 .type = cpu_to_be16(ETH_P_IPV6), 373 .callbacks = { 374 .gso_segment = ipv6_gso_segment, 375 .gro_receive = ipv6_gro_receive, 376 .gro_complete = ipv6_gro_complete, 377 }, 378}; 379 380static struct sk_buff *sit_gso_segment(struct sk_buff *skb, 381 netdev_features_t features) 382{ 383 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4)) 384 return ERR_PTR(-EINVAL); 385 386 return ipv6_gso_segment(skb, features); 387} 388 389static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb, 390 netdev_features_t features) 391{ 392 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6)) 393 return ERR_PTR(-EINVAL); 394 395 return inet_gso_segment(skb, features); 396} 397 398static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb, 399 netdev_features_t features) 400{ 401 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6)) 402 return ERR_PTR(-EINVAL); 403 404 return ipv6_gso_segment(skb, features); 405} 406 407static const struct net_offload sit_offload = { 408 .callbacks = { 409 .gso_segment = sit_gso_segment, 410 .gro_receive = sit_ip6ip6_gro_receive, 411 .gro_complete = sit_gro_complete, 412 }, 413}; 414 415static const struct net_offload ip4ip6_offload = { 416 .callbacks = { 417 .gso_segment = ip4ip6_gso_segment, 418 .gro_receive = ip4ip6_gro_receive, 419 .gro_complete = ip4ip6_gro_complete, 420 }, 421}; 422 423static const struct net_offload ip6ip6_offload = { 424 .callbacks = { 425 .gso_segment = ip6ip6_gso_segment, 426 .gro_receive = sit_ip6ip6_gro_receive, 427 .gro_complete = ip6ip6_gro_complete, 428 }, 429}; 430static int __init ipv6_offload_init(void) 431{ 432 433 if (tcpv6_offload_init() < 0) 434 pr_crit("%s: Cannot add TCP protocol offload\n", __func__); 435 if (ipv6_exthdrs_offload_init() < 0) 436 pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__); 437 438 dev_add_offload(&ipv6_packet_offload); 439 440 inet_add_offload(&sit_offload, IPPROTO_IPV6); 441 inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6); 442 inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP); 443 444 return 0; 445} 446 447fs_initcall(ipv6_offload_init); 448