1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * IPV6 GSO/GRO offload support
4 * Linux INET implementation
5 *
6 * Copyright (C) 2016 secunet Security Networks AG
7 * Author: Steffen Klassert <steffen.klassert@secunet.com>
8 *
9 * ESP GRO support
10 */
11
12#include <linux/skbuff.h>
13#include <linux/init.h>
14#include <net/protocol.h>
15#include <crypto/aead.h>
16#include <crypto/authenc.h>
17#include <linux/err.h>
18#include <linux/module.h>
19#include <net/ip.h>
20#include <net/xfrm.h>
21#include <net/esp.h>
22#include <linux/scatterlist.h>
23#include <linux/kernel.h>
24#include <linux/slab.h>
25#include <linux/spinlock.h>
26#include <net/ip6_route.h>
27#include <net/ipv6.h>
28#include <linux/icmpv6.h>
29
30static __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen)
31{
32	int off = sizeof(struct ipv6hdr);
33	struct ipv6_opt_hdr *exthdr;
34
35	if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP))
36		return offsetof(struct ipv6hdr, nexthdr);
37
38	while (off < nhlen) {
39		exthdr = (void *)ipv6_hdr + off;
40		if (exthdr->nexthdr == NEXTHDR_ESP)
41			return off;
42
43		off += ipv6_optlen(exthdr);
44	}
45
46	return 0;
47}
48
49static struct sk_buff *esp6_gro_receive(struct list_head *head,
50					struct sk_buff *skb)
51{
52	int offset = skb_gro_offset(skb);
53	struct xfrm_offload *xo;
54	struct xfrm_state *x;
55	__be32 seq;
56	__be32 spi;
57	int nhoff;
58	int err;
59
60	if (!pskb_pull(skb, offset))
61		return NULL;
62
63	if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
64		goto out;
65
66	xo = xfrm_offload(skb);
67	if (!xo || !(xo->flags & CRYPTO_DONE)) {
68		struct sec_path *sp = secpath_set(skb);
69
70		if (!sp)
71			goto out;
72
73		if (sp->len == XFRM_MAX_DEPTH)
74			goto out_reset;
75
76		x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
77				      (xfrm_address_t *)&ipv6_hdr(skb)->daddr,
78				      spi, IPPROTO_ESP, AF_INET6);
79		if (!x)
80			goto out_reset;
81
82		skb->mark = xfrm_smark_get(skb->mark, x);
83
84		sp->xvec[sp->len++] = x;
85		sp->olen++;
86
87		xo = xfrm_offload(skb);
88		if (!xo)
89			goto out_reset;
90	}
91
92	xo->flags |= XFRM_GRO;
93
94	nhoff = esp6_nexthdr_esp_offset(ipv6_hdr(skb), offset);
95	if (!nhoff)
96		goto out;
97
98	IP6CB(skb)->nhoff = nhoff;
99	XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
100	XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
101	XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
102	XFRM_SPI_SKB_CB(skb)->seq = seq;
103
104	/* We don't need to handle errors from xfrm_input, it does all
105	 * the error handling and frees the resources on error. */
106	xfrm_input(skb, IPPROTO_ESP, spi, -2);
107
108	return ERR_PTR(-EINPROGRESS);
109out_reset:
110	secpath_reset(skb);
111out:
112	skb_push(skb, offset);
113	NAPI_GRO_CB(skb)->same_flow = 0;
114	NAPI_GRO_CB(skb)->flush = 1;
115
116	return NULL;
117}
118
119static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
120{
121	struct ip_esp_hdr *esph;
122	struct ipv6hdr *iph = ipv6_hdr(skb);
123	struct xfrm_offload *xo = xfrm_offload(skb);
124	u8 proto = iph->nexthdr;
125
126	skb_push(skb, -skb_network_offset(skb));
127
128	if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) {
129		__be16 frag;
130
131		ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &proto, &frag);
132	}
133
134	esph = ip_esp_hdr(skb);
135	*skb_mac_header(skb) = IPPROTO_ESP;
136
137	esph->spi = x->id.spi;
138	esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
139
140	xo->proto = proto;
141}
142
143static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
144						struct sk_buff *skb,
145						netdev_features_t features)
146{
147	__skb_push(skb, skb->mac_len);
148	return skb_mac_gso_segment(skb, features);
149}
150
151static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x,
152						   struct sk_buff *skb,
153						   netdev_features_t features)
154{
155	const struct net_offload *ops;
156	struct sk_buff *segs = ERR_PTR(-EINVAL);
157	struct xfrm_offload *xo = xfrm_offload(skb);
158
159	skb->transport_header += x->props.header_len;
160	ops = rcu_dereference(inet6_offloads[xo->proto]);
161	if (likely(ops && ops->callbacks.gso_segment))
162		segs = ops->callbacks.gso_segment(skb, features);
163
164	return segs;
165}
166
167static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x,
168					      struct sk_buff *skb,
169					      netdev_features_t features)
170{
171	struct xfrm_offload *xo = xfrm_offload(skb);
172	struct sk_buff *segs = ERR_PTR(-EINVAL);
173	const struct net_offload *ops;
174	u8 proto = xo->proto;
175
176	skb->transport_header += x->props.header_len;
177
178	if (x->sel.family != AF_INET6) {
179		skb->transport_header -=
180			(sizeof(struct ipv6hdr) - sizeof(struct iphdr));
181
182		if (proto == IPPROTO_BEETPH) {
183			struct ip_beet_phdr *ph =
184				(struct ip_beet_phdr *)skb->data;
185
186			skb->transport_header += ph->hdrlen * 8;
187			proto = ph->nexthdr;
188		} else {
189			skb->transport_header -= IPV4_BEET_PHMAXLEN;
190		}
191
192		if (proto == IPPROTO_TCP)
193			skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
194	} else {
195		__be16 frag;
196
197		skb->transport_header +=
198			ipv6_skip_exthdr(skb, 0, &proto, &frag);
199	}
200
201	if (proto == IPPROTO_IPIP)
202		skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
203
204	__skb_pull(skb, skb_transport_offset(skb));
205	ops = rcu_dereference(inet6_offloads[proto]);
206	if (likely(ops && ops->callbacks.gso_segment))
207		segs = ops->callbacks.gso_segment(skb, features);
208
209	return segs;
210}
211
212static struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x,
213						    struct sk_buff *skb,
214						    netdev_features_t features)
215{
216	switch (x->outer_mode.encap) {
217	case XFRM_MODE_TUNNEL:
218		return xfrm6_tunnel_gso_segment(x, skb, features);
219	case XFRM_MODE_TRANSPORT:
220		return xfrm6_transport_gso_segment(x, skb, features);
221	case XFRM_MODE_BEET:
222		return xfrm6_beet_gso_segment(x, skb, features);
223	}
224
225	return ERR_PTR(-EOPNOTSUPP);
226}
227
228static struct sk_buff *esp6_gso_segment(struct sk_buff *skb,
229				        netdev_features_t features)
230{
231	struct xfrm_state *x;
232	struct ip_esp_hdr *esph;
233	struct crypto_aead *aead;
234	netdev_features_t esp_features = features;
235	struct xfrm_offload *xo = xfrm_offload(skb);
236	struct sec_path *sp;
237
238	if (!xo)
239		return ERR_PTR(-EINVAL);
240
241	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP))
242		return ERR_PTR(-EINVAL);
243
244	sp = skb_sec_path(skb);
245	x = sp->xvec[sp->len - 1];
246	aead = x->data;
247	esph = ip_esp_hdr(skb);
248
249	if (esph->spi != x->id.spi)
250		return ERR_PTR(-EINVAL);
251
252	if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))
253		return ERR_PTR(-EINVAL);
254
255	__skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));
256
257	skb->encap_hdr_csum = 1;
258
259	if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev)
260		esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK |
261					    NETIF_F_SCTP_CRC);
262	else if (!(features & NETIF_F_HW_ESP_TX_CSUM))
263		esp_features = features & ~(NETIF_F_CSUM_MASK |
264					    NETIF_F_SCTP_CRC);
265
266	xo->flags |= XFRM_GSO_SEGMENT;
267
268	return xfrm6_outer_mode_gso_segment(x, skb, esp_features);
269}
270
271static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb)
272{
273	struct crypto_aead *aead = x->data;
274	struct xfrm_offload *xo = xfrm_offload(skb);
275
276	if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead)))
277		return -EINVAL;
278
279	if (!(xo->flags & CRYPTO_DONE))
280		skb->ip_summed = CHECKSUM_NONE;
281
282	return esp6_input_done2(skb, 0);
283}
284
285static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb,  netdev_features_t features)
286{
287	int len;
288	int err;
289	int alen;
290	int blksize;
291	struct xfrm_offload *xo;
292	struct crypto_aead *aead;
293	struct esp_info esp;
294	bool hw_offload = true;
295	__u32 seq;
296
297	esp.inplace = true;
298
299	xo = xfrm_offload(skb);
300
301	if (!xo)
302		return -EINVAL;
303
304	if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) {
305		xo->flags |= CRYPTO_FALLBACK;
306		hw_offload = false;
307	}
308
309	esp.proto = xo->proto;
310
311	/* skb is pure payload to encrypt */
312
313	aead = x->data;
314	alen = crypto_aead_authsize(aead);
315
316	esp.tfclen = 0;
317	/* XXX: Add support for tfc padding here. */
318
319	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
320	esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
321	esp.plen = esp.clen - skb->len - esp.tfclen;
322	esp.tailen = esp.tfclen + esp.plen + alen;
323
324	if (!hw_offload || (hw_offload && !skb_is_gso(skb))) {
325		esp.nfrags = esp6_output_head(x, skb, &esp);
326		if (esp.nfrags < 0)
327			return esp.nfrags;
328	}
329
330	seq = xo->seq.low;
331
332	esp.esph = ip_esp_hdr(skb);
333	esp.esph->spi = x->id.spi;
334
335	skb_push(skb, -skb_network_offset(skb));
336
337	if (xo->flags & XFRM_GSO_SEGMENT) {
338		esp.esph->seq_no = htonl(seq);
339
340		if (!skb_is_gso(skb))
341			xo->seq.low++;
342		else
343			xo->seq.low += skb_shinfo(skb)->gso_segs;
344	}
345
346	if (xo->seq.low < seq)
347		xo->seq.hi++;
348
349	esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));
350
351	len = skb->len - sizeof(struct ipv6hdr);
352	if (len > IPV6_MAXPLEN)
353		len = 0;
354
355	ipv6_hdr(skb)->payload_len = htons(len);
356
357	if (hw_offload) {
358		if (!skb_ext_add(skb, SKB_EXT_SEC_PATH))
359			return -ENOMEM;
360
361		xo = xfrm_offload(skb);
362		if (!xo)
363			return -EINVAL;
364
365		xo->flags |= XFRM_XMIT;
366		return 0;
367	}
368
369	err = esp6_output_tail(x, skb, &esp);
370	if (err)
371		return err;
372
373	secpath_reset(skb);
374
375	if (skb_needs_linearize(skb, skb->dev->features) &&
376	    __skb_linearize(skb))
377		return -ENOMEM;
378	return 0;
379}
380
381static const struct net_offload esp6_offload = {
382	.callbacks = {
383		.gro_receive = esp6_gro_receive,
384		.gso_segment = esp6_gso_segment,
385	},
386};
387
388static const struct xfrm_type_offload esp6_type_offload = {
389	.description	= "ESP6 OFFLOAD",
390	.owner		= THIS_MODULE,
391	.proto	     	= IPPROTO_ESP,
392	.input_tail	= esp6_input_tail,
393	.xmit		= esp6_xmit,
394	.encap		= esp6_gso_encap,
395};
396
397static int __init esp6_offload_init(void)
398{
399	if (xfrm_register_type_offload(&esp6_type_offload, AF_INET6) < 0) {
400		pr_info("%s: can't add xfrm type offload\n", __func__);
401		return -EAGAIN;
402	}
403
404	return inet6_add_offload(&esp6_offload, IPPROTO_ESP);
405}
406
407static void __exit esp6_offload_exit(void)
408{
409	xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6);
410	inet6_del_offload(&esp6_offload, IPPROTO_ESP);
411}
412
413module_init(esp6_offload_init);
414module_exit(esp6_offload_exit);
415MODULE_LICENSE("GPL");
416MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
417MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP);
418MODULE_DESCRIPTION("IPV6 GSO/GRO offload support");
419