1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * IPV4 GSO/GRO offload support
4 * Linux INET implementation
5 *
6 * Copyright (C) 2016 secunet Security Networks AG
7 * Author: Steffen Klassert <steffen.klassert@secunet.com>
8 *
9 * ESP GRO support
10 */
11
12#include <linux/skbuff.h>
13#include <linux/init.h>
14#include <net/protocol.h>
15#include <crypto/aead.h>
16#include <crypto/authenc.h>
17#include <linux/err.h>
18#include <linux/module.h>
19#include <net/ip.h>
20#include <net/xfrm.h>
21#include <net/esp.h>
22#include <linux/scatterlist.h>
23#include <linux/kernel.h>
24#include <linux/slab.h>
25#include <linux/spinlock.h>
26#include <net/udp.h>
27
28static struct sk_buff *esp4_gro_receive(struct list_head *head,
29					struct sk_buff *skb)
30{
31	int offset = skb_gro_offset(skb);
32	struct xfrm_offload *xo;
33	struct xfrm_state *x;
34	__be32 seq;
35	__be32 spi;
36	int err;
37
38	if (!pskb_pull(skb, offset))
39		return NULL;
40
41	if ((err = xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq)) != 0)
42		goto out;
43
44	xo = xfrm_offload(skb);
45	if (!xo || !(xo->flags & CRYPTO_DONE)) {
46		struct sec_path *sp = secpath_set(skb);
47
48		if (!sp)
49			goto out;
50
51		if (sp->len == XFRM_MAX_DEPTH)
52			goto out_reset;
53
54		x = xfrm_state_lookup(dev_net(skb->dev), skb->mark,
55				      (xfrm_address_t *)&ip_hdr(skb)->daddr,
56				      spi, IPPROTO_ESP, AF_INET);
57		if (!x)
58			goto out_reset;
59
60		skb->mark = xfrm_smark_get(skb->mark, x);
61
62		sp->xvec[sp->len++] = x;
63		sp->olen++;
64
65		xo = xfrm_offload(skb);
66		if (!xo)
67			goto out_reset;
68	}
69
70	xo->flags |= XFRM_GRO;
71
72	XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL;
73	XFRM_SPI_SKB_CB(skb)->family = AF_INET;
74	XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr);
75	XFRM_SPI_SKB_CB(skb)->seq = seq;
76
77	/* We don't need to handle errors from xfrm_input, it does all
78	 * the error handling and frees the resources on error. */
79	xfrm_input(skb, IPPROTO_ESP, spi, -2);
80
81	return ERR_PTR(-EINPROGRESS);
82out_reset:
83	secpath_reset(skb);
84out:
85	skb_push(skb, offset);
86	NAPI_GRO_CB(skb)->same_flow = 0;
87	NAPI_GRO_CB(skb)->flush = 1;
88
89	return NULL;
90}
91
92static void esp4_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
93{
94	struct ip_esp_hdr *esph;
95	struct iphdr *iph = ip_hdr(skb);
96	struct xfrm_offload *xo = xfrm_offload(skb);
97	int proto = iph->protocol;
98
99	skb_push(skb, -skb_network_offset(skb));
100	esph = ip_esp_hdr(skb);
101	*skb_mac_header(skb) = IPPROTO_ESP;
102
103	esph->spi = x->id.spi;
104	esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
105
106	xo->proto = proto;
107}
108
109static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x,
110						struct sk_buff *skb,
111						netdev_features_t features)
112{
113	__skb_push(skb, skb->mac_len);
114	return skb_mac_gso_segment(skb, features);
115}
116
117static struct sk_buff *xfrm4_transport_gso_segment(struct xfrm_state *x,
118						   struct sk_buff *skb,
119						   netdev_features_t features)
120{
121	const struct net_offload *ops;
122	struct sk_buff *segs = ERR_PTR(-EINVAL);
123	struct xfrm_offload *xo = xfrm_offload(skb);
124
125	skb->transport_header += x->props.header_len;
126	ops = rcu_dereference(inet_offloads[xo->proto]);
127	if (likely(ops && ops->callbacks.gso_segment))
128		segs = ops->callbacks.gso_segment(skb, features);
129
130	return segs;
131}
132
133static struct sk_buff *xfrm4_beet_gso_segment(struct xfrm_state *x,
134					      struct sk_buff *skb,
135					      netdev_features_t features)
136{
137	struct xfrm_offload *xo = xfrm_offload(skb);
138	struct sk_buff *segs = ERR_PTR(-EINVAL);
139	const struct net_offload *ops;
140	u8 proto = xo->proto;
141
142	skb->transport_header += x->props.header_len;
143
144	if (x->sel.family != AF_INET6) {
145		if (proto == IPPROTO_BEETPH) {
146			struct ip_beet_phdr *ph =
147				(struct ip_beet_phdr *)skb->data;
148
149			skb->transport_header += ph->hdrlen * 8;
150			proto = ph->nexthdr;
151		} else {
152			skb->transport_header -= IPV4_BEET_PHMAXLEN;
153		}
154	} else {
155		__be16 frag;
156
157		skb->transport_header +=
158			ipv6_skip_exthdr(skb, 0, &proto, &frag);
159		if (proto == IPPROTO_TCP)
160			skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
161	}
162
163	if (proto == IPPROTO_IPV6)
164		skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
165
166	__skb_pull(skb, skb_transport_offset(skb));
167	ops = rcu_dereference(inet_offloads[proto]);
168	if (likely(ops && ops->callbacks.gso_segment))
169		segs = ops->callbacks.gso_segment(skb, features);
170
171	return segs;
172}
173
174static struct sk_buff *xfrm4_outer_mode_gso_segment(struct xfrm_state *x,
175						    struct sk_buff *skb,
176						    netdev_features_t features)
177{
178	switch (x->outer_mode.encap) {
179	case XFRM_MODE_TUNNEL:
180		return xfrm4_tunnel_gso_segment(x, skb, features);
181	case XFRM_MODE_TRANSPORT:
182		return xfrm4_transport_gso_segment(x, skb, features);
183	case XFRM_MODE_BEET:
184		return xfrm4_beet_gso_segment(x, skb, features);
185	}
186
187	return ERR_PTR(-EOPNOTSUPP);
188}
189
190static struct sk_buff *esp4_gso_segment(struct sk_buff *skb,
191				        netdev_features_t features)
192{
193	struct xfrm_state *x;
194	struct ip_esp_hdr *esph;
195	struct crypto_aead *aead;
196	netdev_features_t esp_features = features;
197	struct xfrm_offload *xo = xfrm_offload(skb);
198	struct sec_path *sp;
199
200	if (!xo)
201		return ERR_PTR(-EINVAL);
202
203	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP))
204		return ERR_PTR(-EINVAL);
205
206	sp = skb_sec_path(skb);
207	x = sp->xvec[sp->len - 1];
208	aead = x->data;
209	esph = ip_esp_hdr(skb);
210
211	if (esph->spi != x->id.spi)
212		return ERR_PTR(-EINVAL);
213
214	if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))
215		return ERR_PTR(-EINVAL);
216
217	__skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));
218
219	skb->encap_hdr_csum = 1;
220
221	if ((!(skb->dev->gso_partial_features & NETIF_F_HW_ESP) &&
222	     !(features & NETIF_F_HW_ESP)) || x->xso.dev != skb->dev)
223		esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK |
224					    NETIF_F_SCTP_CRC);
225	else if (!(features & NETIF_F_HW_ESP_TX_CSUM) &&
226		 !(skb->dev->gso_partial_features & NETIF_F_HW_ESP_TX_CSUM))
227		esp_features = features & ~(NETIF_F_CSUM_MASK |
228					    NETIF_F_SCTP_CRC);
229
230	xo->flags |= XFRM_GSO_SEGMENT;
231
232	return xfrm4_outer_mode_gso_segment(x, skb, esp_features);
233}
234
235static int esp_input_tail(struct xfrm_state *x, struct sk_buff *skb)
236{
237	struct crypto_aead *aead = x->data;
238	struct xfrm_offload *xo = xfrm_offload(skb);
239
240	if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead)))
241		return -EINVAL;
242
243	if (!(xo->flags & CRYPTO_DONE))
244		skb->ip_summed = CHECKSUM_NONE;
245
246	return esp_input_done2(skb, 0);
247}
248
249static int esp_xmit(struct xfrm_state *x, struct sk_buff *skb,  netdev_features_t features)
250{
251	int err;
252	int alen;
253	int blksize;
254	struct xfrm_offload *xo;
255	struct ip_esp_hdr *esph;
256	struct crypto_aead *aead;
257	struct esp_info esp;
258	bool hw_offload = true;
259	__u32 seq;
260
261	esp.inplace = true;
262
263	xo = xfrm_offload(skb);
264
265	if (!xo)
266		return -EINVAL;
267
268	if ((!(features & NETIF_F_HW_ESP) &&
269	     !(skb->dev->gso_partial_features & NETIF_F_HW_ESP)) ||
270	    x->xso.dev != skb->dev) {
271		xo->flags |= CRYPTO_FALLBACK;
272		hw_offload = false;
273	}
274
275	esp.proto = xo->proto;
276
277	/* skb is pure payload to encrypt */
278
279	aead = x->data;
280	alen = crypto_aead_authsize(aead);
281
282	esp.tfclen = 0;
283	/* XXX: Add support for tfc padding here. */
284
285	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
286	esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
287	esp.plen = esp.clen - skb->len - esp.tfclen;
288	esp.tailen = esp.tfclen + esp.plen + alen;
289
290	esp.esph = ip_esp_hdr(skb);
291
292
293	if (!hw_offload || (hw_offload && !skb_is_gso(skb))) {
294		esp.nfrags = esp_output_head(x, skb, &esp);
295		if (esp.nfrags < 0)
296			return esp.nfrags;
297	}
298
299	seq = xo->seq.low;
300
301	esph = esp.esph;
302	esph->spi = x->id.spi;
303
304	skb_push(skb, -skb_network_offset(skb));
305
306	if (xo->flags & XFRM_GSO_SEGMENT) {
307		esph->seq_no = htonl(seq);
308
309		if (!skb_is_gso(skb))
310			xo->seq.low++;
311		else
312			xo->seq.low += skb_shinfo(skb)->gso_segs;
313	}
314
315	if (xo->seq.low < seq)
316		xo->seq.hi++;
317
318	esp.seqno = cpu_to_be64(seq + ((u64)xo->seq.hi << 32));
319
320	ip_hdr(skb)->tot_len = htons(skb->len);
321	ip_send_check(ip_hdr(skb));
322
323	if (hw_offload) {
324		if (!skb_ext_add(skb, SKB_EXT_SEC_PATH))
325			return -ENOMEM;
326
327		xo = xfrm_offload(skb);
328		if (!xo)
329			return -EINVAL;
330
331		xo->flags |= XFRM_XMIT;
332		return 0;
333	}
334
335	err = esp_output_tail(x, skb, &esp);
336	if (err)
337		return err;
338
339	secpath_reset(skb);
340
341	if (skb_needs_linearize(skb, skb->dev->features) &&
342	    __skb_linearize(skb))
343		return -ENOMEM;
344	return 0;
345}
346
347static const struct net_offload esp4_offload = {
348	.callbacks = {
349		.gro_receive = esp4_gro_receive,
350		.gso_segment = esp4_gso_segment,
351	},
352};
353
354static const struct xfrm_type_offload esp_type_offload = {
355	.description	= "ESP4 OFFLOAD",
356	.owner		= THIS_MODULE,
357	.proto	     	= IPPROTO_ESP,
358	.input_tail	= esp_input_tail,
359	.xmit		= esp_xmit,
360	.encap		= esp4_gso_encap,
361};
362
363static int __init esp4_offload_init(void)
364{
365	if (xfrm_register_type_offload(&esp_type_offload, AF_INET) < 0) {
366		pr_info("%s: can't add xfrm type offload\n", __func__);
367		return -EAGAIN;
368	}
369
370	return inet_add_offload(&esp4_offload, IPPROTO_ESP);
371}
372
373static void __exit esp4_offload_exit(void)
374{
375	xfrm_unregister_type_offload(&esp_type_offload, AF_INET);
376	inet_del_offload(&esp4_offload, IPPROTO_ESP);
377}
378
379module_init(esp4_offload_init);
380module_exit(esp4_offload_exit);
381MODULE_LICENSE("GPL");
382MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
383MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET, XFRM_PROTO_ESP);
384MODULE_DESCRIPTION("IPV4 GSO/GRO offload support");
385