xref: /kernel/linux/linux-5.10/net/ipv4/tcp_bpf.c (revision 8c2ecf20) 
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4#include <linux/skmsg.h>
5#include <linux/filter.h>
6#include <linux/bpf.h>
7#include <linux/init.h>
8#include <linux/wait.h>
9#include <linux/util_macros.h>
10
11#include <net/inet_common.h>
12#include <net/tls.h>
13
14int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock,
15		      struct msghdr *msg, int len, int flags)
16{
17	struct iov_iter *iter = &msg->msg_iter;
18	int peek = flags & MSG_PEEK;
19	struct sk_msg *msg_rx;
20	int i, copied = 0;
21
22	msg_rx = list_first_entry_or_null(&psock->ingress_msg,
23					  struct sk_msg, list);
24
25	while (copied != len) {
26		struct scatterlist *sge;
27
28		if (unlikely(!msg_rx))
29			break;
30
31		i = msg_rx->sg.start;
32		do {
33			struct page *page;
34			int copy;
35
36			sge = sk_msg_elem(msg_rx, i);
37			copy = sge->length;
38			page = sg_page(sge);
39			if (copied + copy > len)
40				copy = len - copied;
41			copy = copy_page_to_iter(page, sge->offset, copy, iter);
42			if (!copy)
43				return copied ? copied : -EFAULT;
44
45			copied += copy;
46			if (likely(!peek)) {
47				sge->offset += copy;
48				sge->length -= copy;
49				if (!msg_rx->skb)
50					sk_mem_uncharge(sk, copy);
51				msg_rx->sg.size -= copy;
52
53				if (!sge->length) {
54					sk_msg_iter_var_next(i);
55					if (!msg_rx->skb)
56						put_page(page);
57				}
58			} else {
59				/* Lets not optimize peek case if copy_page_to_iter
60				 * didn't copy the entire length lets just break.
61				 */
62				if (copy != sge->length)
63					return copied;
64				sk_msg_iter_var_next(i);
65			}
66
67			if (copied == len)
68				break;
69		} while (i != msg_rx->sg.end);
70
71		if (unlikely(peek)) {
72			if (msg_rx == list_last_entry(&psock->ingress_msg,
73						      struct sk_msg, list))
74				break;
75			msg_rx = list_next_entry(msg_rx, list);
76			continue;
77		}
78
79		msg_rx->sg.start = i;
80		if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) {
81			list_del(&msg_rx->list);
82			if (msg_rx->skb)
83				consume_skb(msg_rx->skb);
84			kfree(msg_rx);
85		}
86		msg_rx = list_first_entry_or_null(&psock->ingress_msg,
87						  struct sk_msg, list);
88	}
89
90	return copied;
91}
92EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg);
93
94static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
95			   struct sk_msg *msg, u32 apply_bytes, int flags)
96{
97	bool apply = apply_bytes;
98	struct scatterlist *sge;
99	u32 size, copied = 0;
100	struct sk_msg *tmp;
101	int i, ret = 0;
102
103	tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL);
104	if (unlikely(!tmp))
105		return -ENOMEM;
106
107	lock_sock(sk);
108	tmp->sg.start = msg->sg.start;
109	i = msg->sg.start;
110	do {
111		sge = sk_msg_elem(msg, i);
112		size = (apply && apply_bytes < sge->length) ?
113			apply_bytes : sge->length;
114		if (!sk_wmem_schedule(sk, size)) {
115			if (!copied)
116				ret = -ENOMEM;
117			break;
118		}
119
120		sk_mem_charge(sk, size);
121		sk_msg_xfer(tmp, msg, i, size);
122		copied += size;
123		if (sge->length)
124			get_page(sk_msg_page(tmp, i));
125		sk_msg_iter_var_next(i);
126		tmp->sg.end = i;
127		if (apply) {
128			apply_bytes -= size;
129			if (!apply_bytes) {
130				if (sge->length)
131					sk_msg_iter_var_prev(i);
132				break;
133			}
134		}
135	} while (i != msg->sg.end);
136
137	if (!ret) {
138		msg->sg.start = i;
139		sk_psock_queue_msg(psock, tmp);
140		sk_psock_data_ready(sk, psock);
141	} else {
142		sk_msg_free(sk, tmp);
143		kfree(tmp);
144	}
145
146	release_sock(sk);
147	return ret;
148}
149
150static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes,
151			int flags, bool uncharge)
152{
153	bool apply = apply_bytes;
154	struct scatterlist *sge;
155	struct page *page;
156	int size, ret = 0;
157	u32 off;
158
159	while (1) {
160		bool has_tx_ulp;
161
162		sge = sk_msg_elem(msg, msg->sg.start);
163		size = (apply && apply_bytes < sge->length) ?
164			apply_bytes : sge->length;
165		off  = sge->offset;
166		page = sg_page(sge);
167
168		tcp_rate_check_app_limited(sk);
169retry:
170		has_tx_ulp = tls_sw_has_ctx_tx(sk);
171		if (has_tx_ulp) {
172			flags |= MSG_SENDPAGE_NOPOLICY;
173			ret = kernel_sendpage_locked(sk,
174						     page, off, size, flags);
175		} else {
176			ret = do_tcp_sendpages(sk, page, off, size, flags);
177		}
178
179		if (ret <= 0)
180			return ret;
181		if (apply)
182			apply_bytes -= ret;
183		msg->sg.size -= ret;
184		sge->offset += ret;
185		sge->length -= ret;
186		if (uncharge)
187			sk_mem_uncharge(sk, ret);
188		if (ret != size) {
189			size -= ret;
190			off  += ret;
191			goto retry;
192		}
193		if (!sge->length) {
194			put_page(page);
195			sk_msg_iter_next(msg, start);
196			sg_init_table(sge, 1);
197			if (msg->sg.start == msg->sg.end)
198				break;
199		}
200		if (apply && !apply_bytes)
201			break;
202	}
203
204	return 0;
205}
206
207static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,
208			       u32 apply_bytes, int flags, bool uncharge)
209{
210	int ret;
211
212	lock_sock(sk);
213	ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge);
214	release_sock(sk);
215	return ret;
216}
217
218int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,
219			  u32 bytes, int flags)
220{
221	bool ingress = sk_msg_to_ingress(msg);
222	struct sk_psock *psock = sk_psock_get(sk);
223	int ret;
224
225	if (unlikely(!psock))
226		return -EPIPE;
227
228	ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :
229			tcp_bpf_push_locked(sk, msg, bytes, flags, false);
230	sk_psock_put(sk, psock);
231	return ret;
232}
233EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir);
234
235#ifdef CONFIG_BPF_STREAM_PARSER
236static bool tcp_bpf_stream_read(const struct sock *sk)
237{
238	struct sk_psock *psock;
239	bool empty = true;
240
241	rcu_read_lock();
242	psock = sk_psock(sk);
243	if (likely(psock))
244		empty = list_empty(&psock->ingress_msg);
245	rcu_read_unlock();
246	return !empty;
247}
248
249static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock,
250			     int flags, long timeo, int *err)
251{
252	DEFINE_WAIT_FUNC(wait, woken_wake_function);
253	int ret = 0;
254
255	if (sk->sk_shutdown & RCV_SHUTDOWN)
256		return 1;
257
258	if (!timeo)
259		return ret;
260
261	add_wait_queue(sk_sleep(sk), &wait);
262	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
263	ret = sk_wait_event(sk, &timeo,
264			    !list_empty(&psock->ingress_msg) ||
265			    !skb_queue_empty_lockless(&sk->sk_receive_queue), &wait);
266	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
267	remove_wait_queue(sk_sleep(sk), &wait);
268	return ret;
269}
270
271static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
272		    int nonblock, int flags, int *addr_len)
273{
274	struct sk_psock *psock;
275	int copied, ret;
276
277	if (unlikely(flags & MSG_ERRQUEUE))
278		return inet_recv_error(sk, msg, len, addr_len);
279
280	psock = sk_psock_get(sk);
281	if (unlikely(!psock))
282		return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
283	if (!skb_queue_empty(&sk->sk_receive_queue) &&
284	    sk_psock_queue_empty(psock)) {
285		sk_psock_put(sk, psock);
286		return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
287	}
288	lock_sock(sk);
289msg_bytes_ready:
290	copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags);
291	if (!copied) {
292		int data, err = 0;
293		long timeo;
294
295		timeo = sock_rcvtimeo(sk, nonblock);
296		data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err);
297		if (data) {
298			if (!sk_psock_queue_empty(psock))
299				goto msg_bytes_ready;
300			release_sock(sk);
301			sk_psock_put(sk, psock);
302			return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len);
303		}
304		if (err) {
305			ret = err;
306			goto out;
307		}
308		copied = -EAGAIN;
309	}
310	ret = copied;
311out:
312	release_sock(sk);
313	sk_psock_put(sk, psock);
314	return ret;
315}
316
317static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,
318				struct sk_msg *msg, int *copied, int flags)
319{
320	bool cork = false, enospc = sk_msg_full(msg);
321	struct sock *sk_redir;
322	u32 tosend, origsize, sent, delta = 0;
323	u32 eval;
324	int ret;
325
326more_data:
327	if (psock->eval == __SK_NONE) {
328		/* Track delta in msg size to add/subtract it on SK_DROP from
329		 * returned to user copied size. This ensures user doesn't
330		 * get a positive return code with msg_cut_data and SK_DROP
331		 * verdict.
332		 */
333		delta = msg->sg.size;
334		psock->eval = sk_psock_msg_verdict(sk, psock, msg);
335		delta -= msg->sg.size;
336	}
337
338	if (msg->cork_bytes &&
339	    msg->cork_bytes > msg->sg.size && !enospc) {
340		psock->cork_bytes = msg->cork_bytes - msg->sg.size;
341		if (!psock->cork) {
342			psock->cork = kzalloc(sizeof(*psock->cork),
343					      GFP_ATOMIC | __GFP_NOWARN);
344			if (!psock->cork)
345				return -ENOMEM;
346		}
347		memcpy(psock->cork, msg, sizeof(*msg));
348		return 0;
349	}
350
351	tosend = msg->sg.size;
352	if (psock->apply_bytes && psock->apply_bytes < tosend)
353		tosend = psock->apply_bytes;
354	eval = __SK_NONE;
355
356	switch (psock->eval) {
357	case __SK_PASS:
358		ret = tcp_bpf_push(sk, msg, tosend, flags, true);
359		if (unlikely(ret)) {
360			*copied -= sk_msg_free(sk, msg);
361			break;
362		}
363		sk_msg_apply_bytes(psock, tosend);
364		break;
365	case __SK_REDIRECT:
366		sk_redir = psock->sk_redir;
367		sk_msg_apply_bytes(psock, tosend);
368		if (!psock->apply_bytes) {
369			/* Clean up before releasing the sock lock. */
370			eval = psock->eval;
371			psock->eval = __SK_NONE;
372			psock->sk_redir = NULL;
373		}
374		if (psock->cork) {
375			cork = true;
376			psock->cork = NULL;
377		}
378		sk_msg_return(sk, msg, tosend);
379		release_sock(sk);
380
381		origsize = msg->sg.size;
382		ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);
383		sent = origsize - msg->sg.size;
384
385		if (eval == __SK_REDIRECT)
386			sock_put(sk_redir);
387
388		lock_sock(sk);
389		if (unlikely(ret < 0)) {
390			int free = sk_msg_free_nocharge(sk, msg);
391
392			if (!cork)
393				*copied -= free;
394		}
395		if (cork) {
396			sk_msg_free(sk, msg);
397			kfree(msg);
398			msg = NULL;
399			ret = 0;
400		}
401		break;
402	case __SK_DROP:
403	default:
404		sk_msg_free_partial(sk, msg, tosend);
405		sk_msg_apply_bytes(psock, tosend);
406		*copied -= (tosend + delta);
407		return -EACCES;
408	}
409
410	if (likely(!ret)) {
411		if (!psock->apply_bytes) {
412			psock->eval =  __SK_NONE;
413			if (psock->sk_redir) {
414				sock_put(psock->sk_redir);
415				psock->sk_redir = NULL;
416			}
417		}
418		if (msg &&
419		    msg->sg.data[msg->sg.start].page_link &&
420		    msg->sg.data[msg->sg.start].length) {
421			if (eval == __SK_REDIRECT)
422				sk_mem_charge(sk, tosend - sent);
423			goto more_data;
424		}
425	}
426	return ret;
427}
428
429static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
430{
431	struct sk_msg tmp, *msg_tx = NULL;
432	int copied = 0, err = 0;
433	struct sk_psock *psock;
434	long timeo;
435	int flags;
436
437	/* Don't let internal do_tcp_sendpages() flags through */
438	flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED);
439	flags |= MSG_NO_SHARED_FRAGS;
440
441	psock = sk_psock_get(sk);
442	if (unlikely(!psock))
443		return tcp_sendmsg(sk, msg, size);
444
445	lock_sock(sk);
446	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
447	while (msg_data_left(msg)) {
448		bool enospc = false;
449		u32 copy, osize;
450
451		if (sk->sk_err) {
452			err = -sk->sk_err;
453			goto out_err;
454		}
455
456		copy = msg_data_left(msg);
457		if (!sk_stream_memory_free(sk))
458			goto wait_for_sndbuf;
459		if (psock->cork) {
460			msg_tx = psock->cork;
461		} else {
462			msg_tx = &tmp;
463			sk_msg_init(msg_tx);
464		}
465
466		osize = msg_tx->sg.size;
467		err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1);
468		if (err) {
469			if (err != -ENOSPC)
470				goto wait_for_memory;
471			enospc = true;
472			copy = msg_tx->sg.size - osize;
473		}
474
475		err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx,
476					       copy);
477		if (err < 0) {
478			sk_msg_trim(sk, msg_tx, osize);
479			goto out_err;
480		}
481
482		copied += copy;
483		if (psock->cork_bytes) {
484			if (size > psock->cork_bytes)
485				psock->cork_bytes = 0;
486			else
487				psock->cork_bytes -= size;
488			if (psock->cork_bytes && !enospc)
489				goto out_err;
490			/* All cork bytes are accounted, rerun the prog. */
491			psock->eval = __SK_NONE;
492			psock->cork_bytes = 0;
493		}
494
495		err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);
496		if (unlikely(err < 0))
497			goto out_err;
498		continue;
499wait_for_sndbuf:
500		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
501wait_for_memory:
502		err = sk_stream_wait_memory(sk, &timeo);
503		if (err) {
504			if (msg_tx && msg_tx != psock->cork)
505				sk_msg_free(sk, msg_tx);
506			goto out_err;
507		}
508	}
509out_err:
510	if (err < 0)
511		err = sk_stream_error(sk, msg->msg_flags, err);
512	release_sock(sk);
513	sk_psock_put(sk, psock);
514	return copied > 0 ? copied : err;
515}
516
517static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset,
518			    size_t size, int flags)
519{
520	struct sk_msg tmp, *msg = NULL;
521	int err = 0, copied = 0;
522	struct sk_psock *psock;
523	bool enospc = false;
524
525	psock = sk_psock_get(sk);
526	if (unlikely(!psock))
527		return tcp_sendpage(sk, page, offset, size, flags);
528
529	lock_sock(sk);
530	if (psock->cork) {
531		msg = psock->cork;
532	} else {
533		msg = &tmp;
534		sk_msg_init(msg);
535	}
536
537	/* Catch case where ring is full and sendpage is stalled. */
538	if (unlikely(sk_msg_full(msg)))
539		goto out_err;
540
541	sk_msg_page_add(msg, page, size, offset);
542	sk_mem_charge(sk, size);
543	copied = size;
544	if (sk_msg_full(msg))
545		enospc = true;
546	if (psock->cork_bytes) {
547		if (size > psock->cork_bytes)
548			psock->cork_bytes = 0;
549		else
550			psock->cork_bytes -= size;
551		if (psock->cork_bytes && !enospc)
552			goto out_err;
553		/* All cork bytes are accounted, rerun the prog. */
554		psock->eval = __SK_NONE;
555		psock->cork_bytes = 0;
556	}
557
558	err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags);
559out_err:
560	release_sock(sk);
561	sk_psock_put(sk, psock);
562	return copied ? copied : err;
563}
564
565enum {
566	TCP_BPF_IPV4,
567	TCP_BPF_IPV6,
568	TCP_BPF_NUM_PROTS,
569};
570
571enum {
572	TCP_BPF_BASE,
573	TCP_BPF_TX,
574	TCP_BPF_NUM_CFGS,
575};
576
577static struct proto *tcpv6_prot_saved __read_mostly;
578static DEFINE_SPINLOCK(tcpv6_prot_lock);
579static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS];
580
581static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS],
582				   struct proto *base)
583{
584	prot[TCP_BPF_BASE]			= *base;
585	prot[TCP_BPF_BASE].close		= sock_map_close;
586	prot[TCP_BPF_BASE].recvmsg		= tcp_bpf_recvmsg;
587	prot[TCP_BPF_BASE].stream_memory_read	= tcp_bpf_stream_read;
588
589	prot[TCP_BPF_TX]			= prot[TCP_BPF_BASE];
590	prot[TCP_BPF_TX].sendmsg		= tcp_bpf_sendmsg;
591	prot[TCP_BPF_TX].sendpage		= tcp_bpf_sendpage;
592}
593
594static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops)
595{
596	if (unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) {
597		spin_lock_bh(&tcpv6_prot_lock);
598		if (likely(ops != tcpv6_prot_saved)) {
599			tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops);
600			smp_store_release(&tcpv6_prot_saved, ops);
601		}
602		spin_unlock_bh(&tcpv6_prot_lock);
603	}
604}
605
606static int __init tcp_bpf_v4_build_proto(void)
607{
608	tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot);
609	return 0;
610}
611late_initcall(tcp_bpf_v4_build_proto);
612
613static int tcp_bpf_assert_proto_ops(struct proto *ops)
614{
615	/* In order to avoid retpoline, we make assumptions when we call
616	 * into ops if e.g. a psock is not present. Make sure they are
617	 * indeed valid assumptions.
618	 */
619	return ops->recvmsg  == tcp_recvmsg &&
620	       ops->sendmsg  == tcp_sendmsg &&
621	       ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP;
622}
623
624struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock)
625{
626	int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4;
627	int config = psock->progs.msg_parser   ? TCP_BPF_TX   : TCP_BPF_BASE;
628
629	if (sk->sk_family == AF_INET6) {
630		if (tcp_bpf_assert_proto_ops(psock->sk_proto))
631			return ERR_PTR(-EINVAL);
632
633		tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
634	}
635
636	return &tcp_bpf_prots[family][config];
637}
638
639/* If a child got cloned from a listening socket that had tcp_bpf
640 * protocol callbacks installed, we need to restore the callbacks to
641 * the default ones because the child does not inherit the psock state
642 * that tcp_bpf callbacks expect.
643 */
644void tcp_bpf_clone(const struct sock *sk, struct sock *newsk)
645{
646	struct proto *prot = newsk->sk_prot;
647
648	if (is_insidevar(prot, tcp_bpf_prots))
649		newsk->sk_prot = sk->sk_prot_creator;
650}
651#endif /* CONFIG_BPF_STREAM_PARSER */
652