Lines Matching refs:psock
32 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,
77 sk_psock_queue_msg(psock, tmp);
78 sk_psock_data_ready(sk, psock);
164 struct sk_psock *psock = sk_psock_get(sk);
167 if (unlikely(!psock))
170 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :
172 sk_psock_put(sk, psock);
178 static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock,
193 !list_empty(&psock->ingress_msg) ||
200 static bool is_next_msg_fin(struct sk_psock *psock)
206 msg_rx = sk_psock_peek_msg(psock);
227 struct sk_psock *psock;
236 psock = sk_psock_get(sk);
237 if (unlikely(!psock))
262 copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
269 bool is_fin = is_next_msg_fin(psock);
309 data = tcp_msg_wait_data(sk, psock, timeo);
314 if (data && !sk_psock_queue_empty(psock))
327 sk_psock_put(sk, psock);
334 struct sk_psock *psock;
343 psock = sk_psock_get(sk);
344 if (unlikely(!psock))
347 sk_psock_queue_empty(psock)) {
348 sk_psock_put(sk, psock);
353 copied = sk_msg_recvmsg(sk, psock, msg, len, flags);
359 data = tcp_msg_wait_data(sk, psock, timeo);
365 if (!sk_psock_queue_empty(psock))
368 sk_psock_put(sk, psock);
377 sk_psock_put(sk, psock);
381 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,
391 if (psock->eval == __SK_NONE) {
398 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
404 psock->cork_bytes = msg->cork_bytes - msg->sg.size;
405 if (!psock->cork) {
406 psock->cork = kzalloc(sizeof(*psock->cork),
408 if (!psock->cork)
411 memcpy(psock->cork, msg, sizeof(*msg));
416 if (psock->apply_bytes && psock->apply_bytes < tosend)
417 tosend = psock->apply_bytes;
420 switch (psock->eval) {
427 sk_msg_apply_bytes(psock, tosend);
430 redir_ingress = psock->redir_ingress;
431 sk_redir = psock->sk_redir;
432 sk_msg_apply_bytes(psock, tosend);
433 if (!psock->apply_bytes) {
435 eval = psock->eval;
436 psock->eval = __SK_NONE;
437 psock->sk_redir = NULL;
439 if (psock->cork) {
441 psock->cork = NULL;
471 sk_msg_apply_bytes(psock, tosend);
477 if (!psock->apply_bytes) {
478 psock->eval = __SK_NONE;
479 if (psock->sk_redir) {
480 sock_put(psock->sk_redir);
481 psock->sk_redir = NULL;
499 struct sk_psock *psock;
507 psock = sk_psock_get(sk);
508 if (unlikely(!psock))
525 if (psock->cork) {
526 msg_tx = psock->cork;
549 if (psock->cork_bytes) {
550 if (size > psock->cork_bytes)
551 psock->cork_bytes = 0;
553 psock->cork_bytes -= size;
554 if (psock->cork_bytes && !enospc)
557 psock->eval = __SK_NONE;
558 psock->cork_bytes = 0;
561 err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);
570 if (msg_tx && msg_tx != psock->cork)
579 sk_psock_put(sk, psock);
642 * into ops if e.g. a psock is not present. Make sure they are
649 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)
652 int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE;
654 if (psock->progs.stream_verdict || psock->progs.skb_verdict) {
662 * unhash routine because the associated psock is being
665 WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash);
666 tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);
668 sk->sk_write_space = psock->saved_write_space;
670 sock_replace_proto(sk, psock->sk_proto);
676 if (tcp_bpf_assert_proto_ops(psock->sk_proto))
679 tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);
690 * the default ones because the child does not inherit the psock state