Lines Matching refs:psock
431 struct sk_psock *psock,
454 sk_psock_queue_msg(psock, msg);
455 sk_psock_data_ready(sk, psock);
459 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb);
461 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb)
463 struct sock *sk = psock->sk;
472 return sk_psock_skb_ingress_self(psock, skb);
484 err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg);
494 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb)
497 struct sock *sk = psock->sk;
504 err = sk_psock_skb_ingress_enqueue(skb, psock, sk, msg);
510 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
514 if (!sock_writeable(psock->sk))
516 return skb_send_sock_locked(psock->sk, skb, off, len);
518 return sk_psock_skb_ingress(psock, skb);
523 struct sk_psock *psock = container_of(work, struct sk_psock, work);
524 struct sk_psock_work_state *state = &psock->work_state;
531 lock_sock(psock->sk);
540 while ((skb = skb_dequeue(&psock->ingress_skb))) {
547 if (likely(psock->sk->sk_socket))
548 ret = sk_psock_handle_skb(psock, skb, off,
558 sk_psock_report_error(psock, ret ? -ret : EPIPE);
559 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
571 release_sock(psock->sk);
576 struct sk_psock *psock;
582 psock = ERR_PTR(-EINVAL);
587 psock = ERR_PTR(-EBUSY);
591 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);
592 if (!psock) {
593 psock = ERR_PTR(-ENOMEM);
598 psock->sk = sk;
599 psock->eval = __SK_NONE;
600 psock->sk_proto = prot;
601 psock->saved_unhash = prot->unhash;
602 psock->saved_close = prot->close;
603 psock->saved_write_space = sk->sk_write_space;
605 INIT_LIST_HEAD(&psock->link);
606 spin_lock_init(&psock->link_lock);
608 INIT_WORK(&psock->work, sk_psock_backlog);
609 INIT_LIST_HEAD(&psock->ingress_msg);
610 skb_queue_head_init(&psock->ingress_skb);
612 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
613 refcount_set(&psock->refcnt, 1);
615 __rcu_assign_sk_user_data_with_flags(sk, psock,
622 return psock;
626 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)
630 spin_lock_bh(&psock->link_lock);
631 link = list_first_entry_or_null(&psock->link, struct sk_psock_link,
635 spin_unlock_bh(&psock->link_lock);
639 void __sk_psock_purge_ingress_msg(struct sk_psock *psock)
643 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {
645 sk_msg_free(psock->sk, msg);
650 static void sk_psock_zap_ingress(struct sk_psock *psock)
652 __skb_queue_purge(&psock->ingress_skb);
653 __sk_psock_purge_ingress_msg(psock);
656 static void sk_psock_link_destroy(struct sk_psock *psock)
660 list_for_each_entry_safe(link, tmp, &psock->link, list) {
668 struct sk_psock *psock = container_of(gc, struct sk_psock, gc);
673 if (psock->progs.skb_parser)
674 strp_done(&psock->parser.strp);
676 cancel_work_sync(&psock->work);
678 psock_progs_drop(&psock->progs);
680 sk_psock_link_destroy(psock);
681 sk_psock_cork_free(psock);
682 sk_psock_zap_ingress(psock);
684 if (psock->sk_redir)
685 sock_put(psock->sk_redir);
686 sock_put(psock->sk);
687 kfree(psock);
692 struct sk_psock *psock = container_of(rcu, struct sk_psock, rcu);
694 INIT_WORK(&psock->gc, sk_psock_destroy_deferred);
695 schedule_work(&psock->gc);
698 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
700 sk_psock_cork_free(psock);
701 sk_psock_zap_ingress(psock);
704 sk_psock_restore_proto(sk, psock);
706 if (psock->progs.skb_parser)
707 sk_psock_stop_strp(sk, psock);
708 else if (psock->progs.skb_verdict)
709 sk_psock_stop_verdict(sk, psock);
711 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
713 call_rcu(&psock->rcu, sk_psock_destroy);
730 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
737 prog = READ_ONCE(psock->progs.msg_parser);
747 psock->apply_bytes = msg->apply_bytes;
749 if (psock->sk_redir)
750 sock_put(psock->sk_redir);
751 psock->sk_redir = msg->sk_redir;
752 if (!psock->sk_redir) {
756 sock_hold(psock->sk_redir);
764 static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog,
820 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)
826 prog = READ_ONCE(psock->progs.skb_verdict);
828 skb->sk = psock->sk;
830 ret = sk_psock_bpf_run(psock, prog, skb);
834 sk_psock_tls_verdict_apply(skb, psock->sk, ret);
840 static void sk_psock_verdict_apply(struct sk_psock *psock,
849 sk_other = psock->sk;
851 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
864 if (skb_queue_empty(&psock->ingress_skb)) {
865 err = sk_psock_skb_ingress_self(psock, skb);
868 skb_queue_tail(&psock->ingress_skb, skb);
869 schedule_work(&psock->work);
884 struct sk_psock *psock;
891 psock = sk_psock(sk);
892 if (unlikely(!psock)) {
896 prog = READ_ONCE(psock->progs.skb_verdict);
900 ret = sk_psock_bpf_run(psock, prog, skb);
904 sk_psock_verdict_apply(psock, skb, ret);
916 struct sk_psock *psock = sk_psock_from_strp(strp);
921 prog = READ_ONCE(psock->progs.skb_parser);
923 skb->sk = psock->sk;
924 ret = sk_psock_bpf_run(psock, prog, skb);
934 struct sk_psock *psock;
937 psock = sk_psock(sk);
938 if (likely(psock)) {
940 psock->parser.saved_data_ready(sk);
943 strp_data_ready(&psock->parser.strp);
954 struct sk_psock *psock;
967 psock = sk_psock(sk);
968 if (unlikely(!psock)) {
973 prog = READ_ONCE(psock->progs.skb_verdict);
977 ret = sk_psock_bpf_run(psock, prog, skb);
981 sk_psock_verdict_apply(psock, skb, ret);
1004 struct sk_psock *psock;
1008 psock = sk_psock(sk);
1009 if (likely(psock)) {
1010 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
1011 schedule_work(&psock->work);
1012 write_space = psock->saved_write_space;
1019 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
1027 psock->parser.enabled = false;
1028 return strp_init(&psock->parser.strp, sk, &cb);
1031 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
1033 struct sk_psock_parser *parser = &psock->parser;
1044 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
1046 struct sk_psock_parser *parser = &psock->parser;
1057 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
1059 struct sk_psock_parser *parser = &psock->parser;
1070 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
1072 struct sk_psock_parser *parser = &psock->parser;