Lines Matching refs:psock
411 /* Receive sk_msg from psock->ingress_msg to @msg. */
412 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
420 msg_rx = sk_psock_peek_msg(psock);
470 msg_rx = sk_psock_next_msg(psock, msg_rx);
478 msg_rx = sk_psock_dequeue_msg(psock);
481 msg_rx = sk_psock_peek_msg(psock);
490 struct sk_psock *psock;
494 psock = sk_psock(sk);
495 if (likely(psock))
496 empty = list_empty(&psock->ingress_msg);
527 struct sk_psock *psock,
554 sk_psock_queue_msg(psock, msg);
555 sk_psock_data_ready(sk, psock);
559 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
562 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
565 struct sock *sk = psock->sk;
574 return sk_psock_skb_ingress_self(psock, skb, off, len);
586 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
596 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
600 struct sock *sk = psock->sk;
606 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
612 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
618 if (!sock_writeable(psock->sk))
620 return skb_send_sock(psock->sk, skb, off, len);
623 err = sk_psock_skb_ingress(psock, skb, off, len);
629 static void sk_psock_skb_state(struct sk_psock *psock,
633 spin_lock_bh(&psock->ingress_lock);
634 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
638 spin_unlock_bh(&psock->ingress_lock);
644 struct sk_psock *psock = container_of(dwork, struct sk_psock, work);
645 struct sk_psock_work_state *state = &psock->work_state;
651 mutex_lock(&psock->work_mutex);
657 while ((skb = skb_peek(&psock->ingress_skb))) {
670 if (!sock_flag(psock->sk, SOCK_DEAD))
671 ret = sk_psock_handle_skb(psock, skb, off,
675 sk_psock_skb_state(psock, state, len, off);
680 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
681 schedule_delayed_work(&psock->work, 1);
685 sk_psock_report_error(psock, ret ? -ret : EPIPE);
686 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
693 skb = skb_dequeue(&psock->ingress_skb);
697 mutex_unlock(&psock->work_mutex);
702 struct sk_psock *psock;
708 psock = ERR_PTR(-EINVAL);
713 psock = ERR_PTR(-EBUSY);
717 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);
718 if (!psock) {
719 psock = ERR_PTR(-ENOMEM);
724 psock->sk = sk;
725 psock->eval = __SK_NONE;
726 psock->sk_proto = prot;
727 psock->saved_unhash = prot->unhash;
728 psock->saved_destroy = prot->destroy;
729 psock->saved_close = prot->close;
730 psock->saved_write_space = sk->sk_write_space;
732 INIT_LIST_HEAD(&psock->link);
733 spin_lock_init(&psock->link_lock);
735 INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);
736 mutex_init(&psock->work_mutex);
737 INIT_LIST_HEAD(&psock->ingress_msg);
738 spin_lock_init(&psock->ingress_lock);
739 skb_queue_head_init(&psock->ingress_skb);
741 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
742 refcount_set(&psock->refcnt, 1);
744 __rcu_assign_sk_user_data_with_flags(sk, psock,
751 return psock;
755 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)
759 spin_lock_bh(&psock->link_lock);
760 link = list_first_entry_or_null(&psock->link, struct sk_psock_link,
764 spin_unlock_bh(&psock->link_lock);
768 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)
772 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {
774 sk_msg_free(psock->sk, msg);
779 static void __sk_psock_zap_ingress(struct sk_psock *psock)
783 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
785 sock_drop(psock->sk, skb);
787 __sk_psock_purge_ingress_msg(psock);
790 static void sk_psock_link_destroy(struct sk_psock *psock)
794 list_for_each_entry_safe(link, tmp, &psock->link, list) {
800 void sk_psock_stop(struct sk_psock *psock)
802 spin_lock_bh(&psock->ingress_lock);
803 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
804 sk_psock_cork_free(psock);
805 spin_unlock_bh(&psock->ingress_lock);
808 static void sk_psock_done_strp(struct sk_psock *psock);
812 struct sk_psock *psock = container_of(to_rcu_work(work),
816 sk_psock_done_strp(psock);
818 cancel_delayed_work_sync(&psock->work);
819 __sk_psock_zap_ingress(psock);
820 mutex_destroy(&psock->work_mutex);
822 psock_progs_drop(&psock->progs);
824 sk_psock_link_destroy(psock);
825 sk_psock_cork_free(psock);
827 if (psock->sk_redir)
828 sock_put(psock->sk_redir);
829 if (psock->sk_pair)
830 sock_put(psock->sk_pair);
831 sock_put(psock->sk);
832 kfree(psock);
835 void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
838 sk_psock_restore_proto(sk, psock);
840 if (psock->progs.stream_parser)
841 sk_psock_stop_strp(sk, psock);
842 else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
843 sk_psock_stop_verdict(sk, psock);
846 sk_psock_stop(psock);
848 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
849 queue_rcu_work(system_wq, &psock->rwork);
866 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
873 prog = READ_ONCE(psock->progs.msg_parser);
883 psock->apply_bytes = msg->apply_bytes;
885 if (psock->sk_redir) {
886 sock_put(psock->sk_redir);
887 psock->sk_redir = NULL;
893 psock->redir_ingress = sk_msg_to_ingress(msg);
894 psock->sk_redir = msg->sk_redir;
895 sock_hold(psock->sk_redir);
955 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)
961 prog = READ_ONCE(psock->progs.stream_verdict);
963 skb->sk = psock->sk;
970 sk_psock_tls_verdict_apply(skb, psock, ret);
976 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
986 sk_other = psock->sk;
988 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
999 if (skb_queue_empty(&psock->ingress_skb)) {
1008 err = sk_psock_skb_ingress_self(psock, skb, off, len);
1011 spin_lock_bh(&psock->ingress_lock);
1012 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
1013 skb_queue_tail(&psock->ingress_skb, skb);
1014 schedule_delayed_work(&psock->work, 0);
1017 spin_unlock_bh(&psock->ingress_lock);
1023 tcp_eat_skb(psock->sk, skb);
1024 err = sk_psock_skb_redirect(psock, skb);
1030 tcp_eat_skb(psock->sk, skb);
1031 sock_drop(psock->sk, skb);
1039 struct sk_psock *psock;
1043 psock = sk_psock(sk);
1044 if (likely(psock)) {
1045 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
1046 schedule_delayed_work(&psock->work, 0);
1047 write_space = psock->saved_write_space;
1057 struct sk_psock *psock;
1064 psock = sk_psock(sk);
1065 if (unlikely(!psock)) {
1069 prog = READ_ONCE(psock->progs.stream_verdict);
1079 sk_psock_verdict_apply(psock, skb, ret);
1091 struct sk_psock *psock = container_of(strp, struct sk_psock, strp);
1096 prog = READ_ONCE(psock->progs.stream_parser);
1098 skb->sk = psock->sk;
1109 struct sk_psock *psock;
1114 psock = sk_psock(sk);
1115 if (likely(psock)) {
1117 psock->saved_data_ready(sk);
1120 strp_data_ready(&psock->strp);
1127 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
1137 ret = strp_init(&psock->strp, sk, &cb);
1139 sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);
1144 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
1146 if (psock->saved_data_ready)
1149 psock->saved_data_ready = sk->sk_data_ready;
1154 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
1156 psock_set_prog(&psock->progs.stream_parser, NULL);
1158 if (!psock->saved_data_ready)
1161 sk->sk_data_ready = psock->saved_data_ready;
1162 psock->saved_data_ready = NULL;
1163 strp_stop(&psock->strp);
1166 static void sk_psock_done_strp(struct sk_psock *psock)
1169 if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))
1170 strp_done(&psock->strp);
1173 static void sk_psock_done_strp(struct sk_psock *psock)
1180 struct sk_psock *psock;
1186 psock = sk_psock(sk);
1187 if (unlikely(!psock)) {
1193 prog = READ_ONCE(psock->progs.stream_verdict);
1195 prog = READ_ONCE(psock->progs.skb_verdict);
1202 ret = sk_psock_verdict_apply(psock, skb, ret);
1225 struct sk_psock *psock;
1228 psock = sk_psock(sk);
1229 if (psock) {
1231 sk_psock_data_ready(sk, psock);
1238 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
1240 if (psock->saved_data_ready)
1243 psock->saved_data_ready = sk->sk_data_ready;
1248 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
1250 psock_set_prog(&psock->progs.stream_verdict, NULL);
1251 psock_set_prog(&psock->progs.skb_verdict, NULL);
1253 if (!psock->saved_data_ready)
1256 sk->sk_data_ready = psock->saved_data_ready;
1257 psock->saved_data_ready = NULL;