Lines Matching refs:kcm
26 #include <net/kcm.h>
29 #include <uapi/linux/kcm.h>
121 static void kcm_rcv_ready(struct kcm_sock *kcm)
123 struct kcm_mux *mux = kcm->mux;
127 if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))
131 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
134 WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
143 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
145 WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
162 list_add_tail(&kcm->wait_rx_list,
163 &kcm->mux->kcm_rx_waiters);
165 WRITE_ONCE(kcm->rx_wait, true);
171 struct kcm_sock *kcm = kcm_sk(sk);
172 struct kcm_mux *mux = kcm->mux;
181 if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) &&
184 kcm_rcv_ready(kcm);
215 /* Requeue received messages for a kcm socket to other kcm sockets. This is
216 * called with a kcm socket is receive disabled.
222 struct kcm_sock *kcm;
234 kcm = list_first_entry(&mux->kcm_rx_waiters,
237 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
239 list_del(&kcm->wait_rx_list);
241 WRITE_ONCE(kcm->rx_wait, false);
256 struct kcm_sock *kcm;
281 kcm = list_first_entry(&mux->kcm_rx_waiters,
283 list_del(&kcm->wait_rx_list);
285 WRITE_ONCE(kcm->rx_wait, false);
287 psock->rx_kcm = kcm;
289 WRITE_ONCE(kcm->rx_psock, psock);
293 return kcm;
296 static void kcm_done(struct kcm_sock *kcm);
307 struct kcm_sock *kcm = psock->rx_kcm;
310 if (!kcm)
317 WRITE_ONCE(kcm->rx_psock, NULL);
319 /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with
324 if (unlikely(kcm->done)) {
330 INIT_WORK(&kcm->done_work, kcm_done_work);
331 schedule_work(&kcm->done_work);
335 if (unlikely(kcm->rx_disabled)) {
336 requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
337 } else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {
341 kcm_rcv_ready(kcm);
364 struct kcm_sock *kcm;
367 kcm = reserve_rx_kcm(psock, skb);
368 if (!kcm) {
375 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
415 struct kcm_sock *kcm;
427 kcm = psock->tx_kcm;
428 if (kcm && !unlikely(kcm->tx_stopped))
429 queue_work(kcm_wq, &kcm->tx_work);
436 static void unreserve_psock(struct kcm_sock *kcm);
438 /* kcm sock is locked. */
439 static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)
441 struct kcm_mux *mux = kcm->mux;
444 psock = kcm->tx_psock;
449 WARN_ON(kcm->tx_wait);
451 unreserve_psock(kcm);
453 return kcm->tx_psock;
461 psock = kcm->tx_psock;
463 WARN_ON(kcm->tx_wait);
465 return kcm->tx_psock;
473 if (kcm->tx_wait) {
474 list_del(&kcm->wait_psock_list);
475 kcm->tx_wait = false;
477 kcm->tx_psock = psock;
478 psock->tx_kcm = kcm;
480 } else if (!kcm->tx_wait) {
481 list_add_tail(&kcm->wait_psock_list,
483 kcm->tx_wait = true;
495 struct kcm_sock *kcm;
501 kcm = list_first_entry(&mux->kcm_tx_waiters,
504 list_del(&kcm->wait_psock_list);
505 kcm->tx_wait = false;
506 psock->tx_kcm = kcm;
513 kcm->tx_psock = psock;
515 queue_work(kcm_wq, &kcm->tx_work);
519 /* kcm sock is locked. */
520 static void unreserve_psock(struct kcm_sock *kcm)
523 struct kcm_mux *mux = kcm->mux;
527 psock = kcm->tx_psock;
538 WARN_ON(kcm->tx_wait);
540 kcm->tx_psock = NULL;
566 static void kcm_report_tx_retry(struct kcm_sock *kcm)
568 struct kcm_mux *mux = kcm->mux;
575 /* Write any messages ready on the kcm socket. Called with kcm sock lock
578 static int kcm_write_msgs(struct kcm_sock *kcm)
580 struct sock *sk = &kcm->sk;
588 kcm->tx_wait_more = false;
589 psock = kcm->tx_psock;
594 unreserve_psock(kcm);
595 kcm_report_tx_retry(kcm);
623 psock = reserve_psock(kcm);
676 unreserve_psock(kcm);
679 kcm_report_tx_retry(kcm);
715 unreserve_psock(kcm);
726 struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);
727 struct sock *sk = &kcm->sk;
735 err = kcm_write_msgs(kcm);
739 report_csk_error(&kcm->sk, -err);
754 static void kcm_push(struct kcm_sock *kcm)
756 if (kcm->tx_wait_more)
757 kcm_write_msgs(kcm);
765 struct kcm_sock *kcm = kcm_sk(sk);
786 if (kcm->seq_skb) {
788 head = kcm->seq_skb;
803 kcm_push(kcm);
821 kcm_push(kcm);
830 kcm_push(kcm);
862 kcm->seq_skb = NULL;
863 KCM_STATS_INCR(kcm->stats.tx_msgs);
866 kcm->tx_wait_more = true;
867 } else if (kcm->tx_wait_more || not_busy) {
868 err = kcm_write_msgs(kcm);
876 report_csk_error(&kcm->sk, -err);
881 kcm->seq_skb = head;
885 KCM_STATS_ADD(kcm->stats.tx_bytes, size);
891 kcm_push(kcm);
906 struct kcm_sock *kcm = kcm_sk(sk);
922 if (kcm->seq_skb) {
924 head = kcm->seq_skb;
931 kcm_push(kcm);
942 kcm_push(kcm);
1020 kcm_push(kcm);
1032 kcm->seq_skb = NULL;
1033 KCM_STATS_INCR(kcm->stats.tx_msgs);
1037 kcm->tx_wait_more = true;
1038 } else if (kcm->tx_wait_more || not_busy) {
1039 err = kcm_write_msgs(kcm);
1047 report_csk_error(&kcm->sk, -err);
1054 kcm->seq_skb = head;
1059 KCM_STATS_ADD(kcm->stats.tx_bytes, copied);
1065 kcm_push(kcm);
1073 if (head != kcm->seq_skb)
1077 kcm->seq_skb = NULL;
1095 struct kcm_sock *kcm = kcm_sk(sk);
1118 KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1131 KCM_STATS_INCR(kcm->stats.rx_msgs);
1146 struct kcm_sock *kcm = kcm_sk(sk);
1171 KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1190 /* kcm sock lock held */
1191 static void kcm_recv_disable(struct kcm_sock *kcm)
1193 struct kcm_mux *mux = kcm->mux;
1195 if (kcm->rx_disabled)
1200 kcm->rx_disabled = 1;
1203 if (!kcm->rx_psock) {
1204 if (kcm->rx_wait) {
1205 list_del(&kcm->wait_rx_list);
1207 WRITE_ONCE(kcm->rx_wait, false);
1210 requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
1216 /* kcm sock lock held */
1217 static void kcm_recv_enable(struct kcm_sock *kcm)
1219 struct kcm_mux *mux = kcm->mux;
1221 if (!kcm->rx_disabled)
1226 kcm->rx_disabled = 0;
1227 kcm_rcv_ready(kcm);
1235 struct kcm_sock *kcm = kcm_sk(sock->sk);
1252 lock_sock(&kcm->sk);
1254 kcm_recv_disable(kcm);
1256 kcm_recv_enable(kcm);
1257 release_sock(&kcm->sk);
1269 struct kcm_sock *kcm = kcm_sk(sock->sk);
1284 val = kcm->rx_disabled;
1297 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)
1307 kcm->sk.sk_state = TCP_ESTABLISHED;
1309 /* Add to mux's kcm sockets list */
1310 kcm->mux = mux;
1321 list_add(&kcm->kcm_sock_list, head);
1322 kcm->index = index;
1327 INIT_WORK(&kcm->tx_work, kcm_tx_work);
1330 kcm_rcv_ready(kcm);
1337 struct kcm_sock *kcm = kcm_sk(sock->sk);
1338 struct kcm_mux *mux = kcm->mux;
1475 * be no way to reserve a kcm for this psock.
1522 * up in the kcm paths, we need kcm lock which can not be
1565 struct kcm_sock *kcm = kcm_sk(sock->sk);
1566 struct kcm_mux *mux = kcm->mux;
1621 /* Clone a kcm socket. */
1742 static void kcm_done(struct kcm_sock *kcm)
1744 struct kcm_mux *mux = kcm->mux;
1745 struct sock *sk = &kcm->sk;
1749 if (kcm->rx_psock) {
1751 WARN_ON(kcm->done);
1752 kcm->rx_disabled = 1;
1753 kcm->done = 1;
1758 if (kcm->rx_wait) {
1759 list_del(&kcm->wait_rx_list);
1761 WRITE_ONCE(kcm->rx_wait, false);
1763 /* Move any pending receive messages to other kcm sockets */
1774 list_del(&kcm->kcm_sock_list);
1785 WARN_ON(kcm->rx_wait);
1787 sock_put(&kcm->sk);
1796 struct kcm_sock *kcm;
1803 kcm = kcm_sk(sk);
1804 mux = kcm->mux;
1808 kfree_skb(kcm->seq_skb);
1816 /* Set tx_stopped. This is checked when psock is bound to a kcm and we
1820 kcm->tx_stopped = 1;
1825 if (kcm->tx_wait) {
1827 * that a psock will be assigned to this kcm.
1829 list_del(&kcm->wait_psock_list);
1830 kcm->tx_wait = false;
1835 * to the kcm socket.
1837 cancel_work_sync(&kcm->tx_work);
1840 psock = kcm->tx_psock;
1844 * need to do this after removing kcm from tx_wait list.
1847 unreserve_psock(kcm);
1851 WARN_ON(kcm->tx_wait);
1852 WARN_ON(kcm->tx_psock);
1856 kcm_done(kcm);
1904 /* Create proto operation for kcm sockets */
1930 /* Allocate a kcm mux, shared between KCM sockets */