Lines Matching refs:kcm
27 #include <net/kcm.h>
30 #include <uapi/linux/kcm.h>
123 static void kcm_rcv_ready(struct kcm_sock *kcm)
125 struct kcm_mux *mux = kcm->mux;
129 if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled))
133 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
136 WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
145 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
147 WARN_ON(!sk_rmem_alloc_get(&kcm->sk));
164 list_add_tail(&kcm->wait_rx_list,
165 &kcm->mux->kcm_rx_waiters);
167 WRITE_ONCE(kcm->rx_wait, true);
173 struct kcm_sock *kcm = kcm_sk(sk);
174 struct kcm_mux *mux = kcm->mux;
183 if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) &&
186 kcm_rcv_ready(kcm);
217 /* Requeue received messages for a kcm socket to other kcm sockets. This is
218 * called with a kcm socket is receive disabled.
224 struct kcm_sock *kcm;
236 kcm = list_first_entry(&mux->kcm_rx_waiters,
239 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
241 list_del(&kcm->wait_rx_list);
243 WRITE_ONCE(kcm->rx_wait, false);
258 struct kcm_sock *kcm;
283 kcm = list_first_entry(&mux->kcm_rx_waiters,
285 list_del(&kcm->wait_rx_list);
287 WRITE_ONCE(kcm->rx_wait, false);
289 psock->rx_kcm = kcm;
291 WRITE_ONCE(kcm->rx_psock, psock);
295 return kcm;
298 static void kcm_done(struct kcm_sock *kcm);
309 struct kcm_sock *kcm = psock->rx_kcm;
312 if (!kcm)
319 WRITE_ONCE(kcm->rx_psock, NULL);
321 /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with
326 if (unlikely(kcm->done)) {
332 INIT_WORK(&kcm->done_work, kcm_done_work);
333 schedule_work(&kcm->done_work);
337 if (unlikely(kcm->rx_disabled)) {
338 requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
339 } else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) {
343 kcm_rcv_ready(kcm);
368 struct kcm_sock *kcm;
371 kcm = reserve_rx_kcm(psock, skb);
372 if (!kcm) {
379 if (kcm_queue_rcv_skb(&kcm->sk, skb)) {
419 struct kcm_sock *kcm;
431 kcm = psock->tx_kcm;
432 if (kcm && !unlikely(kcm->tx_stopped))
433 queue_work(kcm_wq, &kcm->tx_work);
440 static void unreserve_psock(struct kcm_sock *kcm);
442 /* kcm sock is locked. */
443 static struct kcm_psock *reserve_psock(struct kcm_sock *kcm)
445 struct kcm_mux *mux = kcm->mux;
448 psock = kcm->tx_psock;
453 WARN_ON(kcm->tx_wait);
455 unreserve_psock(kcm);
457 return kcm->tx_psock;
465 psock = kcm->tx_psock;
467 WARN_ON(kcm->tx_wait);
469 return kcm->tx_psock;
477 if (kcm->tx_wait) {
478 list_del(&kcm->wait_psock_list);
479 kcm->tx_wait = false;
481 kcm->tx_psock = psock;
482 psock->tx_kcm = kcm;
484 } else if (!kcm->tx_wait) {
485 list_add_tail(&kcm->wait_psock_list,
487 kcm->tx_wait = true;
499 struct kcm_sock *kcm;
505 kcm = list_first_entry(&mux->kcm_tx_waiters,
508 list_del(&kcm->wait_psock_list);
509 kcm->tx_wait = false;
510 psock->tx_kcm = kcm;
517 kcm->tx_psock = psock;
519 queue_work(kcm_wq, &kcm->tx_work);
523 /* kcm sock is locked. */
524 static void unreserve_psock(struct kcm_sock *kcm)
527 struct kcm_mux *mux = kcm->mux;
531 psock = kcm->tx_psock;
542 WARN_ON(kcm->tx_wait);
544 kcm->tx_psock = NULL;
570 static void kcm_report_tx_retry(struct kcm_sock *kcm)
572 struct kcm_mux *mux = kcm->mux;
579 /* Write any messages ready on the kcm socket. Called with kcm sock lock
582 static int kcm_write_msgs(struct kcm_sock *kcm)
585 struct sock *sk = &kcm->sk;
590 kcm->tx_wait_more = false;
591 psock = kcm->tx_psock;
596 unreserve_psock(kcm);
597 kcm_report_tx_retry(kcm);
615 psock = reserve_psock(kcm);
663 unreserve_psock(kcm);
667 kcm_report_tx_retry(kcm);
701 unreserve_psock(kcm);
712 struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work);
713 struct sock *sk = &kcm->sk;
721 err = kcm_write_msgs(kcm);
725 report_csk_error(&kcm->sk, -err);
740 static void kcm_push(struct kcm_sock *kcm)
742 if (kcm->tx_wait_more)
743 kcm_write_msgs(kcm);
749 struct kcm_sock *kcm = kcm_sk(sk);
765 if (kcm->seq_skb) {
767 head = kcm->seq_skb;
774 kcm_push(kcm);
785 kcm_push(kcm);
886 kcm_push(kcm);
898 kcm->seq_skb = NULL;
899 KCM_STATS_INCR(kcm->stats.tx_msgs);
903 kcm->tx_wait_more = true;
904 } else if (kcm->tx_wait_more || not_busy) {
905 err = kcm_write_msgs(kcm);
913 report_csk_error(&kcm->sk, -err);
920 kcm->seq_skb = head;
925 KCM_STATS_ADD(kcm->stats.tx_bytes, copied);
931 kcm_push(kcm);
939 if (head != kcm->seq_skb)
943 kcm->seq_skb = NULL;
959 struct kcm_sock *kcm = kcm_sk(sk);
965 kcm_write_msgs(kcm);
973 struct kcm_sock *kcm = kcm_sk(sk);
996 KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1009 KCM_STATS_INCR(kcm->stats.rx_msgs);
1023 struct kcm_sock *kcm = kcm_sk(sk);
1048 KCM_STATS_ADD(kcm->stats.rx_bytes, copied);
1067 /* kcm sock lock held */
1068 static void kcm_recv_disable(struct kcm_sock *kcm)
1070 struct kcm_mux *mux = kcm->mux;
1072 if (kcm->rx_disabled)
1077 kcm->rx_disabled = 1;
1080 if (!kcm->rx_psock) {
1081 if (kcm->rx_wait) {
1082 list_del(&kcm->wait_rx_list);
1084 WRITE_ONCE(kcm->rx_wait, false);
1087 requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);
1093 /* kcm sock lock held */
1094 static void kcm_recv_enable(struct kcm_sock *kcm)
1096 struct kcm_mux *mux = kcm->mux;
1098 if (!kcm->rx_disabled)
1103 kcm->rx_disabled = 0;
1104 kcm_rcv_ready(kcm);
1112 struct kcm_sock *kcm = kcm_sk(sock->sk);
1129 lock_sock(&kcm->sk);
1131 kcm_recv_disable(kcm);
1133 kcm_recv_enable(kcm);
1134 release_sock(&kcm->sk);
1146 struct kcm_sock *kcm = kcm_sk(sock->sk);
1162 val = kcm->rx_disabled;
1175 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)
1185 kcm->sk.sk_state = TCP_ESTABLISHED;
1187 /* Add to mux's kcm sockets list */
1188 kcm->mux = mux;
1199 list_add(&kcm->kcm_sock_list, head);
1200 kcm->index = index;
1205 INIT_WORK(&kcm->tx_work, kcm_tx_work);
1208 kcm_rcv_ready(kcm);
1215 struct kcm_sock *kcm = kcm_sk(sock->sk);
1216 struct kcm_mux *mux = kcm->mux;
1353 * be no way to reserve a kcm for this psock.
1400 * up in the kcm paths, we need kcm lock which can not be
1443 struct kcm_sock *kcm = kcm_sk(sock->sk);
1444 struct kcm_mux *mux = kcm->mux;
1499 /* Clone a kcm socket. */
1620 static void kcm_done(struct kcm_sock *kcm)
1622 struct kcm_mux *mux = kcm->mux;
1623 struct sock *sk = &kcm->sk;
1627 if (kcm->rx_psock) {
1629 WARN_ON(kcm->done);
1630 kcm->rx_disabled = 1;
1631 kcm->done = 1;
1636 if (kcm->rx_wait) {
1637 list_del(&kcm->wait_rx_list);
1639 WRITE_ONCE(kcm->rx_wait, false);
1641 /* Move any pending receive messages to other kcm sockets */
1652 list_del(&kcm->kcm_sock_list);
1663 WARN_ON(kcm->rx_wait);
1665 sock_put(&kcm->sk);
1674 struct kcm_sock *kcm;
1681 kcm = kcm_sk(sk);
1682 mux = kcm->mux;
1686 kfree_skb(kcm->seq_skb);
1694 /* Set tx_stopped. This is checked when psock is bound to a kcm and we
1698 kcm->tx_stopped = 1;
1703 if (kcm->tx_wait) {
1705 * that a psock will be assigned to this kcm.
1707 list_del(&kcm->wait_psock_list);
1708 kcm->tx_wait = false;
1713 * to the kcm socket.
1715 cancel_work_sync(&kcm->tx_work);
1718 psock = kcm->tx_psock;
1722 * need to do this after removing kcm from tx_wait list.
1725 unreserve_psock(kcm);
1729 WARN_ON(kcm->tx_wait);
1730 WARN_ON(kcm->tx_psock);
1734 kcm_done(kcm);
1782 /* Create proto operation for kcm sockets */
1808 /* Allocate a kcm mux, shared between KCM sockets */