Lines Matching refs:psock
53 static void kcm_abort_tx_psock(struct kcm_psock *psock, int err,
56 struct sock *csk = psock->sk;
57 struct kcm_mux *mux = psock->mux;
63 if (psock->tx_stopped) {
68 psock->tx_stopped = 1;
69 KCM_STATS_INCR(psock->stats.tx_aborts);
71 if (!psock->tx_kcm) {
73 list_del(&psock->psock_avail_list);
75 /* In this case psock is being aborted while outside of
76 * write_msgs and psock is reserved. Schedule tx_work
82 queue_work(kcm_wq, &psock->tx_kcm->tx_work);
93 struct kcm_psock *psock)
96 psock->strp.stats.bytes -
97 psock->saved_rx_bytes);
99 psock->strp.stats.msgs - psock->saved_rx_msgs;
100 psock->saved_rx_msgs = psock->strp.stats.msgs;
101 psock->saved_rx_bytes = psock->strp.stats.bytes;
105 struct kcm_psock *psock)
108 psock->stats.tx_bytes - psock->saved_tx_bytes);
110 psock->stats.tx_msgs - psock->saved_tx_msgs;
111 psock->saved_tx_msgs = psock->stats.tx_msgs;
112 psock->saved_tx_bytes = psock->stats.tx_bytes;
119 * pending ready messages on a psock. RX mux lock held.
124 struct kcm_psock *psock;
140 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,
143 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) {
149 /* Consumed the ready message on the psock. Schedule rx_work to
152 list_del(&psock->psock_ready_list);
153 psock->ready_rx_msg = NULL;
157 strp_unpause(&psock->strp);
158 strp_check_rcv(&psock->strp);
252 static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock,
255 struct kcm_mux *mux = psock->mux;
258 WARN_ON(psock->ready_rx_msg);
260 if (psock->rx_kcm)
261 return psock->rx_kcm;
265 if (psock->rx_kcm) {
267 return psock->rx_kcm;
270 kcm_update_rx_mux_stats(mux, psock);
273 psock->ready_rx_msg = head;
274 strp_pause(&psock->strp);
275 list_add_tail(&psock->psock_ready_list,
287 psock->rx_kcm = kcm;
289 WRITE_ONCE(kcm->rx_psock, psock);
304 static void unreserve_rx_kcm(struct kcm_psock *psock,
307 struct kcm_sock *kcm = psock->rx_kcm;
308 struct kcm_mux *mux = psock->mux;
315 psock->rx_kcm = NULL;
349 struct kcm_psock *psock;
353 psock = (struct kcm_psock *)sk->sk_user_data;
354 if (likely(psock))
355 strp_data_ready(&psock->strp);
363 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
367 kcm = reserve_rx_kcm(psock, skb);
369 /* Unable to reserve a KCM, message is held in psock and strp
377 unreserve_rx_kcm(psock, false);
384 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
385 struct bpf_prog *prog = psock->bpf_prog;
394 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp);
396 unreserve_rx_kcm(psock, true);
413 struct kcm_psock *psock;
419 psock = (struct kcm_psock *)sk->sk_user_data;
420 if (unlikely(!psock))
422 mux = psock->mux;
427 kcm = psock->tx_kcm;
442 struct kcm_psock *psock;
444 psock = kcm->tx_psock;
448 if (psock) {
450 if (unlikely(psock->tx_stopped))
458 /* Check again under lock to see if psock was reserved for this
459 * psock via psock_unreserve.
461 psock = kcm->tx_psock;
462 if (unlikely(psock)) {
469 psock = list_first_entry(&mux->psocks_avail,
472 list_del(&psock->psock_avail_list);
477 kcm->tx_psock = psock;
478 psock->tx_kcm = kcm;
479 KCM_STATS_INCR(psock->stats.reserved);
488 return psock;
492 static void psock_now_avail(struct kcm_psock *psock)
494 struct kcm_mux *mux = psock->mux;
498 list_add_tail(&psock->psock_avail_list,
506 psock->tx_kcm = kcm;
513 kcm->tx_psock = psock;
514 KCM_STATS_INCR(psock->stats.reserved);
522 struct kcm_psock *psock;
527 psock = kcm->tx_psock;
529 if (WARN_ON(!psock)) {
536 kcm_update_tx_mux_stats(mux, psock);
541 psock->tx_kcm = NULL;
542 KCM_STATS_INCR(psock->stats.unreserved);
544 if (unlikely(psock->tx_stopped)) {
545 if (psock->done) {
547 list_del(&psock->psock_list);
549 sock_put(psock->sk);
550 fput(psock->sk->sk_socket->file);
551 kmem_cache_free(kcm_psockp, psock);
561 psock_now_avail(psock);
581 struct kcm_psock *psock;
589 psock = kcm->tx_psock;
590 if (unlikely(psock && psock->tx_stopped)) {
591 /* A reserved psock was aborted asynchronously. Unreserve
610 if (WARN_ON(!psock)) {
623 psock = reserve_psock(kcm);
624 if (!psock)
650 ret = kernel_sendpage(psock->sk->sk_socket,
670 * psock since it has lost framing
674 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE,
687 KCM_STATS_ADD(psock->stats.tx_bytes, ret);
709 KCM_STATS_INCR(psock->stats.tx_msgs);
1202 /* If a psock is reserved we'll do cleanup in unreserve */
1340 struct kcm_psock *psock = NULL, *tpsock;
1369 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL);
1370 if (!psock) {
1375 psock->mux = mux;
1376 psock->sk = csk;
1377 psock->bpf_prog = prog;
1386 kmem_cache_free(kcm_psockp, psock);
1391 err = strp_init(&psock->strp, csk, &cb);
1394 kmem_cache_free(kcm_psockp, psock);
1398 psock->save_data_ready = csk->sk_data_ready;
1399 psock->save_write_space = csk->sk_write_space;
1400 psock->save_state_change = csk->sk_state_change;
1401 csk->sk_user_data = psock;
1410 /* Finished initialization, now add the psock to the MUX. */
1420 list_add(&psock->psock_list, head);
1421 psock->index = index;
1425 psock_now_avail(psock);
1429 strp_check_rcv(&psock->strp);
1467 static void kcm_unattach(struct kcm_psock *psock)
1469 struct sock *csk = psock->sk;
1470 struct kcm_mux *mux = psock->mux;
1475 * be no way to reserve a kcm for this psock.
1479 csk->sk_data_ready = psock->save_data_ready;
1480 csk->sk_write_space = psock->save_write_space;
1481 csk->sk_state_change = psock->save_state_change;
1482 strp_stop(&psock->strp);
1484 if (WARN_ON(psock->rx_kcm)) {
1492 /* Stop receiver activities. After this point psock should not be
1495 if (psock->ready_rx_msg) {
1496 list_del(&psock->psock_ready_list);
1497 kfree_skb(psock->ready_rx_msg);
1498 psock->ready_rx_msg = NULL;
1508 strp_done(&psock->strp);
1511 bpf_prog_put(psock->bpf_prog);
1515 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);
1516 save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);
1520 if (psock->tx_kcm) {
1521 /* psock was reserved. Just mark it finished and we will clean
1532 kcm_abort_tx_psock(psock, EPIPE, false);
1535 if (!psock->tx_kcm) {
1536 /* psock now unreserved in window mux was unlocked */
1539 psock->done = 1;
1544 /* Queue tx work to make sure psock->done is handled */
1545 queue_work(kcm_wq, &psock->tx_kcm->tx_work);
1549 if (!psock->tx_stopped)
1550 list_del(&psock->psock_avail_list);
1551 list_del(&psock->psock_list);
1557 kmem_cache_free(kcm_psockp, psock);
1567 struct kcm_psock *psock;
1586 list_for_each_entry(psock, &mux->psocks, psock_list) {
1587 if (psock->sk != csk)
1590 /* Found the matching psock */
1592 if (psock->unattaching || WARN_ON(psock->done)) {
1597 psock->unattaching = 1;
1602 kcm_unattach(psock);
1715 struct kcm_psock *psock, *tmp_psock;
1718 list_for_each_entry_safe(psock, tmp_psock,
1720 if (!WARN_ON(psock->unattaching))
1721 kcm_unattach(psock);
1798 struct kcm_psock *psock;
1816 /* Set tx_stopped. This is checked when psock is bound to a kcm and we
1818 * from the callback (unbinding the psock occurs after canceling work.
1827 * that a psock will be assigned to this kcm.
1840 psock = kcm->tx_psock;
1841 if (psock) {
1842 /* A psock was reserved, so we need to kill it since it
1846 kcm_abort_tx_psock(psock, EPIPE, false);