Lines Matching defs:tfile
227 struct tun_file *tfile = container_of(napi, struct tun_file, napi);
228 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
265 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
268 tfile->napi_enabled = napi_en;
269 tfile->napi_frags_enabled = napi_en && napi_frags;
271 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll);
272 napi_enable(&tfile->napi);
276 static void tun_napi_enable(struct tun_file *tfile)
278 if (tfile->napi_enabled)
279 napi_enable(&tfile->napi);
282 static void tun_napi_disable(struct tun_file *tfile)
284 if (tfile->napi_enabled)
285 napi_disable(&tfile->napi);
288 static void tun_napi_del(struct tun_file *tfile)
290 if (tfile->napi_enabled)
291 netif_napi_del(&tfile->napi);
294 static bool tun_napi_frags_enabled(const struct tun_file *tfile)
296 return tfile->napi_frags_enabled;
474 struct tun_file *tfile)
479 u16 queue_index = tfile->queue_index;
593 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
595 tfile->detached = tun;
596 list_add_tail(&tfile->next, &tun->disabled);
600 static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
602 struct tun_struct *tun = tfile->detached;
604 tfile->detached = NULL;
605 list_del_init(&tfile->next);
624 static void tun_queue_purge(struct tun_file *tfile)
628 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
631 skb_queue_purge(&tfile->sk.sk_write_queue);
632 skb_queue_purge(&tfile->sk.sk_error_queue);
635 static void __tun_detach(struct tun_file *tfile, bool clean)
640 tun = rtnl_dereference(tfile->tun);
643 if (!tfile->detached)
644 tun_napi_disable(tfile);
645 tun_napi_del(tfile);
648 if (tun && !tfile->detached) {
649 u16 index = tfile->queue_index;
662 RCU_INIT_POINTER(tfile->tun, NULL);
663 sock_put(&tfile->sk);
665 tun_disable_queue(tun, tfile);
666 tun_napi_disable(tfile);
672 tun_queue_purge(tfile);
674 } else if (tfile->detached && clean) {
675 tun = tun_enable_queue(tfile);
676 sock_put(&tfile->sk);
688 xdp_rxq_info_unreg(&tfile->xdp_rxq);
689 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
693 static void tun_detach(struct tun_file *tfile, bool clean)
699 tun = rtnl_dereference(tfile->tun);
701 __tun_detach(tfile, clean);
707 sock_put(&tfile->sk);
713 struct tun_file *tfile, *tmp;
717 tfile = rtnl_dereference(tun->tfiles[i]);
718 BUG_ON(!tfile);
719 tun_napi_disable(tfile);
720 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
721 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
722 RCU_INIT_POINTER(tfile->tun, NULL);
725 list_for_each_entry(tfile, &tun->disabled, next) {
726 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
727 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
728 RCU_INIT_POINTER(tfile->tun, NULL);
734 tfile = rtnl_dereference(tun->tfiles[i]);
735 tun_napi_del(tfile);
737 tun_queue_purge(tfile);
738 xdp_rxq_info_unreg(&tfile->xdp_rxq);
739 sock_put(&tfile->sk);
741 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
742 tun_napi_del(tfile);
743 tun_enable_queue(tfile);
744 tun_queue_purge(tfile);
745 xdp_rxq_info_unreg(&tfile->xdp_rxq);
746 sock_put(&tfile->sk);
758 struct tun_file *tfile = file->private_data;
762 err = security_tun_dev_attach(tfile->socket.sk, tun->security);
767 if (rtnl_dereference(tfile->tun) && !tfile->detached)
775 if (!tfile->detached &&
783 lock_sock(tfile->socket.sk);
784 err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
785 release_sock(tfile->socket.sk);
790 if (!tfile->detached &&
791 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
797 tfile->queue_index = tun->numqueues;
798 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
800 if (tfile->detached) {
801 /* Re-attach detached tfile, updating XDP queue_index */
802 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
804 if (tfile->xdp_rxq.queue_index != tfile->queue_index)
805 tfile->xdp_rxq.queue_index = tfile->queue_index;
807 /* Setup XDP RX-queue info, for new tfile getting attached */
808 err = xdp_rxq_info_reg(&tfile->xdp_rxq,
809 tun->dev, tfile->queue_index, 0);
812 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
815 xdp_rxq_info_unreg(&tfile->xdp_rxq);
821 if (tfile->detached) {
822 tun_enable_queue(tfile);
823 tun_napi_enable(tfile);
825 sock_hold(&tfile->sk);
826 tun_napi_init(tun, tfile, napi, napi_frags);
830 sock_set_flag(&tfile->sk, SOCK_XDP);
836 /* Publish tfile->tun and tun->tfiles only after we've fully
837 * initialized tfile; otherwise we risk using half-initialized
841 rcu_assign_pointer(tfile->tun, tun);
842 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
849 static struct tun_struct *tun_get(struct tun_file *tfile)
854 tun = rcu_dereference(tfile->tun);
1077 struct tun_file *tfile;
1081 tfile = rcu_dereference(tun->tfiles[txq]);
1084 if (!tfile) {
1102 if (tfile->socket.sk->sk_filter &&
1103 sk_filter(tfile->socket.sk, skb)) {
1133 if (ptr_ring_produce(&tfile->tx_ring, skb)) {
1143 if (tfile->flags & TUN_FASYNC)
1144 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1145 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1200 struct tun_file *tfile;
1210 tfile = rtnl_dereference(tun->tfiles[i]);
1212 sock_set_flag(&tfile->sk, SOCK_XDP);
1214 sock_reset_flag(&tfile->sk, SOCK_XDP);
1216 list_for_each_entry(tfile, &tun->disabled, next) {
1218 sock_set_flag(&tfile->sk, SOCK_XDP);
1220 sock_reset_flag(&tfile->sk, SOCK_XDP);
1264 static void __tun_xdp_flush_tfile(struct tun_file *tfile)
1267 if (tfile->flags & TUN_FASYNC)
1268 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1269 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1276 struct tun_file *tfile;
1293 tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1295 if (unlikely(!tfile))
1298 spin_lock(&tfile->tx_ring.producer_lock);
1306 if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
1312 spin_unlock(&tfile->tx_ring.producer_lock);
1315 __tun_xdp_flush_tfile(tfile);
1417 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
1419 struct sock *sk = tfile->socket.sk;
1429 struct tun_file *tfile = file->private_data;
1430 struct tun_struct *tun = tun_get(tfile);
1437 sk = tfile->socket.sk;
1441 if (!ptr_ring_empty(&tfile->tx_ring))
1449 if (tun_sock_writeable(tun, tfile) ||
1451 tun_sock_writeable(tun, tfile)))
1461 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
1475 skb = napi_get_frags(&tfile->napi);
1512 napi_free_frags(&tfile->napi);
1518 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
1522 struct sock *sk = tfile->socket.sk;
1545 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
1548 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1555 skb_record_rx_queue(skb, tfile->queue_index);
1576 skb_record_rx_queue(nskb, tfile->queue_index);
1579 skb_record_rx_queue(skb, tfile->queue_index);
1585 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
1591 if (tfile->socket.sk->sk_sndbuf != INT_MAX)
1607 static struct sk_buff *__tun_build_skb(struct tun_file *tfile,
1618 skb_set_owner_w(skb, tfile->socket.sk);
1665 struct tun_file *tfile,
1702 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len,
1715 xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq);
1741 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad);
1750 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1765 bool frags = tun_napi_frags_enabled(tfile);
1821 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
1826 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
1842 mutex_lock(&tfile->napi_mutex);
1843 skb = tun_napi_alloc_frags(tfile, copylen, from);
1853 skb = tun_alloc_skb(tfile, align, copylen, linear,
1921 skb_record_rx_queue(skb, tfile->queue_index);
1947 !tfile->detached)
1971 napi_free_frags(&tfile->napi);
1973 mutex_unlock(&tfile->napi_mutex);
1977 if (likely(napi_schedule_prep(&tfile->napi))) {
1979 napi_gro_frags(&tfile->napi);
1980 napi_complete(&tfile->napi);
1986 mutex_unlock(&tfile->napi_mutex);
1987 } else if (tfile->napi_enabled) {
1988 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1993 if (unlikely(tfile->detached)) {
2005 napi_schedule(&tfile->napi);
2009 tun_rx_batched(tun, tfile, skb, more);
2020 tun_flow_update(tun, rxhash, tfile);
2034 tfile->napi.skb = NULL;
2035 mutex_unlock(&tfile->napi_mutex);
2044 struct tun_file *tfile = file->private_data;
2045 struct tun_struct *tun = tun_get(tfile);
2055 result = tun_get_user(tun, tfile, NULL, from, noblock, false);
2062 struct tun_file *tfile,
2093 struct tun_file *tfile,
2182 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2188 ptr = ptr_ring_consume(&tfile->tx_ring);
2196 add_wait_queue(&tfile->socket.wq.wait, &wait);
2200 ptr = ptr_ring_consume(&tfile->tx_ring);
2207 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
2216 remove_wait_queue(&tfile->socket.wq.wait, &wait);
2223 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
2237 ptr = tun_ring_recv(tfile, noblock, &err);
2245 ret = tun_put_user_xdp(tun, tfile, xdpf, to);
2250 ret = tun_put_user(tun, tfile, skb, to);
2263 struct tun_file *tfile = file->private_data;
2264 struct tun_struct *tun = tun_get(tfile);
2274 ret = tun_do_read(tun, tfile, to, noblock, NULL);
2419 struct tun_file *tfile;
2433 tfile = container_of(sk, struct tun_file, sk);
2434 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
2444 struct tun_file *tfile,
2467 xdp_init_buff(xdp, buflen, &tfile->xdp_rxq);
2518 skb_record_rx_queue(skb, tfile->queue_index);
2529 !tfile->detached)
2532 if (tfile->napi_enabled) {
2533 queue = &tfile->sk.sk_write_queue;
2536 if (unlikely(tfile->detached)) {
2556 tun_flow_update(tun, rxhash, tfile);
2565 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2566 struct tun_struct *tun = tun_get(tfile);
2586 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
2594 if (tfile->napi_enabled && queued > 0)
2595 napi_schedule(&tfile->napi);
2606 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
2617 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2618 struct tun_struct *tun = tun_get(tfile);
2636 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
2668 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2672 tun = tun_get(tfile);
2676 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
2745 struct tun_file *tfile = file->private_data;
2749 if (tfile->detached)
2837 dev->ifindex = tfile->ifindex;
2848 tun->sndbuf = tfile->socket.sk->sk_sndbuf;
2865 rcu_assign_pointer(tfile->tun, tun);
2938 struct tun_file *tfile;
2941 tfile = rtnl_dereference(tun->tfiles[i]);
2942 lock_sock(tfile->socket.sk);
2943 sk_detach_filter(tfile->socket.sk);
2944 release_sock(tfile->socket.sk);
2953 struct tun_file *tfile;
2956 tfile = rtnl_dereference(tun->tfiles[i]);
2957 lock_sock(tfile->socket.sk);
2958 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
2959 release_sock(tfile->socket.sk);
2972 struct tun_file *tfile;
2976 tfile = rtnl_dereference(tun->tfiles[i]);
2977 tfile->socket.sk->sk_sndbuf = tun->sndbuf;
2983 struct tun_file *tfile = file->private_data;
2990 tun = tfile->detached;
3001 tun = rtnl_dereference(tfile->tun);
3002 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
3005 __tun_detach(tfile, false);
3079 struct tun_file *tfile = file->private_data;
3080 struct net *net = sock_net(&tfile->sk);
3118 tun = tun_get(tfile);
3147 tfile->ifindex = ifindex;
3163 if (tfile->detached)
3165 if (!tfile->socket.sk->sk_filter)
3278 sndbuf = tfile->socket.sk->sk_sndbuf;
3447 struct tun_file *tfile = file->private_data;
3450 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
3455 tfile->flags |= TUN_FASYNC;
3457 tfile->flags &= ~TUN_FASYNC;
3466 struct tun_file *tfile;
3468 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
3470 if (!tfile)
3472 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
3473 sk_free(&tfile->sk);
3477 mutex_init(&tfile->napi_mutex);
3478 RCU_INIT_POINTER(tfile->tun, NULL);
3479 tfile->flags = 0;
3480 tfile->ifindex = 0;
3482 init_waitqueue_head(&tfile->socket.wq.wait);
3484 tfile->socket.file = file;
3485 tfile->socket.ops = &tun_socket_ops;
3487 sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid());
3489 tfile->sk.sk_write_space = tun_sock_write_space;
3490 tfile->sk.sk_sndbuf = INT_MAX;
3492 file->private_data = tfile;
3493 INIT_LIST_HEAD(&tfile->next);
3495 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
3504 struct tun_file *tfile = file->private_data;
3506 tun_detach(tfile, true);
3514 struct tun_file *tfile = file->private_data;
3521 tun = tun_get(tfile);
3664 struct tun_file *tfile;
3674 tfile = rtnl_dereference(tun->tfiles[i]);
3675 rings[i] = &tfile->tx_ring;
3677 list_for_each_entry(tfile, &tun->disabled, next)
3678 rings[i++] = &tfile->tx_ring;
3705 struct tun_file *tfile;
3707 tfile = rtnl_dereference(tun->tfiles[i]);
3708 tfile->socket.sk->sk_write_space(tfile->socket.sk);
3769 struct tun_file *tfile;
3772 tfile = file->private_data;
3773 if (!tfile)
3775 return &tfile->socket;
3781 struct tun_file *tfile;
3785 tfile = file->private_data;
3786 if (!tfile)
3788 return &tfile->tx_ring;