Lines Matching defs:tfile
238 struct tun_file *tfile = container_of(napi, struct tun_file, napi);
239 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
276 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
279 tfile->napi_enabled = napi_en;
280 tfile->napi_frags_enabled = napi_en && napi_frags;
282 netif_tx_napi_add(tun->dev, &tfile->napi, tun_napi_poll,
284 napi_enable(&tfile->napi);
288 static void tun_napi_enable(struct tun_file *tfile)
290 if (tfile->napi_enabled)
291 napi_enable(&tfile->napi);
294 static void tun_napi_disable(struct tun_file *tfile)
296 if (tfile->napi_enabled)
297 napi_disable(&tfile->napi);
300 static void tun_napi_del(struct tun_file *tfile)
302 if (tfile->napi_enabled)
303 netif_napi_del(&tfile->napi);
306 static bool tun_napi_frags_enabled(const struct tun_file *tfile)
308 return tfile->napi_frags_enabled;
486 struct tun_file *tfile)
491 u16 queue_index = tfile->queue_index;
605 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
607 tfile->detached = tun;
608 list_add_tail(&tfile->next, &tun->disabled);
612 static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
614 struct tun_struct *tun = tfile->detached;
616 tfile->detached = NULL;
617 list_del_init(&tfile->next);
636 static void tun_queue_purge(struct tun_file *tfile)
640 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
643 skb_queue_purge(&tfile->sk.sk_write_queue);
644 skb_queue_purge(&tfile->sk.sk_error_queue);
647 static void __tun_detach(struct tun_file *tfile, bool clean)
652 tun = rtnl_dereference(tfile->tun);
655 if (!tfile->detached)
656 tun_napi_disable(tfile);
657 tun_napi_del(tfile);
660 if (tun && !tfile->detached) {
661 u16 index = tfile->queue_index;
673 RCU_INIT_POINTER(tfile->tun, NULL);
674 sock_put(&tfile->sk);
676 tun_disable_queue(tun, tfile);
677 tun_napi_disable(tfile);
683 tun_queue_purge(tfile);
685 } else if (tfile->detached && clean) {
686 tun = tun_enable_queue(tfile);
687 sock_put(&tfile->sk);
699 xdp_rxq_info_unreg(&tfile->xdp_rxq);
700 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
704 static void tun_detach(struct tun_file *tfile, bool clean)
710 tun = rtnl_dereference(tfile->tun);
712 __tun_detach(tfile, clean);
718 sock_put(&tfile->sk);
724 struct tun_file *tfile, *tmp;
728 tfile = rtnl_dereference(tun->tfiles[i]);
729 BUG_ON(!tfile);
730 tun_napi_disable(tfile);
731 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
732 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
733 RCU_INIT_POINTER(tfile->tun, NULL);
736 list_for_each_entry(tfile, &tun->disabled, next) {
737 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
738 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
739 RCU_INIT_POINTER(tfile->tun, NULL);
745 tfile = rtnl_dereference(tun->tfiles[i]);
746 tun_napi_del(tfile);
748 tun_queue_purge(tfile);
749 xdp_rxq_info_unreg(&tfile->xdp_rxq);
750 sock_put(&tfile->sk);
752 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
753 tun_napi_del(tfile);
754 tun_enable_queue(tfile);
755 tun_queue_purge(tfile);
756 xdp_rxq_info_unreg(&tfile->xdp_rxq);
757 sock_put(&tfile->sk);
769 struct tun_file *tfile = file->private_data;
773 err = security_tun_dev_attach(tfile->socket.sk, tun->security);
778 if (rtnl_dereference(tfile->tun) && !tfile->detached)
786 if (!tfile->detached &&
794 lock_sock(tfile->socket.sk);
795 err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
796 release_sock(tfile->socket.sk);
801 if (!tfile->detached &&
802 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
808 tfile->queue_index = tun->numqueues;
809 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
811 if (tfile->detached) {
812 /* Re-attach detached tfile, updating XDP queue_index */
813 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
815 if (tfile->xdp_rxq.queue_index != tfile->queue_index)
816 tfile->xdp_rxq.queue_index = tfile->queue_index;
818 /* Setup XDP RX-queue info, for new tfile getting attached */
819 err = xdp_rxq_info_reg(&tfile->xdp_rxq,
820 tun->dev, tfile->queue_index);
823 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
826 xdp_rxq_info_unreg(&tfile->xdp_rxq);
832 if (tfile->detached) {
833 tun_enable_queue(tfile);
834 tun_napi_enable(tfile);
836 sock_hold(&tfile->sk);
837 tun_napi_init(tun, tfile, napi, napi_frags);
841 sock_set_flag(&tfile->sk, SOCK_XDP);
847 /* Publish tfile->tun and tun->tfiles only after we've fully
848 * initialized tfile; otherwise we risk using half-initialized
852 rcu_assign_pointer(tfile->tun, tun);
853 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
860 static struct tun_struct *tun_get(struct tun_file *tfile)
865 tun = rcu_dereference(tfile->tun);
1087 struct tun_file *tfile;
1091 tfile = rcu_dereference(tun->tfiles[txq]);
1094 if (!tfile)
1108 if (tfile->socket.sk->sk_filter &&
1109 sk_filter(tfile->socket.sk, skb))
1128 if (ptr_ring_produce(&tfile->tx_ring, skb))
1136 if (tfile->flags & TUN_FASYNC)
1137 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1138 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1218 struct tun_file *tfile;
1228 tfile = rtnl_dereference(tun->tfiles[i]);
1230 sock_set_flag(&tfile->sk, SOCK_XDP);
1232 sock_reset_flag(&tfile->sk, SOCK_XDP);
1234 list_for_each_entry(tfile, &tun->disabled, next) {
1236 sock_set_flag(&tfile->sk, SOCK_XDP);
1238 sock_reset_flag(&tfile->sk, SOCK_XDP);
1282 static void __tun_xdp_flush_tfile(struct tun_file *tfile)
1285 if (tfile->flags & TUN_FASYNC)
1286 kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1287 tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1294 struct tun_file *tfile;
1312 tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1314 if (unlikely(!tfile))
1317 spin_lock(&tfile->tx_ring.producer_lock);
1325 if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
1331 spin_unlock(&tfile->tx_ring.producer_lock);
1334 __tun_xdp_flush_tfile(tfile);
1427 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
1429 struct sock *sk = tfile->socket.sk;
1439 struct tun_file *tfile = file->private_data;
1440 struct tun_struct *tun = tun_get(tfile);
1447 sk = tfile->socket.sk;
1451 if (!ptr_ring_empty(&tfile->tx_ring))
1459 if (tun_sock_writeable(tun, tfile) ||
1461 tun_sock_writeable(tun, tfile)))
1471 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
1485 skb = napi_get_frags(&tfile->napi);
1521 napi_free_frags(&tfile->napi);
1527 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
1531 struct sock *sk = tfile->socket.sk;
1552 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
1555 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1562 skb_record_rx_queue(skb, tfile->queue_index);
1583 skb_record_rx_queue(nskb, tfile->queue_index);
1586 skb_record_rx_queue(skb, tfile->queue_index);
1592 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
1598 if (tfile->socket.sk->sk_sndbuf != INT_MAX)
1614 static struct sk_buff *__tun_build_skb(struct tun_file *tfile,
1625 skb_set_owner_w(skb, tfile->socket.sk);
1666 struct tun_file *tfile,
1703 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len,
1720 xdp.rxq = &tfile->xdp_rxq;
1746 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad);
1755 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1771 bool frags = tun_napi_frags_enabled(tfile);
1826 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
1831 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp);
1848 mutex_lock(&tfile->napi_mutex);
1849 skb = tun_napi_alloc_frags(tfile, copylen, from);
1856 skb = tun_alloc_skb(tfile, align, copylen, linear,
1864 mutex_unlock(&tfile->napi_mutex);
1879 tfile->napi.skb = NULL;
1880 mutex_unlock(&tfile->napi_mutex);
1891 tfile->napi.skb = NULL;
1892 mutex_unlock(&tfile->napi_mutex);
1942 skb_record_rx_queue(skb, tfile->queue_index);
1957 tfile->napi.skb = NULL;
1958 mutex_unlock(&tfile->napi_mutex);
1972 !tfile->detached)
1995 napi_free_frags(&tfile->napi);
1997 mutex_unlock(&tfile->napi_mutex);
2001 if (likely(napi_schedule_prep(&tfile->napi))) {
2003 napi_gro_frags(&tfile->napi);
2004 napi_complete(&tfile->napi);
2010 mutex_unlock(&tfile->napi_mutex);
2011 } else if (tfile->napi_enabled) {
2012 struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
2021 napi_schedule(&tfile->napi);
2025 tun_rx_batched(tun, tfile, skb, more);
2039 tun_flow_update(tun, rxhash, tfile);
2047 struct tun_file *tfile = file->private_data;
2048 struct tun_struct *tun = tun_get(tfile);
2058 result = tun_get_user(tun, tfile, NULL, from, noblock, false);
2065 struct tun_file *tfile,
2100 struct tun_file *tfile,
2195 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2201 ptr = ptr_ring_consume(&tfile->tx_ring);
2209 add_wait_queue(&tfile->socket.wq.wait, &wait);
2213 ptr = ptr_ring_consume(&tfile->tx_ring);
2220 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
2229 remove_wait_queue(&tfile->socket.wq.wait, &wait);
2236 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
2250 ptr = tun_ring_recv(tfile, noblock, &err);
2258 ret = tun_put_user_xdp(tun, tfile, xdpf, to);
2263 ret = tun_put_user(tun, tfile, skb, to);
2276 struct tun_file *tfile = file->private_data;
2277 struct tun_struct *tun = tun_get(tfile);
2287 ret = tun_do_read(tun, tfile, to, noblock, NULL);
2433 struct tun_file *tfile;
2447 tfile = container_of(sk, struct tun_file, sk);
2448 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
2458 struct tun_file *tfile,
2484 xdp->rxq = &tfile->xdp_rxq;
2535 skb_record_rx_queue(skb, tfile->queue_index);
2544 !tfile->detached)
2559 tun_flow_update(tun, rxhash, tfile);
2568 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2569 struct tun_struct *tun = tun_get(tfile);
2589 tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
2604 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
2615 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2616 struct tun_struct *tun = tun_get(tfile);
2634 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
2666 struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2670 tun = tun_get(tfile);
2674 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
2743 struct tun_file *tfile = file->private_data;
2747 if (tfile->detached)
2835 dev->ifindex = tfile->ifindex;
2846 tun->sndbuf = tfile->socket.sk->sk_sndbuf;
2863 rcu_assign_pointer(tfile->tun, tun);
2927 struct tun_file *tfile;
2930 tfile = rtnl_dereference(tun->tfiles[i]);
2931 lock_sock(tfile->socket.sk);
2932 sk_detach_filter(tfile->socket.sk);
2933 release_sock(tfile->socket.sk);
2942 struct tun_file *tfile;
2945 tfile = rtnl_dereference(tun->tfiles[i]);
2946 lock_sock(tfile->socket.sk);
2947 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
2948 release_sock(tfile->socket.sk);
2961 struct tun_file *tfile;
2965 tfile = rtnl_dereference(tun->tfiles[i]);
2966 tfile->socket.sk->sk_sndbuf = tun->sndbuf;
2972 struct tun_file *tfile = file->private_data;
2979 tun = tfile->detached;
2990 tun = rtnl_dereference(tfile->tun);
2991 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
2994 __tun_detach(tfile, false);
3068 struct tun_file *tfile = file->private_data;
3069 struct net *net = sock_net(&tfile->sk);
3108 tun = tun_get(tfile);
3137 tfile->ifindex = ifindex;
3153 if (tfile->detached)
3155 if (!tfile->socket.sk->sk_filter)
3259 sndbuf = tfile->socket.sk->sk_sndbuf;
3428 struct tun_file *tfile = file->private_data;
3431 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
3436 tfile->flags |= TUN_FASYNC;
3438 tfile->flags &= ~TUN_FASYNC;
3447 struct tun_file *tfile;
3449 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
3451 if (!tfile)
3453 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
3454 sk_free(&tfile->sk);
3458 mutex_init(&tfile->napi_mutex);
3459 RCU_INIT_POINTER(tfile->tun, NULL);
3460 tfile->flags = 0;
3461 tfile->ifindex = 0;
3463 init_waitqueue_head(&tfile->socket.wq.wait);
3465 tfile->socket.file = file;
3466 tfile->socket.ops = &tun_socket_ops;
3468 sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid());
3470 tfile->sk.sk_write_space = tun_sock_write_space;
3471 tfile->sk.sk_sndbuf = INT_MAX;
3473 file->private_data = tfile;
3474 INIT_LIST_HEAD(&tfile->next);
3476 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
3483 struct tun_file *tfile = file->private_data;
3485 tun_detach(tfile, true);
3493 struct tun_file *tfile = file->private_data;
3500 tun = tun_get(tfile);
3639 struct tun_file *tfile;
3649 tfile = rtnl_dereference(tun->tfiles[i]);
3650 rings[i] = &tfile->tx_ring;
3652 list_for_each_entry(tfile, &tun->disabled, next)
3653 rings[i++] = &tfile->tx_ring;
3680 struct tun_file *tfile;
3682 tfile = rtnl_dereference(tun->tfiles[i]);
3683 tfile->socket.sk->sk_write_space(tfile->socket.sk);
3744 struct tun_file *tfile;
3747 tfile = file->private_data;
3748 if (!tfile)
3750 return &tfile->socket;
3756 struct tun_file *tfile;
3760 tfile = file->private_data;
3761 if (!tfile)
3763 return &tfile->tx_ring;