Lines Matching defs:napi
132 #include <trace/events/napi.h>
915 struct napi_struct *napi;
922 napi = napi_by_id(napi_id);
924 return napi ? napi->dev : NULL;
1431 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
4461 struct napi_struct *napi)
4467 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4471 * read on napi->thread. Only call
4474 thread = READ_ONCE(napi->thread);
4478 * makes sure to proceed with napi polling
4482 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
4488 list_add_tail(&napi->poll_list, &sd->poll_list);
4489 WRITE_ONCE(napi->list_owner, smp_processor_id());
5977 static int process_backlog(struct napi_struct *napi, int quota)
5979 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5991 napi->weight = READ_ONCE(dev_rx_weight);
6009 * only current cpu owns and manipulates this napi,
6015 napi->state = 0;
6045 * napi_schedule_prep - check if napi can be scheduled
6046 * @n: napi context
6101 * 1) Don't let napi dequeue from the cpu poll list
6148 * because we will call napi->poll() one more time.
6171 struct napi_struct *napi;
6173 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6174 if (napi->napi_id == napi_id)
6175 return napi;
6182 static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
6185 gro_normal_list(napi);
6186 __napi_schedule(napi);
6190 if (napi->gro_bitmask) {
6194 napi_gro_flush(napi, HZ >= 1000);
6197 gro_normal_list(napi);
6198 clear_bit(NAPI_STATE_SCHED, &napi->state);
6201 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6211 * Since we are about to call napi->poll() once more, we can safely
6217 clear_bit(NAPI_STATE_MISSED, &napi->state);
6218 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6223 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6224 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6225 if (napi->defer_hard_irqs_count && timeout) {
6226 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6234 rc = napi->poll(napi, budget);
6235 /* We can't gro_normal_list() here, because napi->poll() might have
6236 * rearmed the napi (napi_complete_done()) in which case it could
6239 trace_napi_poll(napi, rc, budget);
6242 __busy_poll_stop(napi, skip_schedule);
6251 int (*napi_poll)(struct napi_struct *napi, int budget);
6253 struct napi_struct *napi;
6260 napi = napi_by_id(napi_id);
6261 if (!napi)
6271 unsigned long val = READ_ONCE(napi->state);
6273 /* If multiple threads are competing for this napi,
6274 * we avoid dirtying napi->state as much as we can.
6279 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6282 if (cmpxchg(&napi->state, val,
6286 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6289 have_poll_lock = netpoll_poll_lock(napi);
6290 napi_poll = napi->poll;
6292 work = napi_poll(napi, budget);
6293 trace_napi_poll(napi, work, budget);
6294 gro_normal_list(napi);
6297 __NET_ADD_STATS(dev_net(napi->dev),
6306 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6318 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
6328 static void napi_hash_add(struct napi_struct *napi)
6330 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
6340 napi->napi_id = napi_gen_id;
6342 hlist_add_head_rcu(&napi->napi_hash_node,
6343 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6349 * is respected before freeing memory containing @napi
6351 static void napi_hash_del(struct napi_struct *napi)
6355 hlist_del_init_rcu(&napi->napi_hash_node);
6362 struct napi_struct *napi;
6364 napi = container_of(timer, struct napi_struct, timer);
6369 if (!napi_disable_pending(napi) &&
6370 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6371 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
6372 __napi_schedule_irqoff(napi);
6378 static void init_gro_hash(struct napi_struct *napi)
6383 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6384 napi->gro_hash[i].count = 0;
6386 napi->gro_bitmask = 0;
6391 struct napi_struct *napi;
6398 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6399 if (!napi->thread) {
6400 err = napi_kthread_create(napi);
6416 /* Setting/unsetting threaded mode on a napi might not immediately
6417 * take effect, if the current napi instance is actively being
6422 list_for_each_entry(napi, &dev->napi_list, dev_list)
6423 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
6429 void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
6432 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6435 INIT_LIST_HEAD(&napi->poll_list);
6436 INIT_HLIST_NODE(&napi->napi_hash_node);
6437 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6438 napi->timer.function = napi_watchdog;
6439 init_gro_hash(napi);
6440 napi->skb = NULL;
6441 INIT_LIST_HEAD(&napi->rx_list);
6442 napi->rx_count = 0;
6443 napi->poll = poll;
6447 napi->weight = weight;
6448 napi->dev = dev;
6450 napi->poll_owner = -1;
6452 napi->list_owner = -1;
6453 set_bit(NAPI_STATE_SCHED, &napi->state);
6454 set_bit(NAPI_STATE_NPSVC, &napi->state);
6455 list_add_rcu(&napi->dev_list, &dev->napi_list);
6456 napi_hash_add(napi);
6457 napi_get_frags_check(napi);
6458 /* Create kthread for this napi if dev->threaded is set.
6462 if (dev->threaded && napi_kthread_create(napi))
6512 static void flush_gro_hash(struct napi_struct *napi)
6519 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6521 napi->gro_hash[i].count = 0;
6526 void __netif_napi_del(struct napi_struct *napi)
6528 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
6531 napi_hash_del(napi);
6532 list_del_rcu(&napi->dev_list);
6533 napi_free_frags(napi);
6535 flush_gro_hash(napi);
6536 napi->gro_bitmask = 0;
6538 if (napi->thread) {
6539 kthread_stop(napi->thread);
6540 napi->thread = NULL;
6606 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6636 static int napi_thread_wait(struct napi_struct *napi)
6644 * kthread owns this napi and could poll on this napi.
6648 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
6649 WARN_ON(!list_empty(&napi->poll_list));
6655 /* woken being true indicates this thread owns this napi. */
6687 struct napi_struct *napi = data;
6691 while (!napi_thread_wait(napi)) {
6701 have = netpoll_poll_lock(napi);
6702 __napi_poll(napi, &repoll);
11295 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11299 list_del_init(&napi->poll_list);
11300 if (napi->poll == process_backlog)
11301 napi->state = 0;
11303 ____napi_schedule(sd, napi);