Lines Matching refs:cpu_rcache
864 struct iova_cpu_rcache *cpu_rcache;
873 rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
877 cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
878 spin_lock_init(&cpu_rcache->lock);
879 cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
880 cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
896 struct iova_cpu_rcache *cpu_rcache;
900 cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
901 spin_lock_irqsave(&cpu_rcache->lock, flags);
903 if (!iova_magazine_full(cpu_rcache->loaded)) {
905 } else if (!iova_magazine_full(cpu_rcache->prev)) {
906 swap(cpu_rcache->prev, cpu_rcache->loaded);
915 cpu_rcache->loaded;
917 mag_to_free = cpu_rcache->loaded;
921 cpu_rcache->loaded = new_mag;
927 iova_magazine_push(cpu_rcache->loaded, iova_pfn);
929 spin_unlock_irqrestore(&cpu_rcache->lock, flags);
958 struct iova_cpu_rcache *cpu_rcache;
963 cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
964 spin_lock_irqsave(&cpu_rcache->lock, flags);
966 if (!iova_magazine_empty(cpu_rcache->loaded)) {
968 } else if (!iova_magazine_empty(cpu_rcache->prev)) {
969 swap(cpu_rcache->prev, cpu_rcache->loaded);
974 iova_magazine_free(cpu_rcache->loaded);
975 cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
982 iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
984 spin_unlock_irqrestore(&cpu_rcache->lock, flags);
1012 struct iova_cpu_rcache *cpu_rcache;
1019 cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
1020 iova_magazine_free(cpu_rcache->loaded);
1021 iova_magazine_free(cpu_rcache->prev);
1034 struct iova_cpu_rcache *cpu_rcache;
1041 cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
1042 spin_lock_irqsave(&cpu_rcache->lock, flags);
1043 iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
1044 iova_magazine_free_pfns(cpu_rcache->prev, iovad);
1045 spin_unlock_irqrestore(&cpu_rcache->lock, flags);