Lines Matching refs:vpe
266 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
268 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
269 return vpe->col_idx;
272 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
274 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
281 struct its_vpe *vpe = NULL;
285 vpe = irq_data_get_irq_chip_data(d);
289 vpe = map->vpe;
292 if (vpe) {
293 cpu = vpe_to_cpuid_lock(vpe, flags);
307 struct its_vpe *vpe = NULL;
310 vpe = irq_data_get_irq_chip_data(d);
314 vpe = map->vpe;
317 if (vpe)
318 vpe_to_cpuid_unlock(vpe, flags);
329 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
331 if (valid_col(its->collections + vpe->col_idx))
332 return vpe;
390 struct its_vpe *vpe;
394 struct its_vpe *vpe;
400 struct its_vpe *vpe;
408 struct its_vpe *vpe;
415 struct its_vpe *vpe;
422 struct its_vpe *vpe;
426 struct its_vpe *vpe;
778 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
782 return valid_vpe(its, desc->its_vinvall_cmd.vpe);
794 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
799 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
806 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
816 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
818 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
830 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
835 return valid_vpe(its, desc->its_vmapp_cmd.vpe);
845 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
851 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
858 return valid_vpe(its, desc->its_vmapti_cmd.vpe);
868 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
874 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
881 return valid_vpe(its, desc->its_vmovi_cmd.vpe);
894 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
899 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
904 return valid_vpe(its, desc->its_vmovp_cmd.vpe);
922 return valid_vpe(its, map->vpe);
940 return valid_vpe(its, map->vpe);
958 return valid_vpe(its, map->vpe);
969 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
973 return valid_vpe(its, desc->its_invdb_cmd.vpe);
984 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
993 return valid_vpe(its, desc->its_vsgi_cmd.vpe);
1276 desc.its_vmapti_cmd.vpe = map->vpe;
1290 desc.its_vmovi_cmd.vpe = map->vpe;
1299 struct its_vpe *vpe, bool valid)
1303 desc.its_vmapp_cmd.vpe = vpe;
1305 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1310 static void its_send_vmovp(struct its_vpe *vpe)
1315 int col_id = vpe->col_idx;
1317 desc.its_vmovp_cmd.vpe = vpe;
1337 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1344 if (!require_its_list_vmovp(vpe->its_vm, its))
1354 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1358 desc.its_vinvall_cmd.vpe = vpe;
1404 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1408 desc.its_invdb_cmd.vpe = vpe;
1484 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1805 struct its_vpe *vpe = vm->vpes[i];
1806 struct irq_data *d = irq_get_irq_data(vpe->irq);
1809 vpe->col_idx = cpumask_first(cpu_online_mask);
1810 its_send_vmapp(its, vpe, true);
1811 its_send_vinvall(its, vpe);
1812 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
3730 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3737 if (vpe->vpe_proxy_event == -1)
3740 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3741 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3751 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3753 vpe->vpe_proxy_event = -1;
3756 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3766 its_vpe_db_proxy_unmap_locked(vpe);
3771 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3778 if (vpe->vpe_proxy_event != -1)
3786 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3787 vpe->vpe_proxy_event = vpe_proxy.next_victim;
3790 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3791 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3794 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3807 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3815 its_vpe_db_proxy_map_locked(vpe);
3818 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3819 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3828 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3842 * protect us, and that we must ensure nobody samples vpe->col_idx
3844 * taken on any vLPI handling path that evaluates vpe->col_idx.
3846 from = vpe_to_cpuid_lock(vpe, &flags);
3861 vpe->col_idx = cpu;
3863 its_send_vmovp(vpe);
3864 its_vpe_db_proxy_move(vpe, from, cpu);
3868 vpe_to_cpuid_unlock(vpe, flags);
3887 static void its_vpe_schedule(struct its_vpe *vpe)
3893 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3902 val = virt_to_phys(page_address(vpe->vpt_page)) &
3911 * easily. So in the end, vpe->pending_last is only an
3918 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3923 static void its_vpe_deschedule(struct its_vpe *vpe)
3930 vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3931 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3934 static void its_vpe_invall(struct its_vpe *vpe)
3942 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3949 its_send_vinvall(its, vpe);
3956 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3961 its_vpe_schedule(vpe);
3965 its_vpe_deschedule(vpe);
3973 its_vpe_invall(vpe);
3981 static void its_vpe_send_cmd(struct its_vpe *vpe,
3988 its_vpe_db_proxy_map_locked(vpe);
3989 cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3996 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4001 its_vpe_send_cmd(vpe, its_send_inv);
4027 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4035 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4037 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4039 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
4044 its_vpe_send_cmd(vpe, its_send_int);
4046 its_vpe_send_cmd(vpe, its_send_clear);
4058 .name = "GICv4-vpe",
4087 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4097 its_send_invdb(its, vpe);
4112 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4122 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4127 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4146 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4150 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4151 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4160 vpe->pending_last = true;
4164 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4172 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4175 cpu = vpe_to_cpuid_lock(vpe, &flags);
4182 vpe_to_cpuid_unlock(vpe, flags);
4187 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4192 its_vpe_4_1_schedule(vpe, info);
4196 its_vpe_4_1_deschedule(vpe, info);
4204 its_vpe_4_1_invall(vpe);
4213 .name = "GICv4.1-vpe",
4223 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4226 desc.its_vsgi_cmd.vpe = vpe;
4228 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4229 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4230 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4243 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4245 vpe->sgi_config[d->hwirq].enabled = false;
4251 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4253 vpe->sgi_config[d->hwirq].enabled = true;
4278 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4282 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4295 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4315 cpu = vpe_to_cpuid_lock(vpe, &flags);
4318 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4335 vpe_to_cpuid_unlock(vpe, flags);
4347 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4352 vpe->sgi_config[d->hwirq].priority = info->priority;
4353 vpe->sgi_config[d->hwirq].group = info->group;
4376 struct its_vpe *vpe = args;
4383 vpe->sgi_config[i].priority = 0;
4384 vpe->sgi_config[i].enabled = false;
4385 vpe->sgi_config[i].group = false;
4388 &its_sgi_irq_chip, vpe);
4413 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4425 vpe->sgi_config[d->hwirq].enabled = false;
4447 static int its_vpe_init(struct its_vpe *vpe)
4470 raw_spin_lock_init(&vpe->vpe_lock);
4471 vpe->vpe_id = vpe_id;
4472 vpe->vpt_page = vpt_page;
4474 atomic_set(&vpe->vmapp_count, 0);
4476 vpe->vpe_proxy_event = -1;
4481 static void its_vpe_teardown(struct its_vpe *vpe)
4483 its_vpe_db_proxy_unmap(vpe);
4484 its_vpe_id_free(vpe->vpe_id);
4485 its_free_pending_table(vpe->vpt_page);
4500 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4502 BUG_ON(vm != vpe->its_vm);
4505 its_vpe_teardown(vpe);
4578 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4590 vpe->col_idx = cpumask_first(cpu_online_mask);
4596 its_send_vmapp(its, vpe, true);
4597 its_send_vinvall(its, vpe);
4600 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4608 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4622 its_send_vmapp(its, vpe, false);
4630 if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4631 gic_flush_dcache_to_poc(page_address(vpe->vpt_page),