Lines Matching defs:its
26 static int vgic_its_save_tables_v0(struct vgic_its *its);
27 static int vgic_its_restore_tables_v0(struct vgic_its *its);
28 static int vgic_its_commit_v0(struct vgic_its *its);
34 * If this LPI is already mapped on another ITS, we increase its refcount
171 int (*save_tables)(struct vgic_its *its);
172 int (*restore_tables)(struct vgic_its *its);
173 int (*commit)(struct vgic_its *its);
192 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
194 return &its_table_abi_versions[its->abi_rev];
197 static int vgic_its_set_abi(struct vgic_its *its, u32 rev)
201 its->abi_rev = rev;
202 abi = vgic_its_get_abi(its);
203 return abi->commit(its);
210 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
214 list_for_each_entry(device, &its->device_list, dev_list)
226 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
232 device = find_its_device(its, device_id);
244 #define for_each_lpi_its(dev, ite, its) \
245 list_for_each_entry(dev, &(its)->device_list, dev_list) \
259 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
263 list_for_each_entry(collection, &its->collection_list, coll_list) {
402 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
408 for_each_lpi_its(device, ite, its) {
480 struct vgic_its *its,
483 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
502 struct vgic_its *its,
507 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
513 struct vgic_its *its,
521 return vgic_its_set_abi(its, rev);
525 struct vgic_its *its,
600 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,
623 db = its->vgic_its_base + GITS_TRANSLATER;
676 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
682 if (!its->enabled)
685 ite = find_ite(its, devid, eventid);
696 vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
727 return iodev->its;
737 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
744 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);
786 struct vgic_its *its;
792 its = vgic_msi_to_its(kvm, msi);
793 if (IS_ERR(its))
794 return PTR_ERR(its);
796 mutex_lock(&its->its_lock);
797 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);
798 mutex_unlock(&its->its_lock);
849 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
856 ite = find_ite(its, device_id, event_id);
876 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
886 ite = find_ite(its, device_id, event_id);
893 collection = find_collection(its, coll_id);
905 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)
911 idx = srcu_read_lock(&its->dev->kvm->srcu);
912 ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
913 srcu_read_unlock(&its->dev->kvm->srcu, idx);
924 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
958 return __is_visible_gfn_locked(its, addr);
967 if (kvm_read_guest_lock(its->dev->kvm,
988 return __is_visible_gfn_locked(its, indirect_ptr);
995 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,
998 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1007 return __is_visible_gfn_locked(its, gpa);
1014 static int vgic_its_alloc_collection(struct vgic_its *its,
1027 list_add_tail(&collection->coll_list, &its->collection_list);
1033 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
1044 collection = find_collection(its, coll_id);
1048 for_each_lpi_its(device, ite, its)
1079 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
1092 device = find_its_device(its, device_id);
1096 if (!vgic_its_check_event_id(its, device, event_id))
1108 if (find_ite(its, device_id, event_id))
1111 collection = find_collection(its, coll_id);
1115 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
1118 ret = vgic_its_alloc_collection(its, &collection, coll_id);
1127 vgic_its_free_collection(its, coll_id);
1137 vgic_its_free_collection(its, coll_id);
1165 /* its lock must be held */
1166 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)
1170 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)
1174 /* its lock must be held */
1175 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)
1179 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)
1180 vgic_its_free_collection(its, cur->collection_id);
1184 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
1199 list_add_tail(&device->dev_list, &its->device_list);
1207 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
1216 if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
1222 device = find_its_device(its, device_id);
1239 device = vgic_its_alloc_device(its, device_id, itt_addr,
1249 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
1265 vgic_its_free_collection(its, coll_id);
1268 collection = find_collection(its, coll_id);
1273 if (!vgic_its_check_id(its, its->baser_coll_table,
1277 ret = vgic_its_alloc_collection(its, &collection,
1284 update_affinity_collection(kvm, its, collection);
1295 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1303 ite = find_ite(its, device_id, event_id);
1325 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1333 ite = find_ite(its, device_id, event_id);
1382 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1389 collection = find_collection(its, coll_id);
1407 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1451 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1457 return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1464 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1469 mutex_lock(&its->its_lock);
1472 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1475 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1478 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1481 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1484 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1487 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1490 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1493 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1496 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1499 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1502 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1509 mutex_unlock(&its->its_lock);
1551 struct vgic_its *its,
1554 return extract_bytes(its->cbaser, addr & 7, len);
1557 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1562 if (its->enabled)
1565 mutex_lock(&its->cmd_lock);
1566 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1567 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1568 its->creadr = 0;
1573 its->cwriter = its->creadr;
1574 mutex_unlock(&its->cmd_lock);
1582 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1588 if (!its->enabled)
1591 cbaser = GITS_CBASER_ADDRESS(its->cbaser);
1593 while (its->cwriter != its->creadr) {
1594 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1604 vgic_its_handle_command(kvm, its, cmd_buf);
1606 its->creadr += ITS_CMD_SIZE;
1607 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1608 its->creadr = 0;
1618 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1624 if (!its)
1627 mutex_lock(&its->cmd_lock);
1629 reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1631 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1632 mutex_unlock(&its->cmd_lock);
1635 its->cwriter = reg;
1637 vgic_its_process_commands(kvm, its);
1639 mutex_unlock(&its->cmd_lock);
1643 struct vgic_its *its,
1646 return extract_bytes(its->cwriter, addr & 0x7, len);
1650 struct vgic_its *its,
1653 return extract_bytes(its->creadr, addr & 0x7, len);
1657 struct vgic_its *its,
1664 mutex_lock(&its->cmd_lock);
1666 if (its->enabled) {
1672 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1677 its->creadr = cmd_offset;
1679 mutex_unlock(&its->cmd_lock);
1685 struct vgic_its *its,
1692 reg = its->baser_device_table;
1695 reg = its->baser_coll_table;
1707 struct vgic_its *its,
1711 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1716 if (its->enabled)
1721 regptr = &its->baser_device_table;
1726 regptr = &its->baser_coll_table;
1747 mutex_lock(&its->its_lock);
1750 vgic_its_free_device_list(kvm, its);
1753 vgic_its_free_collection_list(kvm, its);
1756 mutex_unlock(&its->its_lock);
1761 struct vgic_its *its,
1766 mutex_lock(&its->cmd_lock);
1767 if (its->creadr == its->cwriter)
1769 if (its->enabled)
1771 mutex_unlock(&its->cmd_lock);
1776 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1780 mutex_lock(&its->cmd_lock);
1786 if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1787 (!(its->baser_device_table & GITS_BASER_VALID) ||
1788 !(its->baser_coll_table & GITS_BASER_VALID) ||
1789 !(its->cbaser & GITS_CBASER_VALID)))
1792 its->enabled = !!(val & GITS_CTLR_ENABLE);
1793 if (!its->enabled)
1800 vgic_its_process_commands(kvm, its);
1803 mutex_unlock(&its->cmd_lock);
1825 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1867 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1870 struct vgic_io_device *iodev = &its->iodev;
1874 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1879 its->vgic_its_base = addr;
1884 iodev->base_addr = its->vgic_its_base;
1886 iodev->its = its;
1950 struct vgic_its *its;
1955 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);
1956 if (!its)
1965 kfree(its);
1972 mutex_init(&its->its_lock);
1973 mutex_init(&its->cmd_lock);
1977 mutex_lock(&its->cmd_lock);
1978 mutex_lock(&its->its_lock);
1979 mutex_unlock(&its->its_lock);
1980 mutex_unlock(&its->cmd_lock);
1983 its->vgic_its_base = VGIC_ADDR_UNDEF;
1985 INIT_LIST_HEAD(&its->device_list);
1986 INIT_LIST_HEAD(&its->collection_list);
1990 its->enabled = false;
1991 its->dev = dev;
1993 its->baser_device_table = INITIAL_BASER_VALUE |
1995 its->baser_coll_table = INITIAL_BASER_VALUE |
1999 dev->private = its;
2001 ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);
2011 struct vgic_its *its = kvm_dev->private;
2013 mutex_lock(&its->its_lock);
2015 vgic_its_free_device_list(kvm, its);
2016 vgic_its_free_collection_list(kvm, its);
2018 mutex_unlock(&its->its_lock);
2019 kfree(its);
2049 struct vgic_its *its;
2054 its = dev->private;
2081 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
2094 addr = its->vgic_its_base + offset;
2100 ret = region->uaccess_its_write(dev->kvm, its, addr,
2103 region->its_write(dev->kvm, its, addr, len, *reg);
2105 *reg = region->its_read(dev->kvm, its, addr, len);
2143 * @its: its handle
2151 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2158 * @its: its handle
2169 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,
2172 struct kvm *kvm = its->dev->kvm;
2189 next_offset = fn(its, id, entry, opaque);
2207 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
2210 struct kvm *kvm = its->dev->kvm;
2228 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
2233 struct kvm *kvm = its->dev->kvm;
2259 collection = find_collection(its, coll_id);
2263 if (!vgic_its_check_event_id(its, dev, event_id))
2295 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
2297 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2317 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
2327 * @its: its handle
2332 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
2334 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2340 ret = scan_its_table(its, base, max_size, ite_esz, 0,
2353 * @its: ITS handle
2357 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
2360 struct kvm *kvm = its->dev->kvm;
2365 next_offset = compute_next_devid_offset(&its->device_list, dev);
2377 * @its: its handle
2385 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2389 u64 baser = its->baser_device_table;
2410 if (!vgic_its_check_id(its, baser, id, NULL))
2413 dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2417 ret = vgic_its_restore_itt(its, dev);
2419 vgic_its_free_device(its->dev->kvm, dev);
2445 static int vgic_its_save_device_tables(struct vgic_its *its)
2447 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2448 u64 baser = its->baser_device_table;
2455 list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2457 list_for_each_entry(dev, &its->device_list, dev_list) {
2461 if (!vgic_its_check_id(its, baser,
2465 ret = vgic_its_save_itt(its, dev);
2469 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2479 * @its: its handle
2488 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2491 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2505 ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2515 static int vgic_its_restore_device_tables(struct vgic_its *its)
2517 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2518 u64 baser = its->baser_device_table;
2530 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2534 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2543 vgic_its_free_device_list(its->dev->kvm, its);
2548 static int vgic_its_save_cte(struct vgic_its *its,
2558 return vgic_write_guest_lock(its->dev->kvm, gpa, &val, esz);
2566 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2569 struct kvm *kvm = its->dev->kvm;
2589 collection = find_collection(its, coll_id);
2593 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
2596 ret = vgic_its_alloc_collection(its, &collection, coll_id);
2607 static int vgic_its_save_collection_table(struct vgic_its *its)
2609 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2610 u64 baser = its->baser_coll_table;
2622 list_for_each_entry(collection, &its->collection_list, coll_list) {
2623 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2639 ret = vgic_write_guest_lock(its->dev->kvm, gpa, &val, cte_esz);
2648 static int vgic_its_restore_collection_table(struct vgic_its *its)
2650 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2651 u64 baser = its->baser_coll_table;
2665 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2676 vgic_its_free_collection_list(its->dev->kvm, its);
2685 static int vgic_its_save_tables_v0(struct vgic_its *its)
2689 ret = vgic_its_save_device_tables(its);
2693 return vgic_its_save_collection_table(its);
2701 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2705 ret = vgic_its_restore_collection_table(its);
2709 ret = vgic_its_restore_device_tables(its);
2711 vgic_its_free_collection_list(its->dev->kvm, its);
2715 static int vgic_its_commit_v0(struct vgic_its *its)
2719 abi = vgic_its_get_abi(its);
2720 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2721 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2723 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2726 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2731 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)
2734 its->baser_coll_table &= ~GITS_BASER_VALID;
2735 its->baser_device_table &= ~GITS_BASER_VALID;
2736 its->cbaser = 0;
2737 its->creadr = 0;
2738 its->cwriter = 0;
2739 its->enabled = 0;
2740 vgic_its_free_device_list(kvm, its);
2741 vgic_its_free_collection_list(kvm, its);
2772 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
2774 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2788 mutex_lock(&its->its_lock);
2792 vgic_its_reset(kvm, its);
2795 ret = abi->save_tables(its);
2798 ret = abi->restore_tables(its);
2802 mutex_unlock(&its->its_lock);
2815 * dirty guest pages. When vgic/its tables are being saved, the backup
2829 struct vgic_its *its = dev->private;
2844 ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,
2849 return vgic_register_its_iodev(dev->kvm, its, addr);
2852 return vgic_its_ctrl(dev->kvm, its, attr->attr);
2871 struct vgic_its *its = dev->private;
2872 u64 addr = its->vgic_its_base;
2901 .name = "kvm-arm-vgic-its",