Lines Matching refs:iommu
17 #include <linux/iommu.h>
27 #include "iommu.h"
97 static void iommu_write_regl(loongson_iommu *iommu, unsigned long off, u32 val)
99 *(u32 *)(iommu->membase + off) = val;
103 static u32 iommu_read_regl(loongson_iommu *iommu, unsigned long off)
107 val = *(u32 *)(iommu->membase + off);
112 static void iommu_translate_disable(loongson_iommu *iommu)
116 if (iommu == NULL) {
117 pr_err("%s iommu is NULL", __func__);
121 val = iommu_read_regl(iommu, LA_IOMMU_EIVDB);
125 iommu_write_regl(iommu, LA_IOMMU_EIVDB, val);
128 val = iommu_read_regl(iommu, LA_IOMMU_CMD);
130 iommu_write_regl(iommu, LA_IOMMU_CMD, val);
133 static void iommu_translate_enable(loongson_iommu *iommu)
137 if (iommu == NULL) {
138 pr_err("%s iommu is NULL", __func__);
142 val = iommu_read_regl(iommu, LA_IOMMU_EIVDB);
146 iommu_write_regl(iommu, LA_IOMMU_EIVDB, val);
149 val = iommu_read_regl(iommu, LA_IOMMU_CMD);
151 iommu_write_regl(iommu, LA_IOMMU_CMD, val);
173 static int has_dom(loongson_iommu *iommu)
175 spin_lock(&iommu->dom_info_lock);
176 while (!list_empty(&iommu->dom_list)) {
177 spin_unlock(&iommu->dom_info_lock);
180 spin_unlock(&iommu->dom_info_lock);
190 loongson_iommu *iommu;
198 if (dev_data->iommu == NULL) {
199 pr_err("%s iommu is NULL", __func__);
208 iommu = dev_data->iommu;
214 index = find_first_zero_bit(iommu->devtable_bitmap, MAX_ATTACHED_DEV_ID);
216 __set_bit(index, iommu->devtable_bitmap);
224 " iommu segment %d flag %x\n",
226 iommu->segment, flag);
233 iommu_write_regl(iommu, LA_IOMMU_EIVDB, val);
235 val = iommu_read_regl(iommu, LA_IOMMU_CMD);
237 iommu_write_regl(iommu, LA_IOMMU_CMD, val);
242 " iommu segment %d flag %x\n",
244 iommu->segment, flag);
246 val = iommu_read_regl(iommu, LA_IOMMU_EIVDB);
249 iommu_write_regl(iommu, LA_IOMMU_EIVDB, val);
251 val = iommu_read_regl(iommu, LA_IOMMU_CMD);
253 iommu_write_regl(iommu, LA_IOMMU_CMD, val);
256 __clear_bit(index, iommu->devtable_bitmap);
262 static void flush_iotlb(loongson_iommu *iommu)
266 if (iommu == NULL) {
267 pr_err("%s iommu is NULL", __func__);
271 val = iommu_read_regl(iommu, LA_IOMMU_VBTC);
276 iommu_write_regl(iommu, LA_IOMMU_VBTC, val);
278 cmd = iommu_read_regl(iommu, LA_IOMMU_CMD);
280 iommu_write_regl(iommu, LA_IOMMU_CMD, cmd);
283 static int flush_pgtable_is_busy(loongson_iommu *iommu)
287 if (iommu == NULL) {
288 pr_err("%s iommu is NULL", __func__);
292 val = iommu_read_regl(iommu, LA_IOMMU_VBTC);
297 static int __iommu_flush_iotlb_all(loongson_iommu *iommu)
301 if (iommu == NULL) {
302 pr_err("%s iommu is NULL", __func__);
306 flush_iotlb(iommu);
307 while (flush_pgtable_is_busy(iommu)) {
315 iommu_translate_enable(iommu);
320 static void priv_flush_iotlb_pde(loongson_iommu *iommu)
322 if (iommu == NULL) {
323 pr_err("%s iommu is NULL", __func__);
327 __iommu_flush_iotlb_all(iommu);
344 priv_flush_iotlb_pde(dev_data->iommu);
386 static int domain_id_alloc(loongson_iommu *iommu)
390 if (iommu == NULL) {
391 pr_err("%s iommu is NULL", __func__);
395 spin_lock(&iommu->domain_bitmap_lock);
396 id = find_first_zero_bit(iommu->domain_bitmap, MAX_DOMAIN_ID);
398 __set_bit(id, iommu->domain_bitmap);
402 spin_unlock(&iommu->domain_bitmap_lock);
407 static void domain_id_free(loongson_iommu *iommu, int id)
409 if (iommu == NULL) {
410 pr_err("%s iommu is NULL", __func__);
414 spin_lock(&iommu->domain_bitmap_lock);
416 __clear_bit(id, iommu->domain_bitmap);
418 spin_unlock(&iommu->domain_bitmap_lock);
424 static void add_domain_to_list(loongson_iommu *iommu, dom_info *priv)
426 spin_lock(&iommu->dom_info_lock);
427 list_add(&priv->list, &iommu->dom_list);
428 spin_unlock(&iommu->dom_info_lock);
431 static void del_domain_from_list(loongson_iommu *iommu, dom_info *priv)
433 spin_lock(&iommu->dom_info_lock);
435 spin_unlock(&iommu->dom_info_lock);
438 static spt_entry *iommu_zalloc_page(loongson_iommu *iommu)
444 spin_lock(&iommu->pgtable_bitmap_lock);
445 index = find_first_zero_bit(iommu->pgtable_bitmap, iommu->maxpages);
446 if (index < iommu->maxpages)
447 __set_bit(index, iommu->pgtable_bitmap);
448 spin_unlock(&iommu->pgtable_bitmap_lock);
451 if (index < iommu->maxpages) {
465 addr = iommu->pgtbase + index * IOMMU_PAGE_SIZE;
473 spin_lock(&iommu->pgtable_bitmap_lock);
474 __clear_bit(index, iommu->pgtable_bitmap);
475 spin_unlock(&iommu->pgtable_bitmap_lock);
479 static void iommu_free_page(loongson_iommu *iommu, spt_entry *shadw_entry)
483 if (shadw_entry->index < iommu->maxpages) {
487 spin_lock(&iommu->pgtable_bitmap_lock);
488 __clear_bit(shadw_entry->index, iommu->pgtable_bitmap);
489 spin_unlock(&iommu->pgtable_bitmap_lock);
504 loongson_iommu *iommu = iommu_entry->iommu;
509 iommu_free_page(iommu, shd_entry);
522 iommu_free_page(iommu, shd_entry);
528 loongson_iommu *iommu;
530 iommu = iommu_entry->iommu;
594 loongson_iommu *iommu = NULL;
609 iommu = iommu_entry->iommu;
610 if (iommu == NULL) {
611 pr_err("%s iommu is NULL", __func__);
615 domain_id_free(iommu_entry->iommu, iommu_entry->id);
617 mutex_lock(&iommu->loongson_iommu_pgtlock);
619 mutex_unlock(&iommu->loongson_iommu_pgtlock);
626 del_domain_from_list(iommu, priv);
633 loongson_iommu *iommu = NULL;
640 iommu = iommu_entry->iommu;
649 __iommu_flush_iotlb_all(iommu);
651 if (!has_dom(iommu))
652 iommu_translate_disable(iommu);
677 loongson_iommu *iommu = NULL;
687 return iommu;
690 iommu = rlookupentry->loongson_iommu_rlookup_table[devid];
692 return iommu;
702 loongson_iommu *iommu = NULL;
725 iommu = find_iommu_by_dev(pdev);
726 if (iommu == NULL)
727 pci_info(pdev, "%s find iommu failed by dev\n", __func__);
731 dev_data->iommu = iommu;
790 iommu_info *get_iommu_entry(dom_info *priv, loongson_iommu *iommu)
796 if (iommu_entry->iommu == iommu) {
806 iommu_info *domain_attach_iommu(dom_info *priv, loongson_iommu *iommu)
813 iommu_entry = get_iommu_entry(priv, iommu);
824 iommu_entry->iommu = iommu;
825 iommu_entry->id = domain_id_alloc(iommu);
832 shd_entry = iommu_zalloc_page(iommu);
835 domain_id_free(iommu, iommu_entry->id);
844 pgd_pa = iommu_pgt_v2p(iommu, shd_entry->gmem_ptable);
847 iommu_write_regl(iommu, LA_IOMMU_DIR_CTRL(iommu_entry->id), dir_ctrl);
848 iommu_write_regl(iommu, LA_IOMMU_PGD_HI(iommu_entry->id), pgd_hi);
849 iommu_write_regl(iommu, LA_IOMMU_PGD_LO(iommu_entry->id), pgd_lo);
855 add_domain_to_list(iommu, priv);
856 pr_info("%s iommu_entry->iommu %lx id %x\n", __func__,
857 (unsigned long)iommu_entry->iommu, iommu_entry->id);
863 loongson_iommu *iommu, unsigned long bdf)
868 entry = get_iommu_entry(info, iommu);
893 loongson_iommu *iommu;
909 iommu = dev_data->iommu;
910 if (iommu == NULL) {
911 pci_info(pdev, "%s iommu is Invalid\n", __func__);
915 pci_info(pdev, "%s busnum %x bdf %x priv %lx iommu %lx\n", __func__,
916 busnum, bdf, (unsigned long)priv, (unsigned long)iommu);
917 dev_data = iommu_get_devdata(priv, iommu, bdf);
928 iommu_entry = domain_attach_iommu(priv, iommu);
930 pci_info(pdev, "domain attach iommu failed\n");
949 loongson_iommu *iommu;
965 iommu = dev_data->iommu;
966 if (iommu == NULL) {
967 pci_info(pdev, "%s iommu is Invalid\n", __func__);
971 dev_data = iommu_get_devdata(priv, iommu, bdf);
978 iommu = dev_data->iommu;
980 iommu_entry = get_iommu_entry(priv, iommu);
990 pci_info(pdev, "%s iommu devid %x sigment %x\n", __func__,
991 iommu->devid, iommu->segment);
1016 static int _iommu_alloc_ptable(loongson_iommu *iommu,
1024 new_shd_entry = iommu_zalloc_page(iommu);
1033 pte = iommu_pgt_v2p(iommu, new_phwentry) & IOMMU_PAGE_MASK;
1041 static size_t iommu_ptw_map(loongson_iommu *iommu, spt_entry *shd_entry,
1081 ret = _iommu_alloc_ptable(iommu, psentry, phwentry);
1084 iommu_ptw_map(iommu, (spt_entry *)*psentry, start, next, pa, level - 1);
1103 loongson_iommu *iommu;
1106 iommu = iommu_entry->iommu;
1108 mutex_lock(&iommu->loongson_iommu_pgtlock);
1110 map_size = iommu_ptw_map(iommu, entry, start, end, pa, IOMMU_LEVEL_MAX - 1);
1114 if (has_dom(iommu))
1115 __iommu_flush_iotlb_all(iommu);
1116 mutex_unlock(&iommu->loongson_iommu_pgtlock);
1121 static size_t iommu_ptw_unmap(loongson_iommu *iommu, spt_entry *shd_entry,
1148 iommu_ptw_unmap(iommu, (spt_entry *)*psentry, start, next, level - 1);
1188 loongson_iommu *iommu;
1194 iommu = iommu_entry->iommu;
1195 mutex_lock(&iommu->loongson_iommu_pgtlock);
1197 unmap_len = iommu_ptw_unmap(iommu, entry, start, end, (IOMMU_LEVEL_MAX - 1));
1199 if (has_dom(iommu))
1200 __iommu_flush_iotlb_all(iommu);
1201 mutex_unlock(&iommu->loongson_iommu_pgtlock);
1258 loongson_iommu *iommu;
1279 iommu = iommu_entry->iommu;
1281 mutex_lock(&iommu->loongson_iommu_pgtlock);
1284 mutex_unlock(&iommu->loongson_iommu_pgtlock);
1321 loongson_iommu *iommu;
1325 iommu = iommu_entry->iommu;
1327 ret = __iommu_flush_iotlb_all(iommu);
1359 loongson_iommu *iommu = NULL;
1364 list_for_each_entry(iommu, &loongson_iommu_list, list) {
1365 if ((iommu->segment == pcisegment) &&
1366 (iommu->devid == devid)) {
1367 return iommu;
1380 struct loongson_iommu *iommu = NULL;
1383 iommu = loongarch_get_iommu(pdev);
1384 if (iommu == NULL) {
1385 pci_info(pdev, "%s can't find iommu\n", __func__);
1396 iommu->membase = ioremap(base, size);
1397 if (iommu->membase == NULL) {
1398 pci_info(pdev, "%s iommu pci dev bar0 is NULL\n", __func__);
1408 iommu->pgtbase = ioremap(base, size);
1409 if (iommu->pgtbase == NULL)
1412 iommu->maxpages = size / IOMMU_PAGE_SIZE;
1413 pr_info("iommu membase %p pgtbase %p pgtsize %llx maxpages %lx\n", iommu->membase, iommu->pgtbase, size, iommu->maxpages);
1416 iommu->domain_bitmap = bitmap_zalloc(bitmap_sz, GFP_KERNEL);
1417 if (iommu->domain_bitmap == NULL) {
1424 iommu->devtable_bitmap = bitmap_zalloc(bitmap_sz, GFP_KERNEL);
1425 if (iommu->devtable_bitmap == NULL) {
1430 tmp = iommu->maxpages / 8;
1431 bitmap_sz = (iommu->maxpages % 8) ? (tmp + 1) : tmp;
1432 iommu->pgtable_bitmap = bitmap_zalloc(bitmap_sz, GFP_KERNEL);
1433 if (iommu->pgtable_bitmap == NULL) {
1443 kfree(iommu->devtable_bitmap);
1444 iommu->devtable_bitmap = NULL;
1446 kfree(iommu->domain_bitmap);
1447 iommu->domain_bitmap = NULL;
1455 struct loongson_iommu *iommu = NULL;
1457 iommu = loongarch_get_iommu(pdev);
1458 if (iommu == NULL)
1461 if (iommu->domain_bitmap != NULL) {
1462 kfree(iommu->domain_bitmap);
1463 iommu->domain_bitmap = NULL;
1466 if (iommu->devtable_bitmap != NULL) {
1467 kfree(iommu->devtable_bitmap);
1468 iommu->devtable_bitmap = NULL;
1471 if (iommu->pgtable_bitmap != NULL) {
1472 kfree(iommu->pgtable_bitmap);
1473 iommu->pgtable_bitmap = NULL;
1476 iommu->membase = NULL;
1477 iommu->pgtbase = NULL;
1520 static void __init set_iommu_for_device(loongson_iommu *iommu,
1525 rlookupentry = lookup_rlooptable(iommu->segment);
1527 rlookupentry = create_rlookup_entry(iommu->segment);
1530 rlookupentry->loongson_iommu_rlookup_table[devid] = iommu;
1583 pr_err("loongson-iommu: Unsupported IVHD type %#x\n", h->type);
1654 static int __init init_iommu_from_acpi(loongson_iommu *iommu,
1668 pr_err("loongarch iommu: Unsupported IVHD type %#x\n", h->type);
1683 set_iommu_for_device(iommu, dev_i);
1691 set_iommu_for_device(iommu, devid);
1707 set_iommu_for_device(iommu, dev_i);
1725 static int __init init_iommu_one(loongson_iommu *iommu, struct ivhd_header *h)
1730 spin_lock_init(&iommu->domain_bitmap_lock);
1731 spin_lock_init(&iommu->dom_info_lock);
1732 spin_lock_init(&iommu->pgtable_bitmap_lock);
1733 mutex_init(&iommu->loongson_iommu_pgtlock);
1736 INIT_LIST_HEAD(&iommu->dom_list);
1738 list_add_tail(&iommu->list, &loongson_iommu_list);
1741 * Copy data from ACPI table entry to the iommu struct
1743 iommu->devid = h->devid;
1744 iommu->segment = h->pci_seg;
1746 ret = init_iommu_from_acpi(iommu, h);
1748 pr_err("%s init iommu from acpi failed\n", __func__);
1752 rlookupentry = lookup_rlooptable(iommu->segment);
1758 rlookupentry->loongson_iommu_rlookup_table[iommu->devid] = NULL;
1773 loongson_iommu *iommu;
1788 iommu = kzalloc(sizeof(loongson_iommu), GFP_KERNEL);
1789 if (iommu == NULL) {
1790 pr_info("%s alloc memory for iommu failed\n", __func__);
1794 ret = init_iommu_one(iommu, h);
1796 kfree(iommu);
1797 pr_info("%s init iommu failed\n", __func__);
1897 loongson_iommu *iommu;
1905 iommu = kzalloc(sizeof(loongson_iommu), GFP_KERNEL);
1906 if (iommu == NULL) {
1907 pr_info("%s alloc memory for iommu failed\n", __func__);
1911 spin_lock_init(&iommu->domain_bitmap_lock);
1912 spin_lock_init(&iommu->dom_info_lock);
1913 spin_lock_init(&iommu->pgtable_bitmap_lock);
1914 mutex_init(&iommu->loongson_iommu_pgtlock);
1917 INIT_LIST_HEAD(&iommu->dom_list);
1919 list_add_tail(&iommu->list, &loongson_iommu_list);
1922 * Copy data from ACPI table entry to the iommu struct
1924 iommu->devid = 0xd0;
1925 iommu->segment = 0;
1928 set_iommu_for_device(iommu, dev_i);
1930 rlookupentry = lookup_rlooptable(iommu->segment);
1936 rlookupentry->loongson_iommu_rlookup_table[iommu->devid] = NULL;
1944 loongson_iommu *iommu = NULL;
1948 iommu = list_first_entry(&loongson_iommu_list, loongson_iommu, list);
1949 list_del(&iommu->list);
1950 kfree(iommu);
1997 .name = "loongson-iommu",
2011 pr_err("Failed to init iommu by ivrs\n");
2015 pr_err("Failed to init iommu by stub\n");