Lines Matching refs:bo
180 struct iris_bo *bo;
256 static void bo_free(struct iris_bo *bo);
262 struct iris_bo *bo = entry ? entry->data : NULL;
264 if (bo) {
265 assert(iris_bo_is_external(bo));
266 assert(iris_bo_is_real(bo));
267 assert(!bo->real.reusable);
274 if (list_is_linked(&bo->head))
275 list_del(&bo->head);
277 iris_bo_reference(bo);
280 return bo;
438 iris_bo_busy_gem(struct iris_bo *bo)
440 assert(iris_bo_is_real(bo));
442 struct iris_bufmgr *bufmgr = bo->bufmgr;
443 struct drm_i915_gem_busy busy = { .handle = bo->gem_handle };
454 iris_bo_wait_syncobj(struct iris_bo *bo, int64_t timeout_ns)
457 struct iris_bufmgr *bufmgr = bo->bufmgr;
460 if (bo->idle)
465 uint32_t handles[bo->deps_size * IRIS_BATCH_COUNT * 2];
468 for (int d = 0; d < bo->deps_size; d++) {
470 struct iris_syncobj *r = bo->deps[d].read_syncobjs[b];
471 struct iris_syncobj *w = bo->deps[d].write_syncobjs[b];
501 for (int d = 0; d < bo->deps_size; d++) {
503 iris_syncobj_reference(bufmgr, &bo->deps[d].write_syncobjs[b], NULL);
504 iris_syncobj_reference(bufmgr, &bo->deps[d].read_syncobjs[b], NULL);
514 iris_bo_busy_syncobj(struct iris_bo *bo)
516 return iris_bo_wait_syncobj(bo, 0) == -ETIME;
520 iris_bo_busy(struct iris_bo *bo)
523 if (iris_bo_is_external(bo))
524 busy = iris_bo_busy_gem(bo);
526 busy = iris_bo_busy_syncobj(bo);
528 bo->idle = !busy;
534 iris_bo_madvise(struct iris_bo *bo, int state)
537 assert(iris_bo_is_real(bo));
540 .handle = bo->gem_handle,
545 intel_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
553 struct iris_bo *bo = calloc(1, sizeof(*bo));
554 if (!bo)
557 list_inithead(&bo->real.exports);
559 bo->hash = _mesa_hash_pointer(bo);
561 return bo;
565 bo_unmap(struct iris_bo *bo)
567 assert(iris_bo_is_real(bo));
569 VG_NOACCESS(bo->real.map, bo->size);
570 os_munmap(bo->real.map, bo->size);
571 bo->real.map = NULL;
612 struct iris_bo *bo = container_of(entry, struct iris_bo, slab.entry);
614 return !iris_bo_busy(bo);
624 assert(!slab->bo->aux_map_address);
632 struct iris_bo *bo = &slab->entries[i];
633 if (aux_map_ctx && bo->aux_map_address) {
634 intel_aux_map_unmap_range(aux_map_ctx, bo->address, bo->size);
635 bo->aux_map_address = 0;
639 for (int d = 0; d < bo->deps_size; d++) {
641 iris_syncobj_reference(bufmgr, &bo->deps[d].write_syncobjs[b], NULL);
642 iris_syncobj_reference(bufmgr, &bo->deps[d].read_syncobjs[b], NULL);
645 free(bo->deps);
648 iris_bo_unreference(slab->bo);
715 slab->bo =
717 if (!slab->bo)
720 slab_size = slab->bo->size;
732 struct iris_bo *bo = &slab->entries[i];
734 bo->size = entry_size;
735 bo->bufmgr = bufmgr;
736 bo->hash = _mesa_hash_pointer(bo);
737 bo->gem_handle = 0;
738 bo->address = slab->bo->address + i * entry_size;
739 bo->aux_map_address = 0;
740 bo->index = -1;
741 bo->refcount = 0;
742 bo->idle = true;
744 bo->slab.entry.slab = &slab->base;
745 bo->slab.entry.group_index = group_index;
746 bo->slab.entry.entry_size = entry_size;
748 bo->slab.real = iris_get_backing_bo(slab->bo);
750 list_addtail(&bo->slab.entry.head, &slab->base.free);
756 iris_bo_unreference(slab->bo);
831 struct iris_bo *bo = container_of(entry, struct iris_bo, slab.entry);
833 if (bo->aux_map_address && bo->bufmgr->aux_map_ctx) {
840 intel_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->address,
841 bo->size);
842 bo->aux_map_address = 0;
845 p_atomic_set(&bo->refcount, 1);
846 bo->name = name;
847 bo->size = size;
853 void *map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);
855 memset(map, 0, bo->size);
857 pb_slab_free(slabs, &bo->slab.entry);
862 return bo;
877 struct iris_bo *bo = NULL;
905 bo = cur;
913 if (!bo)
916 if (bo->aux_map_address) {
924 if (bo->bufmgr->aux_map_ctx)
925 intel_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->address,
926 bo->size);
927 bo->aux_map_address = 0;
933 if (memzone != iris_memzone_for_address(bo->address) ||
934 bo->address % alignment != 0) {
935 vma_free(bufmgr, bo->address, bo->size);
936 bo->address = 0ull;
943 void *map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);
945 memset(map, 0, bo->size);
947 bo_free(bo);
952 return bo;
958 struct iris_bo *bo = bo_calloc();
959 if (!bo)
962 bo->real.heap = flags_to_heap(bufmgr, flags);
973 switch (bo->real.heap) {
1001 bo->real.heap == IRIS_HEAP_DEVICE_LOCAL_PREFERRED) {
1010 free(bo);
1013 bo->gem_handle = create.handle;
1021 free(bo);
1024 bo->gem_handle = create.handle;
1027 bo->bufmgr = bufmgr;
1028 bo->size = bo_size;
1029 bo->idle = true;
1037 .handle = bo->gem_handle,
1042 intel_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd);
1045 return bo;
1063 struct iris_bo *bo;
1072 bo = alloc_bo_from_slabs(bufmgr, name, size, alignment, flags);
1074 if (bo)
1075 return bo;
1101 bo = alloc_bo_from_cache(bufmgr, bucket, alignment, memzone, mmap_mode,
1105 if (!bo) {
1106 bo = alloc_bo_from_cache(bufmgr, bucket, alignment, memzone, mmap_mode,
1112 if (!bo) {
1113 bo = alloc_fresh_bo(bufmgr, bo_size, flags);
1114 if (!bo)
1118 if (bo->address == 0ull) {
1120 bo->address = vma_alloc(bufmgr, memzone, bo->size, alignment);
1123 if (bo->address == 0ull)
1127 bo->name = name;
1128 p_atomic_set(&bo->refcount, 1);
1129 bo->real.reusable = bucket && bufmgr->bo_reuse;
1130 bo->index = -1;
1131 bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1137 bo->real.kflags |= EXEC_OBJECT_CAPTURE;
1139 assert(bo->real.map == NULL || bo->real.mmap_mode == mmap_mode);
1140 bo->real.mmap_mode = mmap_mode;
1148 .handle = bo->gem_handle,
1154 bo->real.reusable = false;
1157 DBG("bo_create: buf %d (%s) (%s memzone) (%s) %llub\n", bo->gem_handle,
1158 bo->name, memzone_name(memzone), iris_heap_to_string[bo->real.heap],
1161 return bo;
1165 bo_free(bo);
1176 struct iris_bo *bo;
1178 bo = bo_calloc();
1179 if (!bo)
1189 bo->gem_handle = arg.handle;
1194 .handle = bo->gem_handle,
1201 bo->name = name;
1202 bo->size = size;
1203 bo->real.map = ptr;
1205 bo->bufmgr = bufmgr;
1206 bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1209 bo->address = vma_alloc(bufmgr, memzone, size, 1);
1212 if (bo->address == 0ull)
1215 p_atomic_set(&bo->refcount, 1);
1216 bo->real.userptr = true;
1217 bo->index = -1;
1218 bo->idle = true;
1219 bo->real.mmap_mode = IRIS_MMAP_WB;
1221 return bo;
1224 close.handle = bo->gem_handle;
1227 free(bo);
1241 struct iris_bo *bo;
1243 /* At the moment most applications only have a few named bo.
1250 bo = find_and_ref_external_bo(bufmgr->name_table, handle);
1251 if (bo)
1259 bo = NULL;
1266 bo = find_and_ref_external_bo(bufmgr->handle_table, open_arg.handle);
1267 if (bo)
1270 bo = bo_calloc();
1271 if (!bo) {
1277 p_atomic_set(&bo->refcount, 1);
1279 bo->size = open_arg.size;
1280 bo->bufmgr = bufmgr;
1281 bo->gem_handle = open_arg.handle;
1282 bo->name = name;
1283 bo->real.global_name = handle;
1284 bo->real.reusable = false;
1285 bo->real.imported = true;
1286 bo->real.mmap_mode = IRIS_MMAP_NONE;
1287 bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1288 bo->address = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 1);
1290 if (bo->address == 0ull) {
1291 bo_free(bo);
1292 bo = NULL;
1296 _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
1297 _mesa_hash_table_insert(bufmgr->name_table, &bo->real.global_name, bo);
1299 DBG("bo_create_from_handle: %d (%s)\n", handle, bo->name);
1303 return bo;
1307 bo_close(struct iris_bo *bo)
1309 struct iris_bufmgr *bufmgr = bo->bufmgr;
1312 assert(iris_bo_is_real(bo));
1314 if (iris_bo_is_external(bo)) {
1317 if (bo->real.global_name) {
1319 &bo->real.global_name);
1323 entry = _mesa_hash_table_search(bufmgr->handle_table, &bo->gem_handle);
1326 list_for_each_entry_safe(struct bo_export, export, &bo->real.exports, link) {
1334 assert(list_is_empty(&bo->real.exports));
1338 struct drm_gem_close close = { .handle = bo->gem_handle };
1342 bo->gem_handle, bo->name, strerror(errno));
1345 if (bo->aux_map_address && bo->bufmgr->aux_map_ctx) {
1346 intel_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->address,
1347 bo->size);
1351 vma_free(bo->bufmgr, bo->address, bo->size);
1353 for (int d = 0; d < bo->deps_size; d++) {
1355 iris_syncobj_reference(bufmgr, &bo->deps[d].write_syncobjs[b], NULL);
1356 iris_syncobj_reference(bufmgr, &bo->deps[d].read_syncobjs[b], NULL);
1359 free(bo->deps);
1361 free(bo);
1365 bo_free(struct iris_bo *bo)
1367 struct iris_bufmgr *bufmgr = bo->bufmgr;
1370 assert(iris_bo_is_real(bo));
1372 if (!bo->real.userptr && bo->real.map)
1373 bo_unmap(bo);
1375 if (bo->idle) {
1376 bo_close(bo);
1381 list_addtail(&bo->head, &bufmgr->zombie_list);
1399 list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1400 if (time - bo->real.free_time <= 1)
1403 list_del(&bo->head);
1405 bo_free(bo);
1412 list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1413 if (time - bo->real.free_time <= 1)
1416 list_del(&bo->head);
1418 bo_free(bo);
1425 list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1426 if (time - bo->real.free_time <= 1)
1429 list_del(&bo->head);
1431 bo_free(bo);
1435 list_for_each_entry_safe(struct iris_bo, bo, &bufmgr->zombie_list, head) {
1439 if (!bo->idle && iris_bo_busy(bo))
1442 list_del(&bo->head);
1443 bo_close(bo);
1450 bo_unreference_final(struct iris_bo *bo, time_t time)
1452 struct iris_bufmgr *bufmgr = bo->bufmgr;
1455 DBG("bo_unreference final: %d (%s)\n", bo->gem_handle, bo->name);
1457 assert(iris_bo_is_real(bo));
1460 if (bo->real.reusable)
1461 bucket = bucket_for_size(bufmgr, bo->size, bo->real.heap);
1463 if (bucket && iris_bo_madvise(bo, I915_MADV_DONTNEED)) {
1464 bo->real.free_time = time;
1465 bo->name = NULL;
1467 list_addtail(&bo->head, &bucket->head);
1469 bo_free(bo);
1474 iris_bo_unreference(struct iris_bo *bo)
1476 if (bo == NULL)
1479 assert(p_atomic_read(&bo->refcount) > 0);
1481 if (atomic_add_unless(&bo->refcount, -1, 1)) {
1482 struct iris_bufmgr *bufmgr = bo->bufmgr;
1487 if (bo->gem_handle == 0) {
1488 pb_slab_free(get_slabs(bufmgr, bo->size), &bo->slab.entry);
1492 if (p_atomic_dec_zero(&bo->refcount)) {
1493 bo_unreference_final(bo, time.tv_sec);
1504 struct iris_bo *bo,
1507 bool busy = dbg && !bo->idle;
1510 iris_bo_wait_rendering(bo);
1516 action, bo->name, elapsed * 1000);
1540 iris_bo_gem_mmap_legacy(struct util_debug_callback *dbg, struct iris_bo *bo)
1542 struct iris_bufmgr *bufmgr = bo->bufmgr;
1545 assert(iris_bo_is_real(bo));
1546 assert(bo->real.mmap_mode == IRIS_MMAP_WB ||
1547 bo->real.mmap_mode == IRIS_MMAP_WC);
1550 .handle = bo->gem_handle,
1551 .size = bo->size,
1552 .flags = bo->real.mmap_mode == IRIS_MMAP_WC ? I915_MMAP_WC : 0,
1558 __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
1567 iris_bo_gem_mmap_offset(struct util_debug_callback *dbg, struct iris_bo *bo)
1569 struct iris_bufmgr *bufmgr = bo->bufmgr;
1571 assert(iris_bo_is_real(bo));
1574 .handle = bo->gem_handle,
1588 if (bo->real.heap != IRIS_HEAP_SYSTEM_MEMORY)
1589 assert(bo->real.mmap_mode == IRIS_MMAP_WC);
1591 assert(bo->real.mmap_mode == IRIS_MMAP_WB);
1601 assert(bo->real.mmap_mode != IRIS_MMAP_NONE);
1602 assert(bo->real.mmap_mode < ARRAY_SIZE(mmap_offset_for_mode));
1603 mmap_arg.flags = mmap_offset_for_mode[bo->real.mmap_mode];
1610 __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
1615 void *map = mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
1619 __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
1628 struct iris_bo *bo, unsigned flags)
1630 struct iris_bufmgr *bufmgr = bo->bufmgr;
1633 if (bo->gem_handle == 0) {
1634 struct iris_bo *real = iris_get_backing_bo(bo);
1635 uint64_t offset = bo->address - real->address;
1638 assert(bo->real.mmap_mode != IRIS_MMAP_NONE);
1639 if (bo->real.mmap_mode == IRIS_MMAP_NONE)
1642 if (!bo->real.map) {
1643 DBG("iris_bo_map: %d (%s)\n", bo->gem_handle, bo->name);
1644 map = bufmgr->has_mmap_offset ? iris_bo_gem_mmap_offset(dbg, bo)
1645 : iris_bo_gem_mmap_legacy(dbg, bo);
1650 VG_DEFINED(map, bo->size);
1652 if (p_atomic_cmpxchg(&bo->real.map, NULL, map)) {
1653 VG_NOACCESS(map, bo->size);
1654 os_munmap(map, bo->size);
1657 assert(bo->real.map);
1658 map = bo->real.map;
1662 bo->gem_handle, bo->name, bo->real.map);
1666 bo_wait_with_stall_warning(dbg, bo, "memory mapping");
1674 iris_bo_wait_rendering(struct iris_bo *bo)
1679 iris_bo_wait(bo, -1);
1683 iris_bo_wait_gem(struct iris_bo *bo, int64_t timeout_ns)
1685 assert(iris_bo_is_real(bo));
1687 struct iris_bufmgr *bufmgr = bo->bufmgr;
1689 .bo_handle = bo->gem_handle,
1703 * @bo: buffer object to wait for
1728 iris_bo_wait(struct iris_bo *bo, int64_t timeout_ns)
1732 if (iris_bo_is_external(bo))
1733 ret = iris_bo_wait_gem(bo, timeout_ns);
1735 ret = iris_bo_wait_syncobj(bo, timeout_ns);
1740 bo->idle = true;
1766 list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1767 list_del(&bo->head);
1769 bo_free(bo);
1776 list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1777 list_del(&bo->head);
1779 bo_free(bo);
1786 list_for_each_entry_safe(struct iris_bo, bo, &bucket->head, head) {
1787 list_del(&bo->head);
1789 bo_free(bo);
1794 list_for_each_entry_safe(struct iris_bo, bo, &bufmgr->zombie_list, head) {
1795 list_del(&bo->head);
1796 bo_close(bo);
1816 iris_gem_get_tiling(struct iris_bo *bo, uint32_t *tiling)
1818 struct iris_bufmgr *bufmgr = bo->bufmgr;
1825 struct drm_i915_gem_get_tiling ti = { .handle = bo->gem_handle };
1830 bo->gem_handle, strerror(errno));
1839 iris_gem_set_tiling(struct iris_bo *bo, const struct isl_surf *surf)
1841 struct iris_bufmgr *bufmgr = bo->bufmgr;
1856 .handle = bo->gem_handle,
1865 bo->gem_handle, strerror(errno));
1875 struct iris_bo *bo;
1888 * for named buffers, we must not create two bo's pointing at the same
1891 bo = find_and_ref_external_bo(bufmgr->handle_table, handle);
1892 if (bo)
1895 bo = bo_calloc();
1896 if (!bo)
1899 p_atomic_set(&bo->refcount, 1);
1901 /* Determine size of bo. The fd-to-handle ioctl really should
1908 bo->size = ret;
1910 bo->bufmgr = bufmgr;
1911 bo->name = "prime";
1912 bo->real.reusable = false;
1913 bo->real.imported = true;
1914 bo->real.mmap_mode = IRIS_MMAP_NONE;
1915 bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
1916 bo->gem_handle = handle;
1928 bo->address = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 64 * 1024);
1930 if (bo->address == 0ull) {
1931 bo_free(bo);
1932 bo = NULL;
1936 _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
1940 return bo;
1944 iris_bo_mark_exported_locked(struct iris_bo *bo)
1946 struct iris_bufmgr *bufmgr = bo->bufmgr;
1949 assert(iris_bo_is_real(bo));
1952 if (!iris_bo_is_external(bo))
1953 _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
1955 if (!bo->real.exported) {
1960 bo->real.exported = true;
1961 bo->real.reusable = false;
1966 iris_bo_mark_exported(struct iris_bo *bo)
1968 struct iris_bufmgr *bufmgr = bo->bufmgr;
1971 assert(iris_bo_is_real(bo));
1973 if (bo->real.exported) {
1974 assert(!bo->real.reusable);
1979 iris_bo_mark_exported_locked(bo);
1984 iris_bo_export_dmabuf(struct iris_bo *bo, int *prime_fd)
1986 struct iris_bufmgr *bufmgr = bo->bufmgr;
1989 assert(iris_bo_is_real(bo));
1991 iris_bo_mark_exported(bo);
1993 if (drmPrimeHandleToFD(bufmgr->fd, bo->gem_handle,
2001 iris_bo_export_gem_handle(struct iris_bo *bo)
2004 assert(iris_bo_is_real(bo));
2006 iris_bo_mark_exported(bo);
2008 return bo->gem_handle;
2012 iris_bo_flink(struct iris_bo *bo, uint32_t *name)
2014 struct iris_bufmgr *bufmgr = bo->bufmgr;
2017 assert(iris_bo_is_real(bo));
2019 if (!bo->real.global_name) {
2020 struct drm_gem_flink flink = { .handle = bo->gem_handle };
2026 if (!bo->real.global_name) {
2027 iris_bo_mark_exported_locked(bo);
2028 bo->real.global_name = flink.name;
2029 _mesa_hash_table_insert(bufmgr->name_table, &bo->real.global_name, bo);
2034 *name = bo->real.global_name;
2039 iris_bo_export_gem_handle_for_device(struct iris_bo *bo, int drm_fd,
2043 assert(iris_bo_is_real(bo));
2049 struct iris_bufmgr *bufmgr = bo->bufmgr;
2055 *out_handle = iris_bo_export_gem_handle(bo);
2066 int err = iris_bo_export_dmabuf(bo, &dmabuf_fd);
2082 list_for_each_entry(struct bo_export, iter, &bo->real.exports, link) {
2095 list_addtail(&export->link, &bo->real.exports);
2286 struct iris_bo *bo = alloc_fresh_bo(bufmgr, size, 0);
2287 if (!bo) {
2294 bo->address = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 64 * 1024);
2295 if (bo->address == 0ull) {
2297 bo_free(bo);
2304 bo->name = "aux-map";
2305 p_atomic_set(&bo->refcount, 1);
2306 bo->index = -1;
2307 bo->real.kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED |
2309 bo->real.mmap_mode =
2310 bo->real.heap != IRIS_HEAP_SYSTEM_MEMORY ? IRIS_MMAP_WC : IRIS_MMAP_WB;
2312 buf->driver_bo = bo;
2313 buf->gpu = bo->address;
2314 buf->gpu_end = buf->gpu + bo->size;
2315 buf->map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);