1/* 2 * Copyright (C) 2017-2019 Lima Project 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24#include <stdlib.h> 25#include <sys/types.h> 26#include <unistd.h> 27#include <fcntl.h> 28 29#include "xf86drm.h" 30#include "drm-uapi/lima_drm.h" 31 32#include "util/u_hash_table.h" 33#include "util/u_math.h" 34#include "util/os_time.h" 35#include "os/os_mman.h" 36 37#include "frontend/drm_driver.h" 38 39#include "lima_screen.h" 40#include "lima_bo.h" 41#include "lima_util.h" 42 43bool lima_bo_table_init(struct lima_screen *screen) 44{ 45 screen->bo_handles = util_hash_table_create_ptr_keys(); 46 if (!screen->bo_handles) 47 return false; 48 49 screen->bo_flink_names = util_hash_table_create_ptr_keys(); 50 if (!screen->bo_flink_names) 51 goto err_out0; 52 53 mtx_init(&screen->bo_table_lock, mtx_plain); 54 return true; 55 56err_out0: 57 _mesa_hash_table_destroy(screen->bo_handles, NULL); 58 return false; 59} 60 61bool lima_bo_cache_init(struct lima_screen *screen) 62{ 63 mtx_init(&screen->bo_cache_lock, mtx_plain); 64 list_inithead(&screen->bo_cache_time); 65 for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) 66 list_inithead(&screen->bo_cache_buckets[i]); 67 68 return true; 69} 70 71void lima_bo_table_fini(struct lima_screen *screen) 72{ 73 mtx_destroy(&screen->bo_table_lock); 74 _mesa_hash_table_destroy(screen->bo_handles, NULL); 75 _mesa_hash_table_destroy(screen->bo_flink_names, NULL); 76} 77 78static void 79lima_bo_cache_remove(struct lima_bo *bo) 80{ 81 list_del(&bo->size_list); 82 list_del(&bo->time_list); 83} 84 85static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle) 86{ 87 struct drm_gem_close args = { 88 .handle = handle, 89 }; 90 91 drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args); 92} 93 94static void 95lima_bo_free(struct lima_bo *bo) 96{ 97 struct lima_screen *screen = bo->screen; 98 99 if (lima_debug & LIMA_DEBUG_BO_CACHE) 100 fprintf(stderr, "%s: %p (size=%d)\n", __func__, 101 bo, bo->size); 102 103 mtx_lock(&screen->bo_table_lock); 104 _mesa_hash_table_remove_key(screen->bo_handles, 105 (void *)(uintptr_t)bo->handle); 106 if (bo->flink_name) 107 _mesa_hash_table_remove_key(screen->bo_flink_names, 108 (void *)(uintptr_t)bo->flink_name); 109 mtx_unlock(&screen->bo_table_lock); 110 111 if (bo->map) 112 lima_bo_unmap(bo); 113 114 lima_close_kms_handle(screen, bo->handle); 115 free(bo); 116} 117 118void lima_bo_cache_fini(struct lima_screen *screen) 119{ 120 mtx_destroy(&screen->bo_cache_lock); 121 122 list_for_each_entry_safe(struct lima_bo, entry, 123 &screen->bo_cache_time, time_list) { 124 lima_bo_cache_remove(entry); 125 lima_bo_free(entry); 126 } 127} 128 129static bool lima_bo_get_info(struct lima_bo *bo) 130{ 131 struct drm_lima_gem_info req = { 132 .handle = bo->handle, 133 }; 134 135 if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req)) 136 return false; 137 138 bo->offset = req.offset; 139 bo->va = req.va; 140 return true; 141} 142 143static unsigned 144lima_bucket_index(unsigned size) 145{ 146 /* Round down to POT to compute a bucket index */ 147 148 unsigned bucket_index = util_logbase2(size); 149 150 /* Clamp the bucket index; all huge allocations will be 151 * sorted into the largest bucket */ 152 bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, 153 MAX_BO_CACHE_BUCKET); 154 155 /* Reindex from 0 */ 156 return (bucket_index - MIN_BO_CACHE_BUCKET); 157} 158 159static struct list_head * 160lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size) 161{ 162 return &screen->bo_cache_buckets[lima_bucket_index(size)]; 163} 164 165static void 166lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time) 167{ 168 unsigned cnt = 0; 169 list_for_each_entry_safe(struct lima_bo, entry, 170 &screen->bo_cache_time, time_list) { 171 /* Free BOs that are sitting idle for longer than 5 seconds */ 172 if (time - entry->free_time > 6) { 173 lima_bo_cache_remove(entry); 174 lima_bo_free(entry); 175 cnt++; 176 } else 177 break; 178 } 179 if ((lima_debug & LIMA_DEBUG_BO_CACHE) && cnt) 180 fprintf(stderr, "%s: freed %d stale BOs\n", __func__, cnt); 181} 182 183static void 184lima_bo_cache_print_stats(struct lima_screen *screen) 185{ 186 fprintf(stderr, "===============\n"); 187 fprintf(stderr, "BO cache stats:\n"); 188 unsigned total_size = 0; 189 for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) { 190 struct list_head *bucket = &screen->bo_cache_buckets[i]; 191 unsigned bucket_size = 0; 192 list_for_each_entry(struct lima_bo, entry, bucket, size_list) { 193 bucket_size += entry->size; 194 total_size += entry->size; 195 } 196 fprintf(stderr, "Bucket #%d, BOs: %d, size: %u\n", i, 197 list_length(bucket), 198 bucket_size); 199 } 200 fprintf(stderr, "Total size: %u\n", total_size); 201} 202 203static bool 204lima_bo_cache_put(struct lima_bo *bo) 205{ 206 if (!bo->cacheable) 207 return false; 208 209 struct lima_screen *screen = bo->screen; 210 211 mtx_lock(&screen->bo_cache_lock); 212 struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size); 213 214 if (!bucket) { 215 mtx_unlock(&screen->bo_cache_lock); 216 return false; 217 } 218 219 struct timespec time; 220 clock_gettime(CLOCK_MONOTONIC, &time); 221 bo->free_time = time.tv_sec; 222 list_addtail(&bo->size_list, bucket); 223 list_addtail(&bo->time_list, &screen->bo_cache_time); 224 lima_bo_cache_free_stale_bos(screen, time.tv_sec); 225 if (lima_debug & LIMA_DEBUG_BO_CACHE) { 226 fprintf(stderr, "%s: put BO: %p (size=%d)\n", __func__, bo, bo->size); 227 lima_bo_cache_print_stats(screen); 228 } 229 mtx_unlock(&screen->bo_cache_lock); 230 231 return true; 232} 233 234static struct lima_bo * 235lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags) 236{ 237 /* we won't cache heap buffer */ 238 if (flags & LIMA_BO_FLAG_HEAP) 239 return NULL; 240 241 struct lima_bo *bo = NULL; 242 mtx_lock(&screen->bo_cache_lock); 243 struct list_head *bucket = lima_bo_cache_get_bucket(screen, size); 244 245 if (!bucket) { 246 mtx_unlock(&screen->bo_cache_lock); 247 return false; 248 } 249 250 list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) { 251 if (entry->size >= size) { 252 /* Check if BO is idle. If it's not it's better to allocate new one */ 253 if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0)) { 254 if (lima_debug & LIMA_DEBUG_BO_CACHE) { 255 fprintf(stderr, "%s: found BO %p but it's busy\n", __func__, 256 entry); 257 } 258 break; 259 } 260 261 lima_bo_cache_remove(entry); 262 p_atomic_set(&entry->refcnt, 1); 263 entry->flags = flags; 264 bo = entry; 265 if (lima_debug & LIMA_DEBUG_BO_CACHE) { 266 fprintf(stderr, "%s: got BO: %p (size=%d), requested size %d\n", 267 __func__, bo, bo->size, size); 268 lima_bo_cache_print_stats(screen); 269 } 270 break; 271 } 272 } 273 274 mtx_unlock(&screen->bo_cache_lock); 275 276 return bo; 277} 278 279struct lima_bo *lima_bo_create(struct lima_screen *screen, 280 uint32_t size, uint32_t flags) 281{ 282 struct lima_bo *bo; 283 284 size = align(size, LIMA_PAGE_SIZE); 285 286 /* Try to get bo from cache first */ 287 bo = lima_bo_cache_get(screen, size, flags); 288 if (bo) 289 return bo; 290 291 struct drm_lima_gem_create req = { 292 .size = size, 293 .flags = flags, 294 }; 295 296 if (!(bo = calloc(1, sizeof(*bo)))) 297 return NULL; 298 299 list_inithead(&bo->time_list); 300 list_inithead(&bo->size_list); 301 302 if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req)) 303 goto err_out0; 304 305 bo->screen = screen; 306 bo->size = req.size; 307 bo->flags = req.flags; 308 bo->handle = req.handle; 309 bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE || 310 flags & LIMA_BO_FLAG_HEAP); 311 p_atomic_set(&bo->refcnt, 1); 312 313 if (!lima_bo_get_info(bo)) 314 goto err_out1; 315 316 if (lima_debug & LIMA_DEBUG_BO_CACHE) 317 fprintf(stderr, "%s: %p (size=%d)\n", __func__, 318 bo, bo->size); 319 320 return bo; 321 322err_out1: 323 lima_close_kms_handle(screen, bo->handle); 324err_out0: 325 free(bo); 326 return NULL; 327} 328 329void lima_bo_unreference(struct lima_bo *bo) 330{ 331 if (!p_atomic_dec_zero(&bo->refcnt)) 332 return; 333 334 /* Try to put it into cache */ 335 if (lima_bo_cache_put(bo)) 336 return; 337 338 lima_bo_free(bo); 339} 340 341void *lima_bo_map(struct lima_bo *bo) 342{ 343 if (!bo->map) { 344 bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE, 345 MAP_SHARED, bo->screen->fd, bo->offset); 346 if (bo->map == MAP_FAILED) 347 bo->map = NULL; 348 } 349 350 return bo->map; 351} 352 353void lima_bo_unmap(struct lima_bo *bo) 354{ 355 if (bo->map) { 356 os_munmap(bo->map, bo->size); 357 bo->map = NULL; 358 } 359} 360 361bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle) 362{ 363 struct lima_screen *screen = bo->screen; 364 365 /* Don't cache exported BOs */ 366 bo->cacheable = false; 367 368 switch (handle->type) { 369 case WINSYS_HANDLE_TYPE_SHARED: 370 if (!bo->flink_name) { 371 struct drm_gem_flink flink = { 372 .handle = bo->handle, 373 .name = 0, 374 }; 375 if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink)) 376 return false; 377 378 bo->flink_name = flink.name; 379 380 mtx_lock(&screen->bo_table_lock); 381 _mesa_hash_table_insert(screen->bo_flink_names, 382 (void *)(uintptr_t)bo->flink_name, bo); 383 mtx_unlock(&screen->bo_table_lock); 384 } 385 handle->handle = bo->flink_name; 386 return true; 387 388 case WINSYS_HANDLE_TYPE_KMS: 389 mtx_lock(&screen->bo_table_lock); 390 _mesa_hash_table_insert(screen->bo_handles, 391 (void *)(uintptr_t)bo->handle, bo); 392 mtx_unlock(&screen->bo_table_lock); 393 394 handle->handle = bo->handle; 395 return true; 396 397 case WINSYS_HANDLE_TYPE_FD: 398 if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC, 399 (int*)&handle->handle)) 400 return false; 401 402 mtx_lock(&screen->bo_table_lock); 403 _mesa_hash_table_insert(screen->bo_handles, 404 (void *)(uintptr_t)bo->handle, bo); 405 mtx_unlock(&screen->bo_table_lock); 406 return true; 407 408 default: 409 return false; 410 } 411} 412 413struct lima_bo *lima_bo_import(struct lima_screen *screen, 414 struct winsys_handle *handle) 415{ 416 struct lima_bo *bo = NULL; 417 struct drm_gem_open req = {0}; 418 uint32_t dma_buf_size = 0; 419 unsigned h = handle->handle; 420 421 mtx_lock(&screen->bo_table_lock); 422 423 /* Convert a DMA buf handle to a KMS handle now. */ 424 if (handle->type == WINSYS_HANDLE_TYPE_FD) { 425 uint32_t prime_handle; 426 off_t size; 427 428 /* Get a KMS handle. */ 429 if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) { 430 mtx_unlock(&screen->bo_table_lock); 431 return NULL; 432 } 433 434 /* Query the buffer size. */ 435 size = lseek(h, 0, SEEK_END); 436 if (size == (off_t)-1) { 437 mtx_unlock(&screen->bo_table_lock); 438 lima_close_kms_handle(screen, prime_handle); 439 return NULL; 440 } 441 lseek(h, 0, SEEK_SET); 442 443 dma_buf_size = size; 444 h = prime_handle; 445 } 446 447 switch (handle->type) { 448 case WINSYS_HANDLE_TYPE_SHARED: 449 bo = util_hash_table_get(screen->bo_flink_names, 450 (void *)(uintptr_t)h); 451 break; 452 case WINSYS_HANDLE_TYPE_KMS: 453 case WINSYS_HANDLE_TYPE_FD: 454 bo = util_hash_table_get(screen->bo_handles, 455 (void *)(uintptr_t)h); 456 break; 457 default: 458 mtx_unlock(&screen->bo_table_lock); 459 return NULL; 460 } 461 462 if (bo) { 463 p_atomic_inc(&bo->refcnt); 464 /* Don't cache imported BOs */ 465 bo->cacheable = false; 466 mtx_unlock(&screen->bo_table_lock); 467 return bo; 468 } 469 470 if (!(bo = calloc(1, sizeof(*bo)))) { 471 mtx_unlock(&screen->bo_table_lock); 472 if (handle->type == WINSYS_HANDLE_TYPE_FD) 473 lima_close_kms_handle(screen, h); 474 return NULL; 475 } 476 477 /* Don't cache imported BOs */ 478 bo->cacheable = false; 479 list_inithead(&bo->time_list); 480 list_inithead(&bo->size_list); 481 bo->screen = screen; 482 p_atomic_set(&bo->refcnt, 1); 483 484 switch (handle->type) { 485 case WINSYS_HANDLE_TYPE_SHARED: 486 req.name = h; 487 if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) { 488 mtx_unlock(&screen->bo_table_lock); 489 free(bo); 490 return NULL; 491 } 492 bo->handle = req.handle; 493 bo->flink_name = h; 494 bo->size = req.size; 495 break; 496 case WINSYS_HANDLE_TYPE_FD: 497 bo->handle = h; 498 bo->size = dma_buf_size; 499 break; 500 default: 501 /* not possible */ 502 assert(0); 503 } 504 505 if (lima_bo_get_info(bo)) { 506 if (handle->type == WINSYS_HANDLE_TYPE_SHARED) 507 _mesa_hash_table_insert(screen->bo_flink_names, 508 (void *)(uintptr_t)bo->flink_name, bo); 509 _mesa_hash_table_insert(screen->bo_handles, 510 (void*)(uintptr_t)bo->handle, bo); 511 } 512 else { 513 lima_close_kms_handle(screen, bo->handle); 514 free(bo); 515 bo = NULL; 516 } 517 518 mtx_unlock(&screen->bo_table_lock); 519 520 return bo; 521} 522 523bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns) 524{ 525 int64_t abs_timeout; 526 527 if (timeout_ns == 0) 528 abs_timeout = 0; 529 else 530 abs_timeout = os_time_get_absolute_timeout(timeout_ns); 531 532 if (abs_timeout == OS_TIMEOUT_INFINITE) 533 abs_timeout = INT64_MAX; 534 535 struct drm_lima_gem_wait req = { 536 .handle = bo->handle, 537 .op = op, 538 .timeout_ns = abs_timeout, 539 }; 540 541 return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0; 542} 543