1/*
2 * Copyright (C) 2017-2019 Lima Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <stdlib.h>
25#include <sys/types.h>
26#include <unistd.h>
27#include <fcntl.h>
28
29#include "xf86drm.h"
30#include "drm-uapi/lima_drm.h"
31
32#include "util/u_hash_table.h"
33#include "util/u_math.h"
34#include "util/os_time.h"
35#include "os/os_mman.h"
36
37#include "frontend/drm_driver.h"
38
39#include "lima_screen.h"
40#include "lima_bo.h"
41#include "lima_util.h"
42
43bool lima_bo_table_init(struct lima_screen *screen)
44{
45   screen->bo_handles = util_hash_table_create_ptr_keys();
46   if (!screen->bo_handles)
47      return false;
48
49   screen->bo_flink_names = util_hash_table_create_ptr_keys();
50   if (!screen->bo_flink_names)
51      goto err_out0;
52
53   mtx_init(&screen->bo_table_lock, mtx_plain);
54   return true;
55
56err_out0:
57   _mesa_hash_table_destroy(screen->bo_handles, NULL);
58   return false;
59}
60
61bool lima_bo_cache_init(struct lima_screen *screen)
62{
63   mtx_init(&screen->bo_cache_lock, mtx_plain);
64   list_inithead(&screen->bo_cache_time);
65   for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++)
66      list_inithead(&screen->bo_cache_buckets[i]);
67
68   return true;
69}
70
71void lima_bo_table_fini(struct lima_screen *screen)
72{
73   mtx_destroy(&screen->bo_table_lock);
74   _mesa_hash_table_destroy(screen->bo_handles, NULL);
75   _mesa_hash_table_destroy(screen->bo_flink_names, NULL);
76}
77
78static void
79lima_bo_cache_remove(struct lima_bo *bo)
80{
81   list_del(&bo->size_list);
82   list_del(&bo->time_list);
83}
84
85static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle)
86{
87   struct drm_gem_close args = {
88      .handle = handle,
89   };
90
91   drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args);
92}
93
94static void
95lima_bo_free(struct lima_bo *bo)
96{
97   struct lima_screen *screen = bo->screen;
98
99   if (lima_debug & LIMA_DEBUG_BO_CACHE)
100      fprintf(stderr, "%s: %p (size=%d)\n", __func__,
101              bo, bo->size);
102
103   mtx_lock(&screen->bo_table_lock);
104   _mesa_hash_table_remove_key(screen->bo_handles,
105                          (void *)(uintptr_t)bo->handle);
106   if (bo->flink_name)
107      _mesa_hash_table_remove_key(screen->bo_flink_names,
108                             (void *)(uintptr_t)bo->flink_name);
109   mtx_unlock(&screen->bo_table_lock);
110
111   if (bo->map)
112      lima_bo_unmap(bo);
113
114   lima_close_kms_handle(screen, bo->handle);
115   free(bo);
116}
117
118void lima_bo_cache_fini(struct lima_screen *screen)
119{
120   mtx_destroy(&screen->bo_cache_lock);
121
122   list_for_each_entry_safe(struct lima_bo, entry,
123                            &screen->bo_cache_time, time_list) {
124      lima_bo_cache_remove(entry);
125      lima_bo_free(entry);
126   }
127}
128
129static bool lima_bo_get_info(struct lima_bo *bo)
130{
131   struct drm_lima_gem_info req = {
132      .handle = bo->handle,
133   };
134
135   if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req))
136      return false;
137
138   bo->offset = req.offset;
139   bo->va = req.va;
140   return true;
141}
142
143static unsigned
144lima_bucket_index(unsigned size)
145{
146   /* Round down to POT to compute a bucket index */
147
148   unsigned bucket_index = util_logbase2(size);
149
150   /* Clamp the bucket index; all huge allocations will be
151    * sorted into the largest bucket */
152   bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET,
153                        MAX_BO_CACHE_BUCKET);
154
155   /* Reindex from 0 */
156   return (bucket_index - MIN_BO_CACHE_BUCKET);
157}
158
159static struct list_head *
160lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size)
161{
162   return &screen->bo_cache_buckets[lima_bucket_index(size)];
163}
164
165static void
166lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time)
167{
168   unsigned cnt = 0;
169   list_for_each_entry_safe(struct lima_bo, entry,
170                            &screen->bo_cache_time, time_list) {
171      /* Free BOs that are sitting idle for longer than 5 seconds */
172      if (time - entry->free_time > 6) {
173         lima_bo_cache_remove(entry);
174         lima_bo_free(entry);
175         cnt++;
176      } else
177         break;
178   }
179   if ((lima_debug & LIMA_DEBUG_BO_CACHE) && cnt)
180      fprintf(stderr, "%s: freed %d stale BOs\n", __func__, cnt);
181}
182
183static void
184lima_bo_cache_print_stats(struct lima_screen *screen)
185{
186   fprintf(stderr, "===============\n");
187   fprintf(stderr, "BO cache stats:\n");
188   unsigned total_size = 0;
189   for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) {
190      struct list_head *bucket = &screen->bo_cache_buckets[i];
191      unsigned bucket_size = 0;
192      list_for_each_entry(struct lima_bo, entry, bucket, size_list) {
193         bucket_size += entry->size;
194         total_size += entry->size;
195      }
196      fprintf(stderr, "Bucket #%d, BOs: %d, size: %u\n", i,
197              list_length(bucket),
198              bucket_size);
199   }
200   fprintf(stderr, "Total size: %u\n", total_size);
201}
202
203static bool
204lima_bo_cache_put(struct lima_bo *bo)
205{
206   if (!bo->cacheable)
207      return false;
208
209   struct lima_screen *screen = bo->screen;
210
211   mtx_lock(&screen->bo_cache_lock);
212   struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size);
213
214   if (!bucket) {
215      mtx_unlock(&screen->bo_cache_lock);
216      return false;
217   }
218
219   struct timespec time;
220   clock_gettime(CLOCK_MONOTONIC, &time);
221   bo->free_time = time.tv_sec;
222   list_addtail(&bo->size_list, bucket);
223   list_addtail(&bo->time_list, &screen->bo_cache_time);
224   lima_bo_cache_free_stale_bos(screen, time.tv_sec);
225   if (lima_debug & LIMA_DEBUG_BO_CACHE) {
226      fprintf(stderr, "%s: put BO: %p (size=%d)\n", __func__, bo, bo->size);
227      lima_bo_cache_print_stats(screen);
228   }
229   mtx_unlock(&screen->bo_cache_lock);
230
231   return true;
232}
233
234static struct lima_bo *
235lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags)
236{
237   /* we won't cache heap buffer */
238   if (flags & LIMA_BO_FLAG_HEAP)
239      return NULL;
240
241   struct lima_bo *bo = NULL;
242   mtx_lock(&screen->bo_cache_lock);
243   struct list_head *bucket = lima_bo_cache_get_bucket(screen, size);
244
245   if (!bucket) {
246      mtx_unlock(&screen->bo_cache_lock);
247      return false;
248   }
249
250   list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) {
251      if (entry->size >= size) {
252         /* Check if BO is idle. If it's not it's better to allocate new one */
253         if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0)) {
254            if (lima_debug & LIMA_DEBUG_BO_CACHE) {
255               fprintf(stderr, "%s: found BO %p but it's busy\n", __func__,
256                       entry);
257            }
258            break;
259         }
260
261         lima_bo_cache_remove(entry);
262         p_atomic_set(&entry->refcnt, 1);
263         entry->flags = flags;
264         bo = entry;
265         if (lima_debug & LIMA_DEBUG_BO_CACHE) {
266            fprintf(stderr, "%s: got BO: %p (size=%d), requested size %d\n",
267                    __func__, bo, bo->size, size);
268            lima_bo_cache_print_stats(screen);
269         }
270         break;
271      }
272   }
273
274   mtx_unlock(&screen->bo_cache_lock);
275
276   return bo;
277}
278
279struct lima_bo *lima_bo_create(struct lima_screen *screen,
280                               uint32_t size, uint32_t flags)
281{
282   struct lima_bo *bo;
283
284   size = align(size, LIMA_PAGE_SIZE);
285
286   /* Try to get bo from cache first */
287   bo = lima_bo_cache_get(screen, size, flags);
288   if (bo)
289      return bo;
290
291   struct drm_lima_gem_create req = {
292      .size = size,
293      .flags = flags,
294   };
295
296   if (!(bo = calloc(1, sizeof(*bo))))
297      return NULL;
298
299   list_inithead(&bo->time_list);
300   list_inithead(&bo->size_list);
301
302   if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req))
303      goto err_out0;
304
305   bo->screen = screen;
306   bo->size = req.size;
307   bo->flags = req.flags;
308   bo->handle = req.handle;
309   bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE ||
310                     flags & LIMA_BO_FLAG_HEAP);
311   p_atomic_set(&bo->refcnt, 1);
312
313   if (!lima_bo_get_info(bo))
314      goto err_out1;
315
316   if (lima_debug & LIMA_DEBUG_BO_CACHE)
317      fprintf(stderr, "%s: %p (size=%d)\n", __func__,
318              bo, bo->size);
319
320   return bo;
321
322err_out1:
323   lima_close_kms_handle(screen, bo->handle);
324err_out0:
325   free(bo);
326   return NULL;
327}
328
329void lima_bo_unreference(struct lima_bo *bo)
330{
331   if (!p_atomic_dec_zero(&bo->refcnt))
332      return;
333
334   /* Try to put it into cache */
335   if (lima_bo_cache_put(bo))
336      return;
337
338   lima_bo_free(bo);
339}
340
341void *lima_bo_map(struct lima_bo *bo)
342{
343   if (!bo->map) {
344      bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
345                        MAP_SHARED, bo->screen->fd, bo->offset);
346      if (bo->map == MAP_FAILED)
347          bo->map = NULL;
348   }
349
350   return bo->map;
351}
352
353void lima_bo_unmap(struct lima_bo *bo)
354{
355   if (bo->map) {
356      os_munmap(bo->map, bo->size);
357      bo->map = NULL;
358   }
359}
360
361bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle)
362{
363   struct lima_screen *screen = bo->screen;
364
365   /* Don't cache exported BOs */
366   bo->cacheable = false;
367
368   switch (handle->type) {
369   case WINSYS_HANDLE_TYPE_SHARED:
370      if (!bo->flink_name) {
371         struct drm_gem_flink flink = {
372            .handle = bo->handle,
373            .name = 0,
374         };
375         if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink))
376            return false;
377
378         bo->flink_name = flink.name;
379
380         mtx_lock(&screen->bo_table_lock);
381         _mesa_hash_table_insert(screen->bo_flink_names,
382                             (void *)(uintptr_t)bo->flink_name, bo);
383         mtx_unlock(&screen->bo_table_lock);
384      }
385      handle->handle = bo->flink_name;
386      return true;
387
388   case WINSYS_HANDLE_TYPE_KMS:
389      mtx_lock(&screen->bo_table_lock);
390      _mesa_hash_table_insert(screen->bo_handles,
391                          (void *)(uintptr_t)bo->handle, bo);
392      mtx_unlock(&screen->bo_table_lock);
393
394      handle->handle = bo->handle;
395      return true;
396
397   case WINSYS_HANDLE_TYPE_FD:
398      if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC,
399                             (int*)&handle->handle))
400         return false;
401
402      mtx_lock(&screen->bo_table_lock);
403      _mesa_hash_table_insert(screen->bo_handles,
404                          (void *)(uintptr_t)bo->handle, bo);
405      mtx_unlock(&screen->bo_table_lock);
406      return true;
407
408   default:
409      return false;
410   }
411}
412
413struct lima_bo *lima_bo_import(struct lima_screen *screen,
414                               struct winsys_handle *handle)
415{
416   struct lima_bo *bo = NULL;
417   struct drm_gem_open req = {0};
418   uint32_t dma_buf_size = 0;
419   unsigned h = handle->handle;
420
421   mtx_lock(&screen->bo_table_lock);
422
423   /* Convert a DMA buf handle to a KMS handle now. */
424   if (handle->type == WINSYS_HANDLE_TYPE_FD) {
425      uint32_t prime_handle;
426      off_t size;
427
428      /* Get a KMS handle. */
429      if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) {
430         mtx_unlock(&screen->bo_table_lock);
431         return NULL;
432      }
433
434      /* Query the buffer size. */
435      size = lseek(h, 0, SEEK_END);
436      if (size == (off_t)-1) {
437         mtx_unlock(&screen->bo_table_lock);
438         lima_close_kms_handle(screen, prime_handle);
439         return NULL;
440      }
441      lseek(h, 0, SEEK_SET);
442
443      dma_buf_size = size;
444      h = prime_handle;
445   }
446
447   switch (handle->type) {
448   case WINSYS_HANDLE_TYPE_SHARED:
449      bo = util_hash_table_get(screen->bo_flink_names,
450                               (void *)(uintptr_t)h);
451      break;
452   case WINSYS_HANDLE_TYPE_KMS:
453   case WINSYS_HANDLE_TYPE_FD:
454      bo = util_hash_table_get(screen->bo_handles,
455                               (void *)(uintptr_t)h);
456      break;
457   default:
458      mtx_unlock(&screen->bo_table_lock);
459      return NULL;
460   }
461
462   if (bo) {
463      p_atomic_inc(&bo->refcnt);
464      /* Don't cache imported BOs */
465      bo->cacheable = false;
466      mtx_unlock(&screen->bo_table_lock);
467      return bo;
468   }
469
470   if (!(bo = calloc(1, sizeof(*bo)))) {
471      mtx_unlock(&screen->bo_table_lock);
472      if (handle->type == WINSYS_HANDLE_TYPE_FD)
473         lima_close_kms_handle(screen, h);
474      return NULL;
475   }
476
477   /* Don't cache imported BOs */
478   bo->cacheable = false;
479   list_inithead(&bo->time_list);
480   list_inithead(&bo->size_list);
481   bo->screen = screen;
482   p_atomic_set(&bo->refcnt, 1);
483
484   switch (handle->type) {
485   case WINSYS_HANDLE_TYPE_SHARED:
486      req.name = h;
487      if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) {
488         mtx_unlock(&screen->bo_table_lock);
489         free(bo);
490         return NULL;
491      }
492      bo->handle = req.handle;
493      bo->flink_name = h;
494      bo->size = req.size;
495      break;
496   case WINSYS_HANDLE_TYPE_FD:
497      bo->handle = h;
498      bo->size = dma_buf_size;
499      break;
500   default:
501      /* not possible */
502      assert(0);
503   }
504
505   if (lima_bo_get_info(bo)) {
506      if (handle->type == WINSYS_HANDLE_TYPE_SHARED)
507         _mesa_hash_table_insert(screen->bo_flink_names,
508                          (void *)(uintptr_t)bo->flink_name, bo);
509      _mesa_hash_table_insert(screen->bo_handles,
510                          (void*)(uintptr_t)bo->handle, bo);
511   }
512   else {
513      lima_close_kms_handle(screen, bo->handle);
514      free(bo);
515      bo = NULL;
516   }
517
518   mtx_unlock(&screen->bo_table_lock);
519
520   return bo;
521}
522
523bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns)
524{
525   int64_t abs_timeout;
526
527   if (timeout_ns == 0)
528      abs_timeout = 0;
529   else
530      abs_timeout = os_time_get_absolute_timeout(timeout_ns);
531
532   if (abs_timeout == OS_TIMEOUT_INFINITE)
533      abs_timeout = INT64_MAX;
534
535   struct drm_lima_gem_wait req = {
536      .handle = bo->handle,
537      .op = op,
538      .timeout_ns = abs_timeout,
539   };
540
541   return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0;
542}
543