1/*
2 * Copyright © 2014-2017 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include <errno.h>
25#include <err.h>
26#include <sys/mman.h>
27#include <fcntl.h>
28#include <xf86drm.h>
29#include <xf86drmMode.h>
30
31#include "util/u_hash_table.h"
32#include "util/u_memory.h"
33#include "util/ralloc.h"
34
35#include "v3d_context.h"
36#include "v3d_screen.h"
37
38static bool dump_stats = false;
39
40static void
41v3d_bo_cache_free_all(struct v3d_bo_cache *cache);
42
43static void
44v3d_bo_dump_stats(struct v3d_screen *screen)
45{
46        struct v3d_bo_cache *cache = &screen->bo_cache;
47
48        uint32_t cache_count = 0;
49        uint32_t cache_size = 0;
50        list_for_each_entry(struct v3d_bo, bo, &cache->time_list, time_list) {
51                cache_count++;
52                cache_size += bo->size;
53        }
54
55        fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
56        fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
57        fprintf(stderr, "  BOs cached:      %d\n", cache_count);
58        fprintf(stderr, "  BOs cached size: %dkb\n", cache_size / 1024);
59
60        if (!list_is_empty(&cache->time_list)) {
61                struct v3d_bo *first = list_first_entry(&cache->time_list,
62                                                        struct v3d_bo,
63                                                        time_list);
64                struct v3d_bo *last = list_last_entry(&cache->time_list,
65                                                      struct v3d_bo,
66                                                      time_list);
67
68                fprintf(stderr, "  oldest cache time: %ld\n",
69                        (long)first->free_time);
70                fprintf(stderr, "  newest cache time: %ld\n",
71                        (long)last->free_time);
72
73                struct timespec time;
74                clock_gettime(CLOCK_MONOTONIC, &time);
75                fprintf(stderr, "  now:               %jd\n",
76                        (intmax_t)time.tv_sec);
77        }
78}
79
80static void
81v3d_bo_remove_from_cache(struct v3d_bo_cache *cache, struct v3d_bo *bo)
82{
83        list_del(&bo->time_list);
84        list_del(&bo->size_list);
85}
86
87static struct v3d_bo *
88v3d_bo_from_cache(struct v3d_screen *screen, uint32_t size, const char *name)
89{
90        struct v3d_bo_cache *cache = &screen->bo_cache;
91        uint32_t page_index = size / 4096 - 1;
92
93        if (cache->size_list_size <= page_index)
94                return NULL;
95
96        struct v3d_bo *bo = NULL;
97        mtx_lock(&cache->lock);
98        if (!list_is_empty(&cache->size_list[page_index])) {
99                bo = list_first_entry(&cache->size_list[page_index],
100                                      struct v3d_bo, size_list);
101
102                /* Check that the BO has gone idle.  If not, then we want to
103                 * allocate something new instead, since we assume that the
104                 * user will proceed to CPU map it and fill it with stuff.
105                 */
106                if (!v3d_bo_wait(bo, 0, NULL)) {
107                        mtx_unlock(&cache->lock);
108                        return NULL;
109                }
110
111                pipe_reference_init(&bo->reference, 1);
112                v3d_bo_remove_from_cache(cache, bo);
113
114                bo->name = name;
115        }
116        mtx_unlock(&cache->lock);
117        return bo;
118}
119
120struct v3d_bo *
121v3d_bo_alloc(struct v3d_screen *screen, uint32_t size, const char *name)
122{
123        struct v3d_bo *bo;
124        int ret;
125
126        /* The CLIF dumping requires that there is no whitespace in the name.
127         */
128        assert(!strchr(name, ' '));
129
130        size = align(size, 4096);
131
132        bo = v3d_bo_from_cache(screen, size, name);
133        if (bo) {
134                if (dump_stats) {
135                        fprintf(stderr, "Allocated %s %dkb from cache:\n",
136                                name, size / 1024);
137                        v3d_bo_dump_stats(screen);
138                }
139                return bo;
140        }
141
142        bo = CALLOC_STRUCT(v3d_bo);
143        if (!bo)
144                return NULL;
145
146        pipe_reference_init(&bo->reference, 1);
147        bo->screen = screen;
148        bo->size = size;
149        bo->name = name;
150        bo->private = true;
151
152 retry:
153        ;
154
155        bool cleared_and_retried = false;
156        struct drm_v3d_create_bo create = {
157                .size = size
158        };
159
160        ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_CREATE_BO, &create);
161        bo->handle = create.handle;
162        bo->offset = create.offset;
163
164        if (ret != 0) {
165                if (!list_is_empty(&screen->bo_cache.time_list) &&
166                    !cleared_and_retried) {
167                        cleared_and_retried = true;
168                        v3d_bo_cache_free_all(&screen->bo_cache);
169                        goto retry;
170                }
171
172                free(bo);
173                return NULL;
174        }
175
176        screen->bo_count++;
177        screen->bo_size += bo->size;
178        if (dump_stats) {
179                fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
180                v3d_bo_dump_stats(screen);
181        }
182
183        return bo;
184}
185
186void
187v3d_bo_last_unreference(struct v3d_bo *bo)
188{
189        struct v3d_screen *screen = bo->screen;
190
191        struct timespec time;
192        clock_gettime(CLOCK_MONOTONIC, &time);
193        mtx_lock(&screen->bo_cache.lock);
194        v3d_bo_last_unreference_locked_timed(bo, time.tv_sec);
195        mtx_unlock(&screen->bo_cache.lock);
196}
197
198static void
199v3d_bo_free(struct v3d_bo *bo)
200{
201        struct v3d_screen *screen = bo->screen;
202
203        if (bo->map) {
204                if (using_v3d_simulator && bo->name &&
205                    strcmp(bo->name, "winsys") == 0) {
206                        free(bo->map);
207                } else {
208                        munmap(bo->map, bo->size);
209                        VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
210                }
211        }
212
213        struct drm_gem_close c;
214        memset(&c, 0, sizeof(c));
215        c.handle = bo->handle;
216        int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
217        if (ret != 0)
218                fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
219
220        screen->bo_count--;
221        screen->bo_size -= bo->size;
222
223        if (dump_stats) {
224                fprintf(stderr, "Freed %s%s%dkb:\n",
225                        bo->name ? bo->name : "",
226                        bo->name ? " " : "",
227                        bo->size / 1024);
228                v3d_bo_dump_stats(screen);
229        }
230
231        free(bo);
232}
233
234static void
235free_stale_bos(struct v3d_screen *screen, time_t time)
236{
237        struct v3d_bo_cache *cache = &screen->bo_cache;
238        bool freed_any = false;
239
240        list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
241                                 time_list) {
242                /* If it's more than a second old, free it. */
243                if (time - bo->free_time > 2) {
244                        if (dump_stats && !freed_any) {
245                                fprintf(stderr, "Freeing stale BOs:\n");
246                                v3d_bo_dump_stats(screen);
247                                freed_any = true;
248                        }
249                        v3d_bo_remove_from_cache(cache, bo);
250                        v3d_bo_free(bo);
251                } else {
252                        break;
253                }
254        }
255
256        if (dump_stats && freed_any) {
257                fprintf(stderr, "Freed stale BOs:\n");
258                v3d_bo_dump_stats(screen);
259        }
260}
261
262static void
263v3d_bo_cache_free_all(struct v3d_bo_cache *cache)
264{
265        mtx_lock(&cache->lock);
266        list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
267                                 time_list) {
268                v3d_bo_remove_from_cache(cache, bo);
269                v3d_bo_free(bo);
270        }
271        mtx_unlock(&cache->lock);
272}
273
274void
275v3d_bo_last_unreference_locked_timed(struct v3d_bo *bo, time_t time)
276{
277        struct v3d_screen *screen = bo->screen;
278        struct v3d_bo_cache *cache = &screen->bo_cache;
279        uint32_t page_index = bo->size / 4096 - 1;
280
281        if (!bo->private) {
282                v3d_bo_free(bo);
283                return;
284        }
285
286        if (cache->size_list_size <= page_index) {
287                struct list_head *new_list =
288                        ralloc_array(screen, struct list_head, page_index + 1);
289
290                /* Move old list contents over (since the array has moved, and
291                 * therefore the pointers to the list heads have to change).
292                 */
293                for (int i = 0; i < cache->size_list_size; i++) {
294                        struct list_head *old_head = &cache->size_list[i];
295                        if (list_is_empty(old_head))
296                                list_inithead(&new_list[i]);
297                        else {
298                                new_list[i].next = old_head->next;
299                                new_list[i].prev = old_head->prev;
300                                new_list[i].next->prev = &new_list[i];
301                                new_list[i].prev->next = &new_list[i];
302                        }
303                }
304                for (int i = cache->size_list_size; i < page_index + 1; i++)
305                        list_inithead(&new_list[i]);
306
307                cache->size_list = new_list;
308                cache->size_list_size = page_index + 1;
309        }
310
311        bo->free_time = time;
312        list_addtail(&bo->size_list, &cache->size_list[page_index]);
313        list_addtail(&bo->time_list, &cache->time_list);
314        if (dump_stats) {
315                fprintf(stderr, "Freed %s %dkb to cache:\n",
316                        bo->name, bo->size / 1024);
317                v3d_bo_dump_stats(screen);
318        }
319        bo->name = NULL;
320
321        free_stale_bos(screen, time);
322}
323
324static struct v3d_bo *
325v3d_bo_open_handle(struct v3d_screen *screen,
326                   uint32_t handle, uint32_t size)
327{
328        struct v3d_bo *bo;
329
330        assert(size);
331
332        mtx_lock(&screen->bo_handles_mutex);
333
334        bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
335        if (bo) {
336                pipe_reference(NULL, &bo->reference);
337                goto done;
338        }
339
340        bo = CALLOC_STRUCT(v3d_bo);
341        pipe_reference_init(&bo->reference, 1);
342        bo->screen = screen;
343        bo->handle = handle;
344        bo->size = size;
345        bo->name = "winsys";
346        bo->private = false;
347
348#ifdef USE_V3D_SIMULATOR
349        v3d_simulator_open_from_handle(screen->fd, bo->handle, bo->size);
350        bo->map = malloc(bo->size);
351#endif
352
353        struct drm_v3d_get_bo_offset get = {
354                .handle = handle,
355        };
356        int ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_GET_BO_OFFSET, &get);
357        if (ret) {
358                fprintf(stderr, "Failed to get BO offset: %s\n",
359                        strerror(errno));
360                free(bo->map);
361                free(bo);
362                bo = NULL;
363                goto done;
364        }
365        bo->offset = get.offset;
366        assert(bo->offset != 0);
367
368        _mesa_hash_table_insert(screen->bo_handles, (void *)(uintptr_t)handle, bo);
369
370        screen->bo_count++;
371        screen->bo_size += bo->size;
372
373done:
374        mtx_unlock(&screen->bo_handles_mutex);
375        return bo;
376}
377
378struct v3d_bo *
379v3d_bo_open_name(struct v3d_screen *screen, uint32_t name)
380{
381        struct drm_gem_open o = {
382                .name = name
383        };
384        int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
385        if (ret) {
386                fprintf(stderr, "Failed to open bo %d: %s\n",
387                        name, strerror(errno));
388                return NULL;
389        }
390
391        return v3d_bo_open_handle(screen, o.handle, o.size);
392}
393
394struct v3d_bo *
395v3d_bo_open_dmabuf(struct v3d_screen *screen, int fd)
396{
397        uint32_t handle;
398        int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
399        int size;
400        if (ret) {
401                fprintf(stderr, "Failed to get v3d handle for dmabuf %d\n", fd);
402                return NULL;
403        }
404
405        /* Determine the size of the bo we were handed. */
406        size = lseek(fd, 0, SEEK_END);
407        if (size == -1) {
408                fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
409                return NULL;
410        }
411
412        return v3d_bo_open_handle(screen, handle, size);
413}
414
415int
416v3d_bo_get_dmabuf(struct v3d_bo *bo)
417{
418        int fd;
419        int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
420                                     O_CLOEXEC, &fd);
421        if (ret != 0) {
422                fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
423                        bo->handle);
424                return -1;
425        }
426
427        mtx_lock(&bo->screen->bo_handles_mutex);
428        bo->private = false;
429        _mesa_hash_table_insert(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
430        mtx_unlock(&bo->screen->bo_handles_mutex);
431
432        return fd;
433}
434
435bool
436v3d_bo_flink(struct v3d_bo *bo, uint32_t *name)
437{
438        struct drm_gem_flink flink = {
439                .handle = bo->handle,
440        };
441        int ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
442        if (ret) {
443                fprintf(stderr, "Failed to flink bo %d: %s\n",
444                        bo->handle, strerror(errno));
445                free(bo);
446                return false;
447        }
448
449        bo->private = false;
450        *name = flink.name;
451
452        return true;
453}
454
455static int v3d_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
456{
457        struct drm_v3d_wait_bo wait = {
458                .handle = handle,
459                .timeout_ns = timeout_ns,
460        };
461        int ret = v3d_ioctl(fd, DRM_IOCTL_V3D_WAIT_BO, &wait);
462        if (ret == -1)
463                return -errno;
464        else
465                return 0;
466
467}
468
469bool
470v3d_bo_wait(struct v3d_bo *bo, uint64_t timeout_ns, const char *reason)
471{
472        struct v3d_screen *screen = bo->screen;
473
474        if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
475                if (v3d_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
476                        fprintf(stderr, "Blocking on %s BO for %s\n",
477                                bo->name, reason);
478                }
479        }
480
481        int ret = v3d_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
482        if (ret) {
483                if (ret != -ETIME) {
484                        fprintf(stderr, "wait failed: %d\n", ret);
485                        abort();
486                }
487
488                return false;
489        }
490
491        return true;
492}
493
494void *
495v3d_bo_map_unsynchronized(struct v3d_bo *bo)
496{
497        uint64_t offset;
498        int ret;
499
500        if (bo->map)
501                return bo->map;
502
503        struct drm_v3d_mmap_bo map;
504        memset(&map, 0, sizeof(map));
505        map.handle = bo->handle;
506        ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_V3D_MMAP_BO, &map);
507        offset = map.offset;
508        if (ret != 0) {
509                fprintf(stderr, "map ioctl failure\n");
510                abort();
511        }
512
513        bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
514                       bo->screen->fd, offset);
515        if (bo->map == MAP_FAILED) {
516                fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
517                        bo->handle, (long long)offset, bo->size);
518                abort();
519        }
520        VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
521
522        return bo->map;
523}
524
525void *
526v3d_bo_map(struct v3d_bo *bo)
527{
528        void *map = v3d_bo_map_unsynchronized(bo);
529
530        bool ok = v3d_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
531        if (!ok) {
532                fprintf(stderr, "BO wait for map failed\n");
533                abort();
534        }
535
536        return map;
537}
538
539void
540v3d_bufmgr_destroy(struct pipe_screen *pscreen)
541{
542        struct v3d_screen *screen = v3d_screen(pscreen);
543        struct v3d_bo_cache *cache = &screen->bo_cache;
544
545        v3d_bo_cache_free_all(cache);
546
547        if (dump_stats) {
548                fprintf(stderr, "BO stats after screen destroy:\n");
549                v3d_bo_dump_stats(screen);
550        }
551}
552