1
2#include <inttypes.h>
3
4#include "util/simple_mtx.h"
5#include "util/u_inlines.h"
6#include "util/u_memory.h"
7#include "util/list.h"
8
9#include "nouveau_winsys.h"
10#include "nouveau_screen.h"
11#include "nouveau_mm.h"
12
13/* TODO: Higher orders can waste a lot of space for npot size buffers, should
14 * add an extra cache for such buffer objects.
15 *
16 * HACK: Max order == 21 to accommodate TF2's 1.5 MiB, frequently reallocated
17 * vertex buffer (VM flush (?) decreases performance dramatically).
18 */
19
20#define MM_MIN_ORDER 7 /* >= 6 to not violate ARB_map_buffer_alignment */
21#define MM_MAX_ORDER 21
22
23#define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
24
25#define MM_MIN_SIZE (1 << MM_MIN_ORDER)
26#define MM_MAX_SIZE (1 << MM_MAX_ORDER)
27
28struct mm_bucket {
29   struct list_head free;
30   struct list_head used;
31   struct list_head full;
32   int num_free;
33   simple_mtx_t lock;
34};
35
36struct nouveau_mman {
37   struct nouveau_device *dev;
38   struct mm_bucket bucket[MM_NUM_BUCKETS];
39   uint32_t domain;
40   union nouveau_bo_config config;
41   uint64_t allocated;
42};
43
44struct mm_slab {
45   struct list_head head;
46   struct nouveau_bo *bo;
47   struct nouveau_mman *cache;
48   int order;
49   int count;
50   int free;
51   uint32_t bits[0];
52};
53
54static int
55mm_slab_alloc(struct mm_slab *slab)
56{
57   int i, n, b;
58
59   if (slab->free == 0)
60      return -1;
61
62   for (i = 0; i < (slab->count + 31) / 32; ++i) {
63      b = ffs(slab->bits[i]) - 1;
64      if (b >= 0) {
65         n = i * 32 + b;
66         assert(n < slab->count);
67         slab->free--;
68         slab->bits[i] &= ~(1 << b);
69         return n;
70      }
71   }
72   return -1;
73}
74
75static inline void
76mm_slab_free(struct mm_slab *slab, int i)
77{
78   assert(i < slab->count);
79   slab->bits[i / 32] |= 1 << (i % 32);
80   slab->free++;
81   assert(slab->free <= slab->count);
82}
83
84static inline int
85mm_get_order(uint32_t size)
86{
87   int s = __builtin_clz(size) ^ 31;
88
89   if (size > (1 << s))
90      s += 1;
91   return s;
92}
93
94static struct mm_bucket *
95mm_bucket_by_order(struct nouveau_mman *cache, int order)
96{
97   if (order > MM_MAX_ORDER)
98      return NULL;
99   return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
100}
101
102static struct mm_bucket *
103mm_bucket_by_size(struct nouveau_mman *cache, unsigned size)
104{
105   return mm_bucket_by_order(cache, mm_get_order(size));
106}
107
108/* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
109static inline uint32_t
110mm_default_slab_size(unsigned chunk_order)
111{
112   static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
113   {
114      12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22, 22
115   };
116
117   assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
118
119   return 1 << slab_order[chunk_order - MM_MIN_ORDER];
120}
121
122static int
123mm_slab_new(struct nouveau_mman *cache, struct mm_bucket *bucket, int chunk_order)
124{
125   struct mm_slab *slab;
126   int words, ret;
127   const uint32_t size = mm_default_slab_size(chunk_order);
128
129   simple_mtx_assert_locked(&bucket->lock);
130
131   words = ((size >> chunk_order) + 31) / 32;
132   assert(words);
133
134   slab = MALLOC(sizeof(struct mm_slab) + words * 4);
135   if (!slab)
136      return PIPE_ERROR_OUT_OF_MEMORY;
137
138   memset(&slab->bits[0], ~0, words * 4);
139
140   slab->bo = NULL;
141
142   ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
143                        &slab->bo);
144   if (ret) {
145      FREE(slab);
146      return PIPE_ERROR_OUT_OF_MEMORY;
147   }
148
149   list_inithead(&slab->head);
150
151   slab->cache = cache;
152   slab->order = chunk_order;
153   slab->count = slab->free = size >> chunk_order;
154
155   assert(bucket == mm_bucket_by_order(cache, chunk_order));
156   list_add(&slab->head, &bucket->free);
157
158   p_atomic_add(&cache->allocated, size);
159
160   if (nouveau_mesa_debug)
161      debug_printf("MM: new slab, total memory = %"PRIu64" KiB\n",
162                   cache->allocated / 1024);
163
164   return PIPE_OK;
165}
166
167/* @return token to identify slab or NULL if we just allocated a new bo */
168struct nouveau_mm_allocation *
169nouveau_mm_allocate(struct nouveau_mman *cache,
170                    uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
171{
172   struct mm_bucket *bucket;
173   struct mm_slab *slab;
174   struct nouveau_mm_allocation *alloc;
175   int ret;
176
177   bucket = mm_bucket_by_size(cache, size);
178   if (!bucket) {
179      ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
180                           bo);
181      if (ret)
182         debug_printf("bo_new(%x, %x): %i\n",
183                      size, cache->config.nv50.memtype, ret);
184
185      *offset = 0;
186      return NULL;
187   }
188
189   alloc = MALLOC_STRUCT(nouveau_mm_allocation);
190   if (!alloc)
191      return NULL;
192
193   simple_mtx_lock(&bucket->lock);
194   if (!list_is_empty(&bucket->used)) {
195      slab = list_entry(bucket->used.next, struct mm_slab, head);
196   } else {
197      if (list_is_empty(&bucket->free)) {
198         mm_slab_new(cache, bucket, MAX2(mm_get_order(size), MM_MIN_ORDER));
199      }
200      slab = list_entry(bucket->free.next, struct mm_slab, head);
201
202      list_del(&slab->head);
203      list_add(&slab->head, &bucket->used);
204   }
205
206   *offset = mm_slab_alloc(slab) << slab->order;
207
208   nouveau_bo_ref(slab->bo, bo);
209
210   if (slab->free == 0) {
211      list_del(&slab->head);
212      list_add(&slab->head, &bucket->full);
213   }
214   simple_mtx_unlock(&bucket->lock);
215
216   alloc->offset = *offset;
217   alloc->priv = (void *)slab;
218
219   return alloc;
220}
221
222void
223nouveau_mm_free(struct nouveau_mm_allocation *alloc)
224{
225   struct mm_slab *slab = (struct mm_slab *)alloc->priv;
226   struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
227
228   simple_mtx_lock(&bucket->lock);
229   mm_slab_free(slab, alloc->offset >> slab->order);
230
231   if (slab->free == slab->count) {
232      list_del(&slab->head);
233      list_addtail(&slab->head, &bucket->free);
234   } else
235   if (slab->free == 1) {
236      list_del(&slab->head);
237      list_addtail(&slab->head, &bucket->used);
238   }
239   simple_mtx_unlock(&bucket->lock);
240
241   FREE(alloc);
242}
243
244void
245nouveau_mm_free_work(void *data)
246{
247   nouveau_mm_free(data);
248}
249
250struct nouveau_mman *
251nouveau_mm_create(struct nouveau_device *dev, uint32_t domain,
252                  union nouveau_bo_config *config)
253{
254   struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman);
255   int i;
256
257   if (!cache)
258      return NULL;
259
260   cache->dev = dev;
261   cache->domain = domain;
262   cache->config = *config;
263   cache->allocated = 0;
264
265   for (i = 0; i < MM_NUM_BUCKETS; ++i) {
266      list_inithead(&cache->bucket[i].free);
267      list_inithead(&cache->bucket[i].used);
268      list_inithead(&cache->bucket[i].full);
269      simple_mtx_init(&cache->bucket[i].lock, mtx_plain);
270   }
271
272   return cache;
273}
274
275static inline void
276nouveau_mm_free_slabs(struct list_head *head)
277{
278   struct mm_slab *slab, *next;
279
280   LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
281      list_del(&slab->head);
282      nouveau_bo_ref(NULL, &slab->bo);
283      FREE(slab);
284   }
285}
286
287void
288nouveau_mm_destroy(struct nouveau_mman *cache)
289{
290   int i;
291
292   if (!cache)
293      return;
294
295   for (i = 0; i < MM_NUM_BUCKETS; ++i) {
296      if (!list_is_empty(&cache->bucket[i].used) ||
297          !list_is_empty(&cache->bucket[i].full))
298         debug_printf("WARNING: destroying GPU memory cache "
299                      "with some buffers still in use\n");
300
301      nouveau_mm_free_slabs(&cache->bucket[i].free);
302      nouveau_mm_free_slabs(&cache->bucket[i].used);
303      nouveau_mm_free_slabs(&cache->bucket[i].full);
304      simple_mtx_destroy(&cache->bucket[i].lock);
305   }
306
307   FREE(cache);
308}
309