1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Copyright (C) 2023 Loongson Technology Corporation Limited
4 */
5
6#include <drm/drm_drv.h>
7#include <drm/drm_file.h>
8#include <drm/drm_gem.h>
9#include <drm/drm_managed.h>
10#include <drm/drm_prime.h>
11
12#include "lsdc_drv.h"
13#include "lsdc_ttm.h"
14
15const char *lsdc_mem_type_to_str(uint32_t mem_type)
16{
17	switch (mem_type) {
18	case TTM_PL_VRAM:
19		return "VRAM";
20	case TTM_PL_TT:
21		return "GTT";
22	case TTM_PL_SYSTEM:
23		return "SYSTEM";
24	default:
25		break;
26	}
27
28	return "Unknown";
29}
30
31const char *lsdc_domain_to_str(u32 domain)
32{
33	switch (domain) {
34	case LSDC_GEM_DOMAIN_VRAM:
35		return "VRAM";
36	case LSDC_GEM_DOMAIN_GTT:
37		return "GTT";
38	case LSDC_GEM_DOMAIN_SYSTEM:
39		return "SYSTEM";
40	default:
41		break;
42	}
43
44	return "Unknown";
45}
46
47static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
48{
49	u32 c = 0;
50	u32 pflags = 0;
51	u32 i;
52
53	if (lbo->tbo.base.size <= PAGE_SIZE)
54		pflags |= TTM_PL_FLAG_TOPDOWN;
55
56	lbo->placement.placement = lbo->placements;
57	lbo->placement.busy_placement = lbo->placements;
58
59	if (domain & LSDC_GEM_DOMAIN_VRAM) {
60		lbo->placements[c].mem_type = TTM_PL_VRAM;
61		lbo->placements[c++].flags = pflags;
62	}
63
64	if (domain & LSDC_GEM_DOMAIN_GTT) {
65		lbo->placements[c].mem_type = TTM_PL_TT;
66		lbo->placements[c++].flags = pflags;
67	}
68
69	if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
70		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
71		lbo->placements[c++].flags = 0;
72	}
73
74	if (!c) {
75		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
76		lbo->placements[c++].flags = 0;
77	}
78
79	lbo->placement.num_placement = c;
80	lbo->placement.num_busy_placement = c;
81
82	for (i = 0; i < c; ++i) {
83		lbo->placements[i].fpfn = 0;
84		lbo->placements[i].lpfn = 0;
85	}
86}
87
88static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
89{
90	ttm_tt_fini(tt);
91	kfree(tt);
92}
93
94static struct ttm_tt *
95lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
96{
97	struct ttm_tt *tt;
98	int ret;
99
100	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
101	if (!tt)
102		return NULL;
103
104	ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
105	if (ret < 0) {
106		kfree(tt);
107		return NULL;
108	}
109
110	return tt;
111}
112
113static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
114				struct ttm_tt *ttm,
115				struct ttm_operation_ctx *ctx)
116{
117	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
118
119	if (slave && ttm->sg) {
120		drm_prime_sg_to_dma_addr_array(ttm->sg,
121					       ttm->dma_address,
122					       ttm->num_pages);
123
124		return 0;
125	}
126
127	return ttm_pool_alloc(&bdev->pool, ttm, ctx);
128}
129
130static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
131				   struct ttm_tt *ttm)
132{
133	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
134
135	if (slave)
136		return;
137
138	return ttm_pool_free(&bdev->pool, ttm);
139}
140
141static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
142				struct ttm_placement *tplacement)
143{
144	struct ttm_resource *resource = tbo->resource;
145	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
146
147	switch (resource->mem_type) {
148	case TTM_PL_VRAM:
149		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
150		break;
151	case TTM_PL_TT:
152	default:
153		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
154		break;
155	}
156
157	*tplacement = lbo->placement;
158}
159
160static int lsdc_bo_move(struct ttm_buffer_object *tbo,
161			bool evict,
162			struct ttm_operation_ctx *ctx,
163			struct ttm_resource *new_mem,
164			struct ttm_place *hop)
165{
166	struct drm_device *ddev = tbo->base.dev;
167	struct ttm_resource *old_mem = tbo->resource;
168	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
169	int ret;
170
171	if (unlikely(tbo->pin_count > 0)) {
172		drm_warn(ddev, "Can't move a pinned BO\n");
173		return -EINVAL;
174	}
175
176	ret = ttm_bo_wait_ctx(tbo, ctx);
177	if (ret)
178		return ret;
179
180	if (!old_mem) {
181		drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
182			lbo, lsdc_mem_type_to_str(new_mem->mem_type),
183			lsdc_bo_size(lbo));
184		ttm_bo_move_null(tbo, new_mem);
185		return 0;
186	}
187
188	if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
189		ttm_bo_move_null(tbo, new_mem);
190		drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
191			lbo, lsdc_bo_size(lbo));
192		return 0;
193	}
194
195	if (old_mem->mem_type == TTM_PL_SYSTEM &&
196	    new_mem->mem_type == TTM_PL_TT) {
197		drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
198			lbo, lsdc_bo_size(lbo));
199		ttm_bo_move_null(tbo, new_mem);
200		return 0;
201	}
202
203	if (old_mem->mem_type == TTM_PL_TT &&
204	    new_mem->mem_type == TTM_PL_SYSTEM) {
205		drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
206			lbo, lsdc_bo_size(lbo));
207		ttm_resource_free(tbo, &tbo->resource);
208		ttm_bo_assign_mem(tbo, new_mem);
209		return 0;
210	}
211
212	drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
213		lbo,
214		lsdc_mem_type_to_str(old_mem->mem_type),
215		lsdc_mem_type_to_str(new_mem->mem_type),
216		lsdc_bo_size(lbo));
217
218	return ttm_bo_move_memcpy(tbo, ctx, new_mem);
219}
220
221static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
222				  struct ttm_resource *mem)
223{
224	struct lsdc_device *ldev = tdev_to_ldev(bdev);
225
226	switch (mem->mem_type) {
227	case TTM_PL_SYSTEM:
228		break;
229	case TTM_PL_TT:
230		break;
231	case TTM_PL_VRAM:
232		mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
233		mem->bus.is_iomem = true;
234		mem->bus.caching = ttm_write_combined;
235		break;
236	default:
237		return -EINVAL;
238	}
239
240	return 0;
241}
242
243static struct ttm_device_funcs lsdc_bo_driver = {
244	.ttm_tt_create = lsdc_ttm_tt_create,
245	.ttm_tt_populate = lsdc_ttm_tt_populate,
246	.ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
247	.ttm_tt_destroy = lsdc_ttm_tt_destroy,
248	.eviction_valuable = ttm_bo_eviction_valuable,
249	.evict_flags = lsdc_bo_evict_flags,
250	.move = lsdc_bo_move,
251	.io_mem_reserve = lsdc_bo_reserve_io_mem,
252};
253
254u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
255{
256	struct ttm_buffer_object *tbo = &lbo->tbo;
257	struct drm_device *ddev = tbo->base.dev;
258	struct ttm_resource *resource = tbo->resource;
259
260	if (unlikely(!tbo->pin_count)) {
261		drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
262		return 0;
263	}
264
265	if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
266		return 0;
267
268	return resource->start << PAGE_SHIFT;
269}
270
271size_t lsdc_bo_size(struct lsdc_bo *lbo)
272{
273	struct ttm_buffer_object *tbo = &lbo->tbo;
274
275	return tbo->base.size;
276}
277
278int lsdc_bo_reserve(struct lsdc_bo *lbo)
279{
280	return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
281}
282
283void lsdc_bo_unreserve(struct lsdc_bo *lbo)
284{
285	return ttm_bo_unreserve(&lbo->tbo);
286}
287
288int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
289{
290	struct ttm_operation_ctx ctx = { false, false };
291	struct ttm_buffer_object *tbo = &lbo->tbo;
292	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
293	int ret;
294
295	if (tbo->pin_count)
296		goto bo_pinned;
297
298	if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
299		return -EINVAL;
300
301	if (domain)
302		lsdc_bo_set_placement(lbo, domain);
303
304	ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
305	if (unlikely(ret)) {
306		drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
307		return ret;
308	}
309
310	if (domain == LSDC_GEM_DOMAIN_VRAM)
311		ldev->vram_pinned_size += lsdc_bo_size(lbo);
312	else if (domain == LSDC_GEM_DOMAIN_GTT)
313		ldev->gtt_pinned_size += lsdc_bo_size(lbo);
314
315bo_pinned:
316	ttm_bo_pin(tbo);
317
318	if (gpu_addr)
319		*gpu_addr = lsdc_bo_gpu_offset(lbo);
320
321	return 0;
322}
323
324void lsdc_bo_unpin(struct lsdc_bo *lbo)
325{
326	struct ttm_buffer_object *tbo = &lbo->tbo;
327	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
328
329	if (unlikely(!tbo->pin_count)) {
330		drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
331		return;
332	}
333
334	ttm_bo_unpin(tbo);
335
336	if (!tbo->pin_count) {
337		if (tbo->resource->mem_type == TTM_PL_VRAM)
338			ldev->vram_pinned_size -= lsdc_bo_size(lbo);
339		else if (tbo->resource->mem_type == TTM_PL_TT)
340			ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
341	}
342}
343
344void lsdc_bo_ref(struct lsdc_bo *lbo)
345{
346	struct ttm_buffer_object *tbo = &lbo->tbo;
347
348	ttm_bo_get(tbo);
349}
350
351void lsdc_bo_unref(struct lsdc_bo *lbo)
352{
353	struct ttm_buffer_object *tbo = &lbo->tbo;
354
355	ttm_bo_put(tbo);
356}
357
358int lsdc_bo_kmap(struct lsdc_bo *lbo)
359{
360	struct ttm_buffer_object *tbo = &lbo->tbo;
361	struct drm_gem_object *gem = &tbo->base;
362	struct drm_device *ddev = gem->dev;
363	long ret;
364	int err;
365
366	ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
367				    MAX_SCHEDULE_TIMEOUT);
368	if (ret < 0) {
369		drm_warn(ddev, "wait fence timeout\n");
370		return ret;
371	}
372
373	if (lbo->kptr)
374		return 0;
375
376	err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
377	if (err) {
378		drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
379		return err;
380	}
381
382	lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
383
384	return 0;
385}
386
387void lsdc_bo_kunmap(struct lsdc_bo *lbo)
388{
389	if (!lbo->kptr)
390		return;
391
392	lbo->kptr = NULL;
393	ttm_bo_kunmap(&lbo->kmap);
394}
395
396void lsdc_bo_clear(struct lsdc_bo *lbo)
397{
398	lsdc_bo_kmap(lbo);
399
400	if (lbo->is_iomem)
401		memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
402	else
403		memset(lbo->kptr, 0, lbo->size);
404
405	lsdc_bo_kunmap(lbo);
406}
407
408int lsdc_bo_evict_vram(struct drm_device *ddev)
409{
410	struct lsdc_device *ldev = to_lsdc(ddev);
411	struct ttm_device *bdev = &ldev->bdev;
412	struct ttm_resource_manager *man;
413
414	man = ttm_manager_type(bdev, TTM_PL_VRAM);
415	if (unlikely(!man))
416		return 0;
417
418	return ttm_resource_manager_evict_all(bdev, man);
419}
420
421static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
422{
423	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
424	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
425
426	mutex_lock(&ldev->gem.mutex);
427	list_del_init(&lbo->list);
428	mutex_unlock(&ldev->gem.mutex);
429
430	drm_gem_object_release(&tbo->base);
431
432	kfree(lbo);
433}
434
435struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
436			       u32 domain,
437			       size_t size,
438			       bool kernel,
439			       struct sg_table *sg,
440			       struct dma_resv *resv)
441{
442	struct lsdc_device *ldev = to_lsdc(ddev);
443	struct ttm_device *bdev = &ldev->bdev;
444	struct ttm_buffer_object *tbo;
445	struct lsdc_bo *lbo;
446	enum ttm_bo_type bo_type;
447	int ret;
448
449	lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
450	if (!lbo)
451		return ERR_PTR(-ENOMEM);
452
453	INIT_LIST_HEAD(&lbo->list);
454
455	lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
456					LSDC_GEM_DOMAIN_GTT |
457					LSDC_GEM_DOMAIN_SYSTEM);
458
459	tbo = &lbo->tbo;
460
461	size = ALIGN(size, PAGE_SIZE);
462
463	ret = drm_gem_object_init(ddev, &tbo->base, size);
464	if (ret) {
465		kfree(lbo);
466		return ERR_PTR(ret);
467	}
468
469	tbo->bdev = bdev;
470
471	if (kernel)
472		bo_type = ttm_bo_type_kernel;
473	else if (sg)
474		bo_type = ttm_bo_type_sg;
475	else
476		bo_type = ttm_bo_type_device;
477
478	lsdc_bo_set_placement(lbo, domain);
479	lbo->size = size;
480
481	ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
482				   false, sg, resv, lsdc_bo_destroy);
483	if (ret) {
484		kfree(lbo);
485		return ERR_PTR(ret);
486	}
487
488	return lbo;
489}
490
491struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
492					     u32 domain,
493					     size_t size)
494{
495	struct lsdc_bo *lbo;
496	int ret;
497
498	lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
499	if (IS_ERR(lbo))
500		return ERR_CAST(lbo);
501
502	ret = lsdc_bo_reserve(lbo);
503	if (unlikely(ret)) {
504		lsdc_bo_unref(lbo);
505		return ERR_PTR(ret);
506	}
507
508	ret = lsdc_bo_pin(lbo, domain, NULL);
509	lsdc_bo_unreserve(lbo);
510	if (unlikely(ret)) {
511		lsdc_bo_unref(lbo);
512		return ERR_PTR(ret);
513	}
514
515	return lbo;
516}
517
518void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
519{
520	int ret;
521
522	ret = lsdc_bo_reserve(lbo);
523	if (unlikely(ret))
524		return;
525
526	lsdc_bo_unpin(lbo);
527	lsdc_bo_unreserve(lbo);
528
529	lsdc_bo_unref(lbo);
530}
531
532static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
533{
534	struct lsdc_device *ldev = (struct lsdc_device *)data;
535
536	ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
537	ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
538
539	ttm_device_fini(&ldev->bdev);
540
541	drm_dbg(ddev, "ttm finished\n");
542}
543
544int lsdc_ttm_init(struct lsdc_device *ldev)
545{
546	struct drm_device *ddev = &ldev->base;
547	unsigned long num_vram_pages;
548	unsigned long num_gtt_pages;
549	int ret;
550
551	ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
552			      ddev->anon_inode->i_mapping,
553			      ddev->vma_offset_manager, false, true);
554	if (ret)
555		return ret;
556
557	num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
558
559	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
560	if (unlikely(ret))
561		return ret;
562
563	drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
564
565	/* 512M is far enough for us now */
566	ldev->gtt_size = 512 << 20;
567
568	num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
569
570	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
571	if (unlikely(ret))
572		return ret;
573
574	drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
575
576	return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
577}
578
579void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
580{
581	struct ttm_device *bdev = &ldev->bdev;
582	struct drm_device *ddev = &ldev->base;
583	struct drm_minor *minor = ddev->primary;
584	struct dentry *root = minor->debugfs_root;
585	struct ttm_resource_manager *vram_man;
586	struct ttm_resource_manager *gtt_man;
587
588	vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
589	gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
590
591	ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
592	ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
593}
594