1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2016 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include <linux/vmalloc.h>
8#include <linux/sched/mm.h>
9
10#include "msm_drv.h"
11#include "msm_gem.h"
12#include "msm_gpu.h"
13#include "msm_gpu_trace.h"
14
15/* Default disabled for now until it has some more testing on the different
16 * iommu combinations that can be paired with the driver:
17 */
18static bool enable_eviction = true;
19MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers");
20module_param(enable_eviction, bool, 0600);
21
22static bool can_swap(void)
23{
24	return enable_eviction && get_nr_swap_pages() > 0;
25}
26
27static bool can_block(struct shrink_control *sc)
28{
29	if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM))
30		return false;
31	return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM);
32}
33
34static unsigned long
35msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
36{
37	struct msm_drm_private *priv =
38		container_of(shrinker, struct msm_drm_private, shrinker);
39	unsigned count = priv->lru.dontneed.count;
40
41	if (can_swap())
42		count += priv->lru.willneed.count;
43
44	return count;
45}
46
47static bool
48purge(struct drm_gem_object *obj)
49{
50	if (!is_purgeable(to_msm_bo(obj)))
51		return false;
52
53	if (msm_gem_active(obj))
54		return false;
55
56	msm_gem_purge(obj);
57
58	return true;
59}
60
61static bool
62evict(struct drm_gem_object *obj)
63{
64	if (is_unevictable(to_msm_bo(obj)))
65		return false;
66
67	if (msm_gem_active(obj))
68		return false;
69
70	msm_gem_evict(obj);
71
72	return true;
73}
74
75static bool
76wait_for_idle(struct drm_gem_object *obj)
77{
78	enum dma_resv_usage usage = dma_resv_usage_rw(true);
79	return dma_resv_wait_timeout(obj->resv, usage, false, 1000) > 0;
80}
81
82static bool
83active_purge(struct drm_gem_object *obj)
84{
85	if (!wait_for_idle(obj))
86		return false;
87
88	return purge(obj);
89}
90
91static bool
92active_evict(struct drm_gem_object *obj)
93{
94	if (!wait_for_idle(obj))
95		return false;
96
97	return evict(obj);
98}
99
100static unsigned long
101msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
102{
103	struct msm_drm_private *priv =
104		container_of(shrinker, struct msm_drm_private, shrinker);
105	struct {
106		struct drm_gem_lru *lru;
107		bool (*shrink)(struct drm_gem_object *obj);
108		bool cond;
109		unsigned long freed;
110		unsigned long remaining;
111	} stages[] = {
112		/* Stages of progressively more aggressive/expensive reclaim: */
113		{ &priv->lru.dontneed, purge,        true },
114		{ &priv->lru.willneed, evict,        can_swap() },
115		{ &priv->lru.dontneed, active_purge, can_block(sc) },
116		{ &priv->lru.willneed, active_evict, can_swap() && can_block(sc) },
117	};
118	long nr = sc->nr_to_scan;
119	unsigned long freed = 0;
120	unsigned long remaining = 0;
121
122	for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
123		if (!stages[i].cond)
124			continue;
125		stages[i].freed =
126			drm_gem_lru_scan(stages[i].lru, nr,
127					&stages[i].remaining,
128					 stages[i].shrink);
129		nr -= stages[i].freed;
130		freed += stages[i].freed;
131		remaining += stages[i].remaining;
132	}
133
134	if (freed) {
135		trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed,
136				     stages[1].freed, stages[2].freed,
137				     stages[3].freed);
138	}
139
140	return (freed > 0 && remaining > 0) ? freed : SHRINK_STOP;
141}
142
143#ifdef CONFIG_DEBUG_FS
144unsigned long
145msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
146{
147	struct msm_drm_private *priv = dev->dev_private;
148	struct shrink_control sc = {
149		.nr_to_scan = nr_to_scan,
150	};
151	int ret;
152
153	fs_reclaim_acquire(GFP_KERNEL);
154	ret = msm_gem_shrinker_scan(&priv->shrinker, &sc);
155	fs_reclaim_release(GFP_KERNEL);
156
157	return ret;
158}
159#endif
160
161/* since we don't know any better, lets bail after a few
162 * and if necessary the shrinker will be invoked again.
163 * Seems better than unmapping *everything*
164 */
165static const int vmap_shrink_limit = 15;
166
167static bool
168vmap_shrink(struct drm_gem_object *obj)
169{
170	if (!is_vunmapable(to_msm_bo(obj)))
171		return false;
172
173	msm_gem_vunmap(obj);
174
175	return true;
176}
177
178static int
179msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
180{
181	struct msm_drm_private *priv =
182		container_of(nb, struct msm_drm_private, vmap_notifier);
183	struct drm_gem_lru *lrus[] = {
184		&priv->lru.dontneed,
185		&priv->lru.willneed,
186		&priv->lru.pinned,
187		NULL,
188	};
189	unsigned idx, unmapped = 0;
190	unsigned long remaining = 0;
191
192	for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
193		unmapped += drm_gem_lru_scan(lrus[idx],
194					     vmap_shrink_limit - unmapped,
195					     &remaining,
196					     vmap_shrink);
197	}
198
199	*(unsigned long *)ptr += unmapped;
200
201	if (unmapped > 0)
202		trace_msm_gem_purge_vmaps(unmapped);
203
204	return NOTIFY_DONE;
205}
206
207/**
208 * msm_gem_shrinker_init - Initialize msm shrinker
209 * @dev: drm device
210 *
211 * This function registers and sets up the msm shrinker.
212 */
213void msm_gem_shrinker_init(struct drm_device *dev)
214{
215	struct msm_drm_private *priv = dev->dev_private;
216	priv->shrinker.count_objects = msm_gem_shrinker_count;
217	priv->shrinker.scan_objects = msm_gem_shrinker_scan;
218	priv->shrinker.seeks = DEFAULT_SEEKS;
219	WARN_ON(register_shrinker(&priv->shrinker, "drm-msm_gem"));
220
221	priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
222	WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
223}
224
225/**
226 * msm_gem_shrinker_cleanup - Clean up msm shrinker
227 * @dev: drm device
228 *
229 * This function unregisters the msm shrinker.
230 */
231void msm_gem_shrinker_cleanup(struct drm_device *dev)
232{
233	struct msm_drm_private *priv = dev->dev_private;
234
235	if (priv->shrinker.nr_deferred) {
236		WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
237		unregister_shrinker(&priv->shrinker);
238	}
239}
240