1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/slab.h>
8#include <linux/workqueue.h>
9
10#include "i915_active.h"
11#include "gem/i915_gem_context.h"
12#include "gem/i915_gem_object.h"
13#include "i915_globals.h"
14#include "i915_request.h"
15#include "i915_scheduler.h"
16#include "i915_vma.h"
17
18static LIST_HEAD(globals);
19
20static atomic_t active;
21static atomic_t epoch;
22static struct park_work {
23	struct delayed_work work;
24	struct rcu_head rcu;
25	unsigned long flags;
26#define PENDING 0
27	int epoch;
28} park;
29
30static void i915_globals_shrink(void)
31{
32	struct i915_global *global;
33
34	/*
35	 * kmem_cache_shrink() discards empty slabs and reorders partially
36	 * filled slabs to prioritise allocating from the mostly full slabs,
37	 * with the aim of reducing fragmentation.
38	 */
39	list_for_each_entry(global, &globals, link)
40		global->shrink();
41}
42
43static void __i915_globals_grace(struct rcu_head *rcu)
44{
45	/* Ratelimit parking as shrinking is quite slow */
46	schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ));
47}
48
49static void __i915_globals_queue_rcu(void)
50{
51	park.epoch = atomic_inc_return(&epoch);
52	if (!atomic_read(&active)) {
53		init_rcu_head(&park.rcu);
54		call_rcu(&park.rcu, __i915_globals_grace);
55	}
56}
57
58static void __i915_globals_park(struct work_struct *work)
59{
60	destroy_rcu_head(&park.rcu);
61
62	/* Confirm nothing woke up in the last grace period */
63	if (park.epoch != atomic_read(&epoch)) {
64		__i915_globals_queue_rcu();
65		return;
66	}
67
68	clear_bit(PENDING, &park.flags);
69	i915_globals_shrink();
70}
71
72void __init i915_global_register(struct i915_global *global)
73{
74	GEM_BUG_ON(!global->shrink);
75	GEM_BUG_ON(!global->exit);
76
77	list_add_tail(&global->link, &globals);
78}
79
80static void __i915_globals_cleanup(void)
81{
82	struct i915_global *global, *next;
83
84	list_for_each_entry_safe_reverse(global, next, &globals, link)
85		global->exit();
86}
87
88static __initconst int (* const initfn[])(void) = {
89	i915_global_active_init,
90	i915_global_buddy_init,
91	i915_global_context_init,
92	i915_global_gem_context_init,
93	i915_global_objects_init,
94	i915_global_request_init,
95	i915_global_scheduler_init,
96	i915_global_vma_init,
97};
98
99int __init i915_globals_init(void)
100{
101	int i;
102
103	for (i = 0; i < ARRAY_SIZE(initfn); i++) {
104		int err;
105
106		err = initfn[i]();
107		if (err) {
108			__i915_globals_cleanup();
109			return err;
110		}
111	}
112
113	INIT_DELAYED_WORK(&park.work, __i915_globals_park);
114	return 0;
115}
116
117void i915_globals_park(void)
118{
119	/*
120	 * Defer shrinking the global slab caches (and other work) until
121	 * after a RCU grace period has completed with no activity. This
122	 * is to try and reduce the latency impact on the consumers caused
123	 * by us shrinking the caches the same time as they are trying to
124	 * allocate, with the assumption being that if we idle long enough
125	 * for an RCU grace period to elapse since the last use, it is likely
126	 * to be longer until we need the caches again.
127	 */
128	if (!atomic_dec_and_test(&active))
129		return;
130
131	/* Queue cleanup after the next RCU grace period has freed slabs */
132	if (!test_and_set_bit(PENDING, &park.flags))
133		__i915_globals_queue_rcu();
134}
135
136void i915_globals_unpark(void)
137{
138	atomic_inc(&epoch);
139	atomic_inc(&active);
140}
141
142static void __exit __i915_globals_flush(void)
143{
144	atomic_inc(&active); /* skip shrinking */
145
146	rcu_barrier(); /* wait for the work to be queued */
147	flush_delayed_work(&park.work);
148
149	atomic_dec(&active);
150}
151
152void __exit i915_globals_exit(void)
153{
154	GEM_BUG_ON(atomic_read(&active));
155
156	__i915_globals_flush();
157	__i915_globals_cleanup();
158
159	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
160	rcu_barrier();
161}
162