1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#include "habanalabs.h"
9
10#include <linux/slab.h>
11
12static void hl_ctx_fini(struct hl_ctx *ctx)
13{
14	struct hl_device *hdev = ctx->hdev;
15	u64 idle_mask = 0;
16	int i;
17
18	/*
19	 * If we arrived here, there are no jobs waiting for this context
20	 * on its queues so we can safely remove it.
21	 * This is because for each CS, we increment the ref count and for
22	 * every CS that was finished we decrement it and we won't arrive
23	 * to this function unless the ref count is 0
24	 */
25
26	for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
27		hl_fence_put(ctx->cs_pending[i]);
28
29	kfree(ctx->cs_pending);
30
31	if (ctx->asid != HL_KERNEL_ASID_ID) {
32		dev_dbg(hdev->dev, "closing user context %d\n", ctx->asid);
33
34		/* The engines are stopped as there is no executing CS, but the
35		 * Coresight might be still working by accessing addresses
36		 * related to the stopped engines. Hence stop it explicitly.
37		 * Stop only if this is the compute context, as there can be
38		 * only one compute context
39		 */
40		if ((hdev->in_debug) && (hdev->compute_ctx == ctx))
41			hl_device_set_debug_mode(hdev, false);
42
43		hl_cb_va_pool_fini(ctx);
44		hl_vm_ctx_fini(ctx);
45		hl_asid_free(hdev, ctx->asid);
46
47		if ((!hdev->pldm) && (hdev->pdev) &&
48				(!hdev->asic_funcs->is_device_idle(hdev,
49							&idle_mask, NULL)))
50			dev_notice(hdev->dev,
51				"device not idle after user context is closed (0x%llx)\n",
52				idle_mask);
53	} else {
54		dev_dbg(hdev->dev, "closing kernel context\n");
55		hl_mmu_ctx_fini(ctx);
56	}
57}
58
59void hl_ctx_do_release(struct kref *ref)
60{
61	struct hl_ctx *ctx;
62
63	ctx = container_of(ref, struct hl_ctx, refcount);
64
65	hl_ctx_fini(ctx);
66
67	if (ctx->hpriv)
68		hl_hpriv_put(ctx->hpriv);
69
70	kfree(ctx);
71}
72
73int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
74{
75	struct hl_ctx_mgr *mgr = &hpriv->ctx_mgr;
76	struct hl_ctx *ctx;
77	int rc;
78
79	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
80	if (!ctx) {
81		rc = -ENOMEM;
82		goto out_err;
83	}
84
85	mutex_lock(&mgr->ctx_lock);
86	rc = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
87	mutex_unlock(&mgr->ctx_lock);
88
89	if (rc < 0) {
90		dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
91		goto free_ctx;
92	}
93
94	ctx->handle = rc;
95
96	rc = hl_ctx_init(hdev, ctx, false);
97	if (rc)
98		goto remove_from_idr;
99
100	hl_hpriv_get(hpriv);
101	ctx->hpriv = hpriv;
102
103	/* TODO: remove for multiple contexts per process */
104	hpriv->ctx = ctx;
105
106	/* TODO: remove the following line for multiple process support */
107	hdev->compute_ctx = ctx;
108
109	return 0;
110
111remove_from_idr:
112	mutex_lock(&mgr->ctx_lock);
113	idr_remove(&mgr->ctx_handles, ctx->handle);
114	mutex_unlock(&mgr->ctx_lock);
115free_ctx:
116	kfree(ctx);
117out_err:
118	return rc;
119}
120
121void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx)
122{
123	if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)
124		return;
125
126	dev_warn(hdev->dev,
127		"user process released device but its command submissions are still executing\n");
128}
129
130int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
131{
132	int rc = 0;
133
134	ctx->hdev = hdev;
135
136	kref_init(&ctx->refcount);
137
138	ctx->cs_sequence = 1;
139	spin_lock_init(&ctx->cs_lock);
140	atomic_set(&ctx->thread_ctx_switch_token, 1);
141	ctx->thread_ctx_switch_wait_token = 0;
142	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
143				sizeof(struct hl_fence *),
144				GFP_KERNEL);
145	if (!ctx->cs_pending)
146		return -ENOMEM;
147
148	if (is_kernel_ctx) {
149		ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
150		rc = hl_mmu_ctx_init(ctx);
151		if (rc) {
152			dev_err(hdev->dev, "Failed to init mmu ctx module\n");
153			goto err_free_cs_pending;
154		}
155	} else {
156		ctx->asid = hl_asid_alloc(hdev);
157		if (!ctx->asid) {
158			dev_err(hdev->dev, "No free ASID, failed to create context\n");
159			rc = -ENOMEM;
160			goto err_free_cs_pending;
161		}
162
163		rc = hl_vm_ctx_init(ctx);
164		if (rc) {
165			dev_err(hdev->dev, "Failed to init mem ctx module\n");
166			rc = -ENOMEM;
167			goto err_asid_free;
168		}
169
170		rc = hl_cb_va_pool_init(ctx);
171		if (rc) {
172			dev_err(hdev->dev,
173				"Failed to init VA pool for mapped CB\n");
174			goto err_vm_ctx_fini;
175		}
176
177		rc = hdev->asic_funcs->ctx_init(ctx);
178		if (rc) {
179			dev_err(hdev->dev, "ctx_init failed\n");
180			goto err_cb_va_pool_fini;
181		}
182
183		dev_dbg(hdev->dev, "create user context %d\n", ctx->asid);
184	}
185
186	return 0;
187
188err_cb_va_pool_fini:
189	hl_cb_va_pool_fini(ctx);
190err_vm_ctx_fini:
191	hl_vm_ctx_fini(ctx);
192err_asid_free:
193	hl_asid_free(hdev, ctx->asid);
194err_free_cs_pending:
195	kfree(ctx->cs_pending);
196
197	return rc;
198}
199
200void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx)
201{
202	kref_get(&ctx->refcount);
203}
204
205int hl_ctx_put(struct hl_ctx *ctx)
206{
207	return kref_put(&ctx->refcount, hl_ctx_do_release);
208}
209
210struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
211{
212	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
213	struct hl_fence *fence;
214
215	spin_lock(&ctx->cs_lock);
216
217	if (seq >= ctx->cs_sequence) {
218		spin_unlock(&ctx->cs_lock);
219		return ERR_PTR(-EINVAL);
220	}
221
222	if (seq + asic_prop->max_pending_cs < ctx->cs_sequence) {
223		spin_unlock(&ctx->cs_lock);
224		return NULL;
225	}
226
227	fence = ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)];
228	hl_fence_get(fence);
229
230	spin_unlock(&ctx->cs_lock);
231
232	return fence;
233}
234
235/*
236 * hl_ctx_mgr_init - initialize the context manager
237 *
238 * @mgr: pointer to context manager structure
239 *
240 * This manager is an object inside the hpriv object of the user process.
241 * The function is called when a user process opens the FD.
242 */
243void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr)
244{
245	mutex_init(&mgr->ctx_lock);
246	idr_init(&mgr->ctx_handles);
247}
248
249/*
250 * hl_ctx_mgr_fini - finalize the context manager
251 *
252 * @hdev: pointer to device structure
253 * @mgr: pointer to context manager structure
254 *
255 * This function goes over all the contexts in the manager and frees them.
256 * It is called when a process closes the FD.
257 */
258void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr)
259{
260	struct hl_ctx *ctx;
261	struct idr *idp;
262	u32 id;
263
264	idp = &mgr->ctx_handles;
265
266	idr_for_each_entry(idp, ctx, id)
267		hl_ctx_free(hdev, ctx);
268
269	idr_destroy(&mgr->ctx_handles);
270	mutex_destroy(&mgr->ctx_lock);
271}
272