1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2/* Copyright(c) 2014 - 2020 Intel Corporation */
3#include <linux/mutex.h>
4#include <linux/list.h>
5#include <linux/bitops.h>
6#include <linux/delay.h>
7#include "adf_accel_devices.h"
8#include "adf_cfg.h"
9#include "adf_common_drv.h"
10
11static LIST_HEAD(service_table);
12static DEFINE_MUTEX(service_lock);
13
14static void adf_service_add(struct service_hndl *service)
15{
16	mutex_lock(&service_lock);
17	list_add(&service->list, &service_table);
18	mutex_unlock(&service_lock);
19}
20
21int adf_service_register(struct service_hndl *service)
22{
23	memset(service->init_status, 0, sizeof(service->init_status));
24	memset(service->start_status, 0, sizeof(service->start_status));
25	adf_service_add(service);
26	return 0;
27}
28
29static void adf_service_remove(struct service_hndl *service)
30{
31	mutex_lock(&service_lock);
32	list_del(&service->list);
33	mutex_unlock(&service_lock);
34}
35
36int adf_service_unregister(struct service_hndl *service)
37{
38	int i;
39
40	for (i = 0; i < ARRAY_SIZE(service->init_status); i++) {
41		if (service->init_status[i] || service->start_status[i]) {
42			pr_err("QAT: Could not remove active service\n");
43			return -EFAULT;
44		}
45	}
46	adf_service_remove(service);
47	return 0;
48}
49
50/**
51 * adf_dev_init() - Init data structures and services for the given accel device
52 * @accel_dev: Pointer to acceleration device.
53 *
54 * Initialize the ring data structures and the admin comms and arbitration
55 * services.
56 *
57 * Return: 0 on success, error code otherwise.
58 */
59int adf_dev_init(struct adf_accel_dev *accel_dev)
60{
61	struct service_hndl *service;
62	struct list_head *list_itr;
63	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
64	int ret;
65
66	if (!hw_data) {
67		dev_err(&GET_DEV(accel_dev),
68			"Failed to init device - hw_data not set\n");
69		return -EFAULT;
70	}
71
72	if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
73		dev_err(&GET_DEV(accel_dev), "Device not configured\n");
74		return -EFAULT;
75	}
76
77	if (adf_init_etr_data(accel_dev)) {
78		dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
79		return -EFAULT;
80	}
81
82	if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
83		dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
84		return -EFAULT;
85	}
86
87	if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
88		dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
89		return -EFAULT;
90	}
91
92	hw_data->enable_ints(accel_dev);
93
94	if (adf_ae_init(accel_dev)) {
95		dev_err(&GET_DEV(accel_dev),
96			"Failed to initialise Acceleration Engine\n");
97		return -EFAULT;
98	}
99	set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
100
101	if (adf_ae_fw_load(accel_dev)) {
102		dev_err(&GET_DEV(accel_dev),
103			"Failed to load acceleration FW\n");
104		return -EFAULT;
105	}
106	set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
107
108	if (hw_data->alloc_irq(accel_dev)) {
109		dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
110		return -EFAULT;
111	}
112	set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
113
114	/*
115	 * Subservice initialisation is divided into two stages: init and start.
116	 * This is to facilitate any ordering dependencies between services
117	 * prior to starting any of the accelerators.
118	 */
119	list_for_each(list_itr, &service_table) {
120		service = list_entry(list_itr, struct service_hndl, list);
121		if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
122			dev_err(&GET_DEV(accel_dev),
123				"Failed to initialise service %s\n",
124				service->name);
125			return -EFAULT;
126		}
127		set_bit(accel_dev->accel_id, service->init_status);
128	}
129
130	hw_data->enable_error_correction(accel_dev);
131	ret = hw_data->enable_vf2pf_comms(accel_dev);
132
133	return ret;
134}
135EXPORT_SYMBOL_GPL(adf_dev_init);
136
137/**
138 * adf_dev_start() - Start acceleration service for the given accel device
139 * @accel_dev:    Pointer to acceleration device.
140 *
141 * Function notifies all the registered services that the acceleration device
142 * is ready to be used.
143 * To be used by QAT device specific drivers.
144 *
145 * Return: 0 on success, error code otherwise.
146 */
147int adf_dev_start(struct adf_accel_dev *accel_dev)
148{
149	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
150	struct service_hndl *service;
151	struct list_head *list_itr;
152
153	set_bit(ADF_STATUS_STARTING, &accel_dev->status);
154
155	if (adf_ae_start(accel_dev)) {
156		dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
157		return -EFAULT;
158	}
159	set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
160
161	if (hw_data->send_admin_init(accel_dev)) {
162		dev_err(&GET_DEV(accel_dev), "Failed to send init message\n");
163		return -EFAULT;
164	}
165
166	list_for_each(list_itr, &service_table) {
167		service = list_entry(list_itr, struct service_hndl, list);
168		if (service->event_hld(accel_dev, ADF_EVENT_START)) {
169			dev_err(&GET_DEV(accel_dev),
170				"Failed to start service %s\n",
171				service->name);
172			return -EFAULT;
173		}
174		set_bit(accel_dev->accel_id, service->start_status);
175	}
176
177	clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
178	set_bit(ADF_STATUS_STARTED, &accel_dev->status);
179
180	if (!list_empty(&accel_dev->crypto_list) &&
181	    (qat_algs_register() || qat_asym_algs_register())) {
182		dev_err(&GET_DEV(accel_dev),
183			"Failed to register crypto algs\n");
184		set_bit(ADF_STATUS_STARTING, &accel_dev->status);
185		clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
186		return -EFAULT;
187	}
188	return 0;
189}
190EXPORT_SYMBOL_GPL(adf_dev_start);
191
192/**
193 * adf_dev_stop() - Stop acceleration service for the given accel device
194 * @accel_dev:    Pointer to acceleration device.
195 *
196 * Function notifies all the registered services that the acceleration device
197 * is shuting down.
198 * To be used by QAT device specific drivers.
199 *
200 * Return: void
201 */
202void adf_dev_stop(struct adf_accel_dev *accel_dev)
203{
204	struct service_hndl *service;
205	struct list_head *list_itr;
206	bool wait = false;
207	int ret;
208
209	if (!adf_dev_started(accel_dev) &&
210	    !test_bit(ADF_STATUS_STARTING, &accel_dev->status))
211		return;
212
213	clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
214	clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
215
216	if (!list_empty(&accel_dev->crypto_list)) {
217		qat_algs_unregister();
218		qat_asym_algs_unregister();
219	}
220
221	list_for_each(list_itr, &service_table) {
222		service = list_entry(list_itr, struct service_hndl, list);
223		if (!test_bit(accel_dev->accel_id, service->start_status))
224			continue;
225		ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
226		if (!ret) {
227			clear_bit(accel_dev->accel_id, service->start_status);
228		} else if (ret == -EAGAIN) {
229			wait = true;
230			clear_bit(accel_dev->accel_id, service->start_status);
231		}
232	}
233
234	if (wait)
235		msleep(100);
236
237	if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
238		if (adf_ae_stop(accel_dev))
239			dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
240		else
241			clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
242	}
243}
244EXPORT_SYMBOL_GPL(adf_dev_stop);
245
246/**
247 * adf_dev_shutdown() - shutdown acceleration services and data strucutures
248 * @accel_dev: Pointer to acceleration device
249 *
250 * Cleanup the ring data structures and the admin comms and arbitration
251 * services.
252 */
253void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
254{
255	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
256	struct service_hndl *service;
257	struct list_head *list_itr;
258
259	if (!hw_data) {
260		dev_err(&GET_DEV(accel_dev),
261			"QAT: Failed to shutdown device - hw_data not set\n");
262		return;
263	}
264
265	if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
266		adf_ae_fw_release(accel_dev);
267		clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
268	}
269
270	if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
271		if (adf_ae_shutdown(accel_dev))
272			dev_err(&GET_DEV(accel_dev),
273				"Failed to shutdown Accel Engine\n");
274		else
275			clear_bit(ADF_STATUS_AE_INITIALISED,
276				  &accel_dev->status);
277	}
278
279	list_for_each(list_itr, &service_table) {
280		service = list_entry(list_itr, struct service_hndl, list);
281		if (!test_bit(accel_dev->accel_id, service->init_status))
282			continue;
283		if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
284			dev_err(&GET_DEV(accel_dev),
285				"Failed to shutdown service %s\n",
286				service->name);
287		else
288			clear_bit(accel_dev->accel_id, service->init_status);
289	}
290
291	hw_data->disable_iov(accel_dev);
292
293	if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) {
294		hw_data->free_irq(accel_dev);
295		clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
296	}
297
298	/* Delete configuration only if not restarting */
299	if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
300		adf_cfg_del_all(accel_dev);
301
302	if (hw_data->exit_arb)
303		hw_data->exit_arb(accel_dev);
304
305	if (hw_data->exit_admin_comms)
306		hw_data->exit_admin_comms(accel_dev);
307
308	adf_cleanup_etr_data(accel_dev);
309	adf_dev_restore(accel_dev);
310}
311EXPORT_SYMBOL_GPL(adf_dev_shutdown);
312
313int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
314{
315	struct service_hndl *service;
316	struct list_head *list_itr;
317
318	list_for_each(list_itr, &service_table) {
319		service = list_entry(list_itr, struct service_hndl, list);
320		if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
321			dev_err(&GET_DEV(accel_dev),
322				"Failed to restart service %s.\n",
323				service->name);
324	}
325	return 0;
326}
327
328int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
329{
330	struct service_hndl *service;
331	struct list_head *list_itr;
332
333	list_for_each(list_itr, &service_table) {
334		service = list_entry(list_itr, struct service_hndl, list);
335		if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
336			dev_err(&GET_DEV(accel_dev),
337				"Failed to restart service %s.\n",
338				service->name);
339	}
340	return 0;
341}
342