Lines Matching refs:wq
18 static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
41 static void free_hw_descs(struct idxd_wq *wq)
45 for (i = 0; i < wq->num_descs; i++)
46 kfree(wq->hw_descs[i]);
48 kfree(wq->hw_descs);
51 static int alloc_hw_descs(struct idxd_wq *wq, int num)
53 struct device *dev = &wq->idxd->pdev->dev;
57 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
59 if (!wq->hw_descs)
63 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
65 if (!wq->hw_descs[i]) {
66 free_hw_descs(wq);
74 static void free_descs(struct idxd_wq *wq)
78 for (i = 0; i < wq->num_descs; i++)
79 kfree(wq->descs[i]);
81 kfree(wq->descs);
84 static int alloc_descs(struct idxd_wq *wq, int num)
86 struct device *dev = &wq->idxd->pdev->dev;
90 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
92 if (!wq->descs)
96 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
98 if (!wq->descs[i]) {
99 free_descs(wq);
108 int idxd_wq_alloc_resources(struct idxd_wq *wq)
110 struct idxd_device *idxd = wq->idxd;
114 if (wq->type != IDXD_WQT_KERNEL)
117 num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;
118 wq->num_descs = num_descs;
120 rc = alloc_hw_descs(wq, num_descs);
124 wq->compls_size = num_descs * idxd->data->compl_size;
125 wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL);
126 if (!wq->compls) {
131 rc = alloc_descs(wq, num_descs);
135 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
141 struct idxd_desc *desc = wq->descs[i];
143 desc->hw = wq->hw_descs[i];
145 desc->completion = &wq->compls[i];
147 desc->iax_completion = &wq->iax_compls[i];
148 desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;
150 desc->wq = wq;
157 free_descs(wq);
159 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
161 free_hw_descs(wq);
165 void idxd_wq_free_resources(struct idxd_wq *wq)
167 struct device *dev = &wq->idxd->pdev->dev;
169 if (wq->type != IDXD_WQT_KERNEL)
172 free_hw_descs(wq);
173 free_descs(wq);
174 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
175 sbitmap_queue_free(&wq->sbq);
178 int idxd_wq_enable(struct idxd_wq *wq)
180 struct idxd_device *idxd = wq->idxd;
184 if (wq->state == IDXD_WQ_ENABLED) {
185 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
189 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
197 wq->state = IDXD_WQ_ENABLED;
198 set_bit(wq->id, idxd->wq_enable_map);
199 dev_dbg(dev, "WQ %d enabled\n", wq->id);
203 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)
205 struct idxd_device *idxd = wq->idxd;
209 dev_dbg(dev, "Disabling WQ %d\n", wq->id);
211 if (wq->state != IDXD_WQ_ENABLED) {
212 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
216 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
225 idxd_wq_disable_cleanup(wq);
226 clear_bit(wq->id, idxd->wq_enable_map);
227 wq->state = IDXD_WQ_DISABLED;
228 dev_dbg(dev, "WQ %d disabled\n", wq->id);
232 void idxd_wq_drain(struct idxd_wq *wq)
234 struct idxd_device *idxd = wq->idxd;
238 if (wq->state != IDXD_WQ_ENABLED) {
239 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
243 dev_dbg(dev, "Draining WQ %d\n", wq->id);
244 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
248 void idxd_wq_reset(struct idxd_wq *wq)
250 struct idxd_device *idxd = wq->idxd;
254 if (wq->state != IDXD_WQ_ENABLED) {
255 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
259 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
261 idxd_wq_disable_cleanup(wq);
264 int idxd_wq_map_portal(struct idxd_wq *wq)
266 struct idxd_device *idxd = wq->idxd;
272 start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
274 wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
275 if (!wq->portal)
281 void idxd_wq_unmap_portal(struct idxd_wq *wq)
283 struct device *dev = &wq->idxd->pdev->dev;
285 devm_iounmap(dev, wq->portal);
286 wq->portal = NULL;
287 wq->portal_offset = 0;
295 struct idxd_wq *wq = idxd->wqs[i];
297 if (wq->portal)
298 idxd_wq_unmap_portal(wq);
302 static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)
304 struct idxd_device *idxd = wq->idxd;
308 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
313 wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];
318 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
322 rc = idxd_wq_disable(wq, false);
326 __idxd_wq_set_pasid_locked(wq, pasid);
328 rc = idxd_wq_enable(wq);
335 int idxd_wq_disable_pasid(struct idxd_wq *wq)
337 struct idxd_device *idxd = wq->idxd;
342 rc = idxd_wq_disable(wq, false);
346 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
354 rc = idxd_wq_enable(wq);
361 static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
363 struct idxd_device *idxd = wq->idxd;
365 lockdep_assert_held(&wq->wq_lock);
366 wq->state = IDXD_WQ_DISABLED;
367 memset(wq->wqcfg, 0, idxd->wqcfg_size);
368 wq->type = IDXD_WQT_NONE;
369 wq->threshold = 0;
370 wq->priority = 0;
371 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
372 wq->flags = 0;
373 memset(wq->name, 0, WQ_NAME_SIZE);
374 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
375 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH);
376 if (wq->opcap_bmap)
377 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
380 static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)
382 lockdep_assert_held(&wq->wq_lock);
384 wq->size = 0;
385 wq->group = NULL;
390 struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);
392 complete(&wq->wq_dead);
395 int idxd_wq_init_percpu_ref(struct idxd_wq *wq)
399 memset(&wq->wq_active, 0, sizeof(wq->wq_active));
400 rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release,
404 reinit_completion(&wq->wq_dead);
405 reinit_completion(&wq->wq_resurrect);
409 void __idxd_wq_quiesce(struct idxd_wq *wq)
411 lockdep_assert_held(&wq->wq_lock);
412 reinit_completion(&wq->wq_resurrect);
413 percpu_ref_kill(&wq->wq_active);
414 complete_all(&wq->wq_resurrect);
415 wait_for_completion(&wq->wq_dead);
418 void idxd_wq_quiesce(struct idxd_wq *wq)
420 mutex_lock(&wq->wq_lock);
421 __idxd_wq_quiesce(wq);
422 mutex_unlock(&wq->wq_lock);
709 struct idxd_wq *wq = idxd->wqs[i];
711 mutex_lock(&wq->wq_lock);
712 idxd_wq_disable_cleanup(wq);
713 idxd_wq_device_reset_cleanup(wq);
714 mutex_unlock(&wq->wq_lock);
723 * Clearing wq state is protected by wq lock.
852 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
906 static int idxd_wq_config_write(struct idxd_wq *wq)
908 struct idxd_device *idxd = wq->idxd;
913 if (!wq->group)
918 * wq reset. This will copy back the sticky values that are present on some devices.
921 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
922 wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);
925 if (wq->size == 0 && wq->type != IDXD_WQT_NONE)
926 wq->size = WQ_DEFAULT_QUEUE_DEPTH;
929 wq->wqcfg->wq_size = wq->size;
932 wq->wqcfg->wq_thresh = wq->threshold;
935 if (wq_dedicated(wq))
936 wq->wqcfg->mode = 1;
949 if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&
951 wq->type == IDXD_WQT_KERNEL) {
956 wq->wqcfg->priority = wq->priority;
959 test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags) &&
960 !test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags))
961 wq->wqcfg->bof = 1;
964 wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
967 wq->wqcfg->wq_prs_disable = test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags);
970 wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
971 idxd_wqcfg_set_max_batch_shift(idxd->data->type, wq->wqcfg, ilog2(wq->max_batch_size));
974 if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) {
975 memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8);
976 for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) {
980 wq->wqcfg->op_config[idx] |= BIT(pos);
984 dev_dbg(dev, "WQ %d CFGs\n", wq->id);
986 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
987 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);
989 wq->id, i, wq_offset,
1001 struct idxd_wq *wq = idxd->wqs[i];
1003 rc = idxd_wq_config_write(wq);
1065 struct idxd_wq *wq;
1077 wq = idxd->wqs[i];
1078 group = wq->group;
1080 if (!wq->group)
1083 if (wq_shared(wq) && !wq_shared_supported(wq)) {
1085 dev_warn(dev, "No shared wq support but configured.\n");
1089 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
1127 static int idxd_wq_load_config(struct idxd_wq *wq)
1129 struct idxd_device *idxd = wq->idxd;
1134 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);
1135 memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);
1137 wq->size = wq->wqcfg->wq_size;
1138 wq->threshold = wq->wqcfg->wq_thresh;
1141 if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)
1144 set_bit(WQ_FLAG_DEDICATED, &wq->flags);
1146 wq->priority = wq->wqcfg->priority;
1148 wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;
1149 idxd_wq_set_max_batch_size(idxd->data->type, wq, 1U << wq->wqcfg->max_batch_shift);
1152 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
1153 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
1170 struct idxd_wq *wq;
1174 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
1180 /* Iterate through all 64 bits and check for wq set */
1188 /* Set group assignment for wq if wq bit is set */
1190 wq = idxd->wqs[id];
1191 wq->group = group;
1234 struct idxd_wq *wq = idxd->wqs[i];
1236 rc = idxd_wq_load_config(wq);
1268 * wq is being disabled. Any remaining descriptors are
1300 void idxd_wq_free_irq(struct idxd_wq *wq)
1302 struct idxd_device *idxd = wq->idxd;
1303 struct idxd_irq_entry *ie = &wq->ie;
1305 if (wq->type != IDXD_WQT_KERNEL)
1318 int idxd_wq_request_irq(struct idxd_wq *wq)
1320 struct idxd_device *idxd = wq->idxd;
1326 if (wq->type != IDXD_WQT_KERNEL)
1329 ie = &wq->ie;
1360 int drv_enable_wq(struct idxd_wq *wq)
1362 struct idxd_device *idxd = wq->idxd;
1366 lockdep_assert_held(&wq->wq_lock);
1373 if (wq->state != IDXD_WQ_DISABLED) {
1374 dev_dbg(dev, "wq %d already enabled.\n", wq->id);
1380 if (!wq->group) {
1381 dev_dbg(dev, "wq %d not attached to group.\n", wq->id);
1386 if (strlen(wq->name) == 0) {
1388 dev_dbg(dev, "wq %d name not set.\n", wq->id);
1393 if (wq_shared(wq)) {
1394 if (!wq_shared_supported(wq)) {
1396 dev_dbg(dev, "PASID not enabled and shared wq.\n");
1400 * Shared wq with the threshold set to 0 means the user
1402 * dedicated wq but did not set threshold. A value
1403 * of 0 would effectively disable the shared wq. The
1407 if (wq->threshold == 0) {
1409 dev_dbg(dev, "Shared wq and threshold 0.\n");
1419 * A dedicated wq that is not 'kernel' type will configure pasid and
1423 if (wq_pasid_enabled(wq)) {
1424 if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {
1425 u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;
1427 __idxd_wq_set_pasid_locked(wq, pasid);
1438 dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);
1442 rc = idxd_wq_enable(wq);
1444 dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);
1448 rc = idxd_wq_map_portal(wq);
1451 dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);
1455 wq->client_count = 0;
1457 rc = idxd_wq_request_irq(wq);
1460 dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
1464 rc = idxd_wq_alloc_resources(wq);
1471 rc = idxd_wq_init_percpu_ref(wq);
1481 idxd_wq_free_resources(wq);
1483 idxd_wq_free_irq(wq);
1485 idxd_wq_unmap_portal(wq);
1487 if (idxd_wq_disable(wq, false))
1488 dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));
1493 void drv_disable_wq(struct idxd_wq *wq)
1495 struct idxd_device *idxd = wq->idxd;
1498 lockdep_assert_held(&wq->wq_lock);
1500 if (idxd_wq_refcount(wq))
1501 dev_warn(dev, "Clients has claim on wq %d: %d\n",
1502 wq->id, idxd_wq_refcount(wq));
1504 idxd_wq_unmap_portal(wq);
1505 idxd_wq_drain(wq);
1506 idxd_wq_free_irq(wq);
1507 idxd_wq_reset(wq);
1508 idxd_wq_free_resources(wq);
1509 percpu_ref_exit(&wq->wq_active);
1510 wq->type = IDXD_WQT_NONE;
1511 wq->client_count = 0;
1579 struct idxd_wq *wq = idxd->wqs[i];
1580 struct device *wq_dev = wq_confdev(wq);
1582 if (wq->state == IDXD_WQ_DISABLED)
1584 dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev));