Lines Matching refs:afu
21 static int afu_control(struct cxl_afu *afu, u64 command, u64 clear,
28 spin_lock(&afu->afu_cntl_lock);
31 trace_cxl_afu_ctrl(afu, command);
33 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
34 cxl_p2n_write(afu, CXL_AFU_Cntl_An, (AFU_Cntl & ~clear) | command);
36 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
39 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
44 if (!cxl_ops->link_ok(afu->adapter, afu)) {
45 afu->enabled = enabled;
53 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
62 cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl & ~CXL_AFU_Cntl_An_RA);
66 afu->enabled = enabled;
68 trace_cxl_afu_ctrl_done(afu, command, rc);
69 spin_unlock(&afu->afu_cntl_lock);
74 static int afu_enable(struct cxl_afu *afu)
78 return afu_control(afu, CXL_AFU_Cntl_An_E, 0,
83 int cxl_afu_disable(struct cxl_afu *afu)
87 return afu_control(afu, 0, CXL_AFU_Cntl_An_E,
93 static int native_afu_reset(struct cxl_afu *afu)
100 rc = afu_control(afu, CXL_AFU_Cntl_An_RA, 0,
110 if (afu->current_mode == 0) {
111 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
113 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
119 static int native_afu_check_and_enable(struct cxl_afu *afu)
121 if (!cxl_ops->link_ok(afu->adapter, afu)) {
122 WARN(1, "Refusing to enable afu while link down!\n");
125 if (afu->enabled)
127 return afu_enable(afu);
130 int cxl_psl_purge(struct cxl_afu *afu)
132 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
133 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
140 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
149 if (!cxl_ops->link_ok(afu->adapter, afu)) {
150 dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n");
157 cxl_afu_disable(afu);
160 cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
163 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
167 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
171 if (!cxl_ops->link_ok(afu->adapter, afu)) {
176 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
181 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
182 dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n",
184 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
186 dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n",
188 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
192 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
197 cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
200 trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc);
222 static int cxl_alloc_spa(struct cxl_afu *afu, int mode)
227 afu->native->spa_order = -1;
229 afu->native->spa_order++;
230 spa_size = (1 << afu->native->spa_order) * PAGE_SIZE;
233 dev_warn(&afu->dev, "num_of_processes too large for the SPA, limiting to %i (0x%x)\n",
234 afu->native->spa_max_procs, afu->native->spa_size);
236 afu->num_procs = afu->native->spa_max_procs;
240 afu->native->spa_size = spa_size;
241 afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size);
242 } while (afu->native->spa_max_procs < afu->num_procs);
244 if (!(afu->native->spa = (struct cxl_process_element *)
245 __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) {
249 pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n",
250 1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs);
255 static void attach_spa(struct cxl_afu *afu)
259 afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa +
260 ((afu->native->spa_max_procs + 3) * 128));
262 spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr;
263 spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
266 afu->native->spa, afu->native->spa_max_procs,
267 afu->native->sw_command_status, spap);
268 cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
271 static inline void detach_spa(struct cxl_afu *afu)
273 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);
276 void cxl_release_spa(struct cxl_afu *afu)
278 if (afu->native->spa) {
279 free_pages((unsigned long) afu->native->spa,
280 afu->native->spa_order);
281 afu->native->spa = NULL;
387 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
392 cxl_p2n_write(afu, CXL_SSTP1_An, 0);
395 if ((rc = cxl_afu_slbia(afu)))
399 cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);
402 cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);
410 struct cxl *adapter = ctx->afu->adapter;
413 WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex));
439 WARN_ON(!ctx->afu->enabled);
443 *(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
445 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
448 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
452 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
453 dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n");
457 state = be64_to_cpup(ctx->afu->native->sw_command_status);
485 mutex_lock(&ctx->afu->native->spa_mutex);
490 mutex_unlock(&ctx->afu->native->spa_mutex);
502 mutex_lock(&ctx->afu->native->spa_mutex);
508 if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
513 mutex_unlock(&ctx->afu->native->spa_mutex);
521 mutex_lock(&ctx->afu->native->spa_mutex);
527 if (cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
535 mutex_unlock(&ctx->afu->native->spa_mutex);
542 if (!ctx->afu->pp_size || ctx->master) {
543 ctx->psn_phys = ctx->afu->psn_phys;
544 ctx->psn_size = ctx->afu->adapter->ps_size;
546 ctx->psn_phys = ctx->afu->psn_phys +
547 (ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe);
548 ctx->psn_size = ctx->afu->pp_size;
552 static int activate_afu_directed(struct cxl_afu *afu)
556 dev_info(&afu->dev, "Activating AFU directed mode\n");
558 afu->num_procs = afu->max_procs_virtualised;
559 if (afu->native->spa == NULL) {
560 if (cxl_alloc_spa(afu, CXL_MODE_DIRECTED))
563 attach_spa(afu);
565 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
567 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
568 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
570 afu->current_mode = CXL_MODE_DIRECTED;
572 if ((rc = cxl_chardev_m_afu_add(afu)))
575 if ((rc = cxl_sysfs_afu_m_add(afu)))
578 if ((rc = cxl_chardev_s_afu_add(afu)))
583 cxl_sysfs_afu_m_remove(afu);
585 cxl_chardev_afu_remove(afu);
707 ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
729 result = cxl_ops->afu_check_and_enable(ctx->afu);
770 ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
780 if ((result = cxl_ops->afu_check_and_enable(ctx->afu)))
786 static int deactivate_afu_directed(struct cxl_afu *afu)
788 dev_info(&afu->dev, "Deactivating AFU directed mode\n");
790 afu->current_mode = 0;
791 afu->num_procs = 0;
793 cxl_sysfs_afu_m_remove(afu);
794 cxl_chardev_afu_remove(afu);
821 if (afu->adapter->native->sl_ops->needs_reset_before_disable)
822 cxl_ops->afu_reset(afu);
823 cxl_afu_disable(afu);
824 cxl_psl_purge(afu);
829 int cxl_activate_dedicated_process_psl9(struct cxl_afu *afu)
831 dev_info(&afu->dev, "Activating dedicated process mode\n");
840 afu->num_procs = 1;
841 if (afu->native->spa == NULL) {
842 if (cxl_alloc_spa(afu, CXL_MODE_DEDICATED))
845 attach_spa(afu);
847 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
848 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
850 afu->current_mode = CXL_MODE_DEDICATED;
852 return cxl_chardev_d_afu_add(afu);
855 int cxl_activate_dedicated_process_psl8(struct cxl_afu *afu)
857 dev_info(&afu->dev, "Activating dedicated process mode\n");
859 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
861 cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
862 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */
863 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
864 cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
865 cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */
866 cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));
868 cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */
869 cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */
870 cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */
872 afu->current_mode = CXL_MODE_DEDICATED;
873 afu->num_procs = 1;
875 return cxl_chardev_d_afu_add(afu);
890 struct cxl_afu *afu = ctx->afu;
892 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
897 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
906 struct cxl_afu *afu = ctx->afu;
914 if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)
915 afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
925 result = cxl_ops->afu_reset(afu);
929 return afu_enable(afu);
934 struct cxl_afu *afu = ctx->afu;
941 cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid);
943 cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx));
945 if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
950 if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)
951 afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
953 cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);
958 if ((rc = cxl_ops->afu_reset(afu)))
961 cxl_p2n_write(afu, CXL_PSL_WED_An, wed);
963 return afu_enable(afu);
966 static int deactivate_dedicated_process(struct cxl_afu *afu)
968 dev_info(&afu->dev, "Deactivating dedicated process mode\n");
970 afu->current_mode = 0;
971 afu->num_procs = 0;
973 cxl_chardev_afu_remove(afu);
978 static int native_afu_deactivate_mode(struct cxl_afu *afu, int mode)
981 return deactivate_afu_directed(afu);
983 return deactivate_dedicated_process(afu);
987 static int native_afu_activate_mode(struct cxl_afu *afu, int mode)
991 if (!(mode & afu->modes_supported))
994 if (!cxl_ops->link_ok(afu->adapter, afu)) {
1000 return activate_afu_directed(afu);
1002 (afu->adapter->native->sl_ops->activate_dedicated_process))
1003 return afu->adapter->native->sl_ops->activate_dedicated_process(afu);
1011 if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu)) {
1017 if ((ctx->afu->current_mode == CXL_MODE_DIRECTED) &&
1018 (ctx->afu->adapter->native->sl_ops->attach_afu_directed))
1019 return ctx->afu->adapter->native->sl_ops->attach_afu_directed(ctx, wed, amr);
1021 if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) &&
1022 (ctx->afu->adapter->native->sl_ops->attach_dedicated_process))
1023 return ctx->afu->adapter->native->sl_ops->attach_dedicated_process(ctx, wed, amr);
1033 * optional like we do in the afu directed path). It does not indicate
1046 cxl_ops->afu_reset(ctx->afu);
1047 cxl_afu_disable(ctx->afu);
1048 cxl_psl_purge(ctx->afu);
1054 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
1056 if ((ctx->afu->current_mode == CXL_MODE_DEDICATED) &&
1057 (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes))
1058 return ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);
1078 if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
1084 static int native_get_irq_info(struct cxl_afu *afu, struct cxl_irq_info *info)
1089 if (!cxl_ops->link_ok(afu->adapter, afu))
1092 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
1093 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
1095 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
1096 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
1097 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
1107 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL9_FIR1);
1109 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
1110 if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
1111 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
1112 cxl_afu_decode_psl_serr(ctx->afu, serr);
1120 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
1121 fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
1122 fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
1123 afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);
1125 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%016llx\n", fir1);
1126 dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%016llx\n", fir2);
1127 if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {
1128 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
1129 cxl_afu_decode_psl_serr(ctx->afu, serr);
1131 dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
1132 dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
1139 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%016llx\n", errstat);
1141 if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers)
1142 ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx);
1144 if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) {
1145 dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
1146 ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter);
1152 static bool cxl_is_translation_fault(struct cxl_afu *afu, u64 dsisr)
1163 irqreturn_t cxl_fail_irq_psl(struct cxl_afu *afu, struct cxl_irq_info *irq_info)
1165 if (cxl_is_translation_fault(afu, irq_info->dsisr))
1166 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
1168 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
1175 struct cxl_afu *afu = data;
1178 u64 phreg = cxl_p2n_read(afu, CXL_PSL_PEHandle_An);
1183 dev_warn(&afu->dev,
1190 if ((res = native_get_irq_info(afu, &irq_info))) {
1192 if (afu->adapter->native->sl_ops->fail_irq)
1193 return afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);
1198 ctx = idr_find(&afu->contexts_idr, ph);
1200 if (afu->adapter->native->sl_ops->handle_interrupt)
1201 ret = afu->adapter->native->sl_ops->handle_interrupt(irq, ctx, &irq_info);
1211 if (afu->adapter->native->sl_ops->fail_irq)
1212 ret = afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);
1227 ph = cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) & 0xffff;
1230 dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An);
1244 dev_warn(&ctx->afu->dev, "WARNING: waiting on DSI for PE %i"
1251 struct cxl_afu *afu = data;
1258 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
1259 errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
1260 afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
1261 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
1262 cxl_afu_decode_psl_serr(afu, serr);
1265 fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
1266 afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
1267 dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%016llx\n", fir_slice);
1268 dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%016llx\n", afu_debug);
1270 dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%016llx\n", errstat);
1271 dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
1272 dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
1277 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
1278 dev_info(&afu->dev, "Further such interrupts will be masked until the AFU is reset\n");
1360 int cxl_native_register_serr_irq(struct cxl_afu *afu)
1365 afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
1366 dev_name(&afu->dev));
1367 if (!afu->err_irq_name)
1370 if ((rc = cxl_register_one_irq(afu->adapter, native_slice_irq_err, afu,
1371 &afu->serr_hwirq,
1372 &afu->serr_virq, afu->err_irq_name))) {
1373 kfree(afu->err_irq_name);
1374 afu->err_irq_name = NULL;
1378 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
1380 serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
1386 serr = (serr & ~0xff0000007fffffffULL) | (afu->serr_hwirq & 0xffff);
1388 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
1393 void cxl_native_release_serr_irq(struct cxl_afu *afu)
1395 if (afu->serr_virq == 0 ||
1396 afu->serr_virq != irq_find_mapping(NULL, afu->serr_hwirq))
1399 cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
1400 cxl_unmap_irq(afu->serr_virq, afu);
1401 cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
1402 kfree(afu->err_irq_name);
1403 afu->serr_virq = 0;
1406 int cxl_native_register_psl_irq(struct cxl_afu *afu)
1410 afu->psl_irq_name = kasprintf(GFP_KERNEL, "cxl-%s",
1411 dev_name(&afu->dev));
1412 if (!afu->psl_irq_name)
1415 if ((rc = cxl_register_one_irq(afu->adapter, native_irq_multiplexed,
1416 afu, &afu->native->psl_hwirq, &afu->native->psl_virq,
1417 afu->psl_irq_name))) {
1418 kfree(afu->psl_irq_name);
1419 afu->psl_irq_name = NULL;
1424 void cxl_native_release_psl_irq(struct cxl_afu *afu)
1426 if (afu->native->psl_virq == 0 ||
1427 afu->native->psl_virq !=
1428 irq_find_mapping(NULL, afu->native->psl_hwirq))
1431 cxl_unmap_irq(afu->native->psl_virq, afu);
1432 cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq);
1433 kfree(afu->psl_irq_name);
1434 afu->native->psl_virq = 0;
1437 static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
1444 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
1445 cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);
1448 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
1455 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
1457 recover_psl_err(ctx->afu, psl_reset_mask);
1462 int cxl_check_error(struct cxl_afu *afu)
1464 return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
1473 static int native_afu_cr_read64(struct cxl_afu *afu, int cr, u64 off, u64 *out)
1475 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1477 if (unlikely(off >= afu->crs_len))
1479 *out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset +
1480 (cr * afu->crs_len) + off);
1484 static int native_afu_cr_read32(struct cxl_afu *afu, int cr, u64 off, u32 *out)
1486 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1488 if (unlikely(off >= afu->crs_len))
1490 *out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1491 (cr * afu->crs_len) + off);
1495 static int native_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off, u16 *out)
1501 rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1507 static int native_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off, u8 *out)
1513 rc = native_afu_cr_read32(afu, cr, aligned_off, &val);
1519 static int native_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
1521 if (unlikely(!cxl_ops->link_ok(afu->adapter, afu)))
1523 if (unlikely(off >= afu->crs_len))
1525 out_le32(afu->native->afu_desc_mmio + afu->crs_offset +
1526 (cr * afu->crs_len) + off, in);
1530 static int native_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
1536 rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
1544 rc = native_afu_cr_write32(afu, cr, aligned_off, val32);
1548 static int native_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
1554 rc = native_afu_cr_read32(afu, cr, aligned_off, &val32);
1561 rc = native_afu_cr_write32(afu, cr, aligned_off, val32);