1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8#include <linux/clk.h> 9#include <linux/device.h> 10#include <linux/dma-mapping.h> 11#include <linux/dmapool.h> 12#include <linux/firmware.h> 13#include <linux/interrupt.h> 14#include <linux/module.h> 15#include <linux/of_platform.h> 16#include <linux/of_irq.h> 17#include <linux/pci.h> 18#include <linux/platform_device.h> 19#include <linux/workqueue.h> 20 21#include <crypto/internal/aead.h> 22#include <crypto/internal/hash.h> 23#include <crypto/internal/skcipher.h> 24 25#include "safexcel.h" 26 27static u32 max_rings = EIP197_MAX_RINGS; 28module_param(max_rings, uint, 0644); 29MODULE_PARM_DESC(max_rings, "Maximum number of rings to use."); 30 31static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv) 32{ 33 int i; 34 35 /* 36 * Map all interfaces/rings to register index 0 37 * so they can share contexts. Without this, the EIP197 will 38 * assume each interface/ring to be in its own memory domain 39 * i.e. have its own subset of UNIQUE memory addresses. 40 * Which would cause records with the SAME memory address to 41 * use DIFFERENT cache buffers, causing both poor cache utilization 42 * AND serious coherence/invalidation issues. 43 */ 44 for (i = 0; i < 4; i++) 45 writel(0, priv->base + EIP197_FLUE_IFC_LUT(i)); 46 47 /* 48 * Initialize other virtualization regs for cache 49 * These may not be in their reset state ... 50 */ 51 for (i = 0; i < priv->config.rings; i++) { 52 writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i)); 53 writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i)); 54 writel(EIP197_FLUE_CONFIG_MAGIC, 55 priv->base + EIP197_FLUE_CONFIG(i)); 56 } 57 writel(0, priv->base + EIP197_FLUE_OFFSETS); 58 writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET); 59} 60 61static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv, 62 u32 addrmid, int *actbank) 63{ 64 u32 val; 65 int curbank; 66 67 curbank = addrmid >> 16; 68 if (curbank != *actbank) { 69 val = readl(priv->base + EIP197_CS_RAM_CTRL); 70 val = (val & ~EIP197_CS_BANKSEL_MASK) | 71 (curbank << EIP197_CS_BANKSEL_OFS); 72 writel(val, priv->base + EIP197_CS_RAM_CTRL); 73 *actbank = curbank; 74 } 75} 76 77static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv, 78 int maxbanks, u32 probemask, u32 stride) 79{ 80 u32 val, addrhi, addrlo, addrmid, addralias, delta, marker; 81 int actbank; 82 83 /* 84 * And probe the actual size of the physically attached cache data RAM 85 * Using a binary subdivision algorithm downto 32 byte cache lines. 86 */ 87 addrhi = 1 << (16 + maxbanks); 88 addrlo = 0; 89 actbank = min(maxbanks - 1, 0); 90 while ((addrhi - addrlo) > stride) { 91 /* write marker to lowest address in top half */ 92 addrmid = (addrhi + addrlo) >> 1; 93 marker = (addrmid ^ 0xabadbabe) & probemask; /* Unique */ 94 eip197_trc_cache_banksel(priv, addrmid, &actbank); 95 writel(marker, 96 priv->base + EIP197_CLASSIFICATION_RAMS + 97 (addrmid & 0xffff)); 98 99 /* write invalid markers to possible aliases */ 100 delta = 1 << __fls(addrmid); 101 while (delta >= stride) { 102 addralias = addrmid - delta; 103 eip197_trc_cache_banksel(priv, addralias, &actbank); 104 writel(~marker, 105 priv->base + EIP197_CLASSIFICATION_RAMS + 106 (addralias & 0xffff)); 107 delta >>= 1; 108 } 109 110 /* read back marker from top half */ 111 eip197_trc_cache_banksel(priv, addrmid, &actbank); 112 val = readl(priv->base + EIP197_CLASSIFICATION_RAMS + 113 (addrmid & 0xffff)); 114 115 if ((val & probemask) == marker) 116 /* read back correct, continue with top half */ 117 addrlo = addrmid; 118 else 119 /* not read back correct, continue with bottom half */ 120 addrhi = addrmid; 121 } 122 return addrhi; 123} 124 125static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv, 126 int cs_rc_max, int cs_ht_wc) 127{ 128 int i; 129 u32 htable_offset, val, offset; 130 131 /* Clear all records in administration RAM */ 132 for (i = 0; i < cs_rc_max; i++) { 133 offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE; 134 135 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) | 136 EIP197_CS_RC_PREV(EIP197_RC_NULL), 137 priv->base + offset); 138 139 val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1); 140 if (i == 0) 141 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL); 142 else if (i == cs_rc_max - 1) 143 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL); 144 writel(val, priv->base + offset + 4); 145 /* must also initialize the address key due to ECC! */ 146 writel(0, priv->base + offset + 8); 147 writel(0, priv->base + offset + 12); 148 } 149 150 /* Clear the hash table entries */ 151 htable_offset = cs_rc_max * EIP197_CS_RC_SIZE; 152 for (i = 0; i < cs_ht_wc; i++) 153 writel(GENMASK(29, 0), 154 priv->base + EIP197_CLASSIFICATION_RAMS + 155 htable_offset + i * sizeof(u32)); 156} 157 158static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv) 159{ 160 u32 val, dsize, asize; 161 int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc; 162 int cs_rc_abs_max, cs_ht_sz; 163 int maxbanks; 164 165 /* Setup (dummy) virtualization for cache */ 166 eip197_trc_cache_setupvirt(priv); 167 168 /* 169 * Enable the record cache memory access and 170 * probe the bank select width 171 */ 172 val = readl(priv->base + EIP197_CS_RAM_CTRL); 173 val &= ~EIP197_TRC_ENABLE_MASK; 174 val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK; 175 writel(val, priv->base + EIP197_CS_RAM_CTRL); 176 val = readl(priv->base + EIP197_CS_RAM_CTRL); 177 maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1; 178 179 /* Clear all ECC errors */ 180 writel(0, priv->base + EIP197_TRC_ECCCTRL); 181 182 /* 183 * Make sure the cache memory is accessible by taking record cache into 184 * reset. Need data memory access here, not admin access. 185 */ 186 val = readl(priv->base + EIP197_TRC_PARAMS); 187 val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS; 188 writel(val, priv->base + EIP197_TRC_PARAMS); 189 190 /* Probed data RAM size in bytes */ 191 dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32); 192 193 /* 194 * Now probe the administration RAM size pretty much the same way 195 * Except that only the lower 30 bits are writable and we don't need 196 * bank selects 197 */ 198 val = readl(priv->base + EIP197_TRC_PARAMS); 199 /* admin access now */ 200 val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK); 201 writel(val, priv->base + EIP197_TRC_PARAMS); 202 203 /* Probed admin RAM size in admin words */ 204 asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4; 205 206 /* Clear any ECC errors detected while probing! */ 207 writel(0, priv->base + EIP197_TRC_ECCCTRL); 208 209 /* Sanity check probing results */ 210 if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) { 211 dev_err(priv->dev, "Record cache probing failed (%d,%d).", 212 dsize, asize); 213 return -ENODEV; 214 } 215 216 /* 217 * Determine optimal configuration from RAM sizes 218 * Note that we assume that the physical RAM configuration is sane 219 * Therefore, we don't do any parameter error checking here ... 220 */ 221 222 /* For now, just use a single record format covering everything */ 223 cs_trc_rec_wc = EIP197_CS_TRC_REC_WC; 224 cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC; 225 226 /* 227 * Step #1: How many records will physically fit? 228 * Hard upper limit is 1023! 229 */ 230 cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023); 231 /* Step #2: Need at least 2 words in the admin RAM per record */ 232 cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1)); 233 /* Step #3: Determine log2 of hash table size */ 234 cs_ht_sz = __fls(asize - cs_rc_max) - 2; 235 /* Step #4: determine current size of hash table in dwords */ 236 cs_ht_wc = 16 << cs_ht_sz; /* dwords, not admin words */ 237 /* Step #5: add back excess words and see if we can fit more records */ 238 cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2)); 239 240 /* Clear the cache RAMs */ 241 eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc); 242 243 /* Disable the record cache memory access */ 244 val = readl(priv->base + EIP197_CS_RAM_CTRL); 245 val &= ~EIP197_TRC_ENABLE_MASK; 246 writel(val, priv->base + EIP197_CS_RAM_CTRL); 247 248 /* Write head and tail pointers of the record free chain */ 249 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) | 250 EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1); 251 writel(val, priv->base + EIP197_TRC_FREECHAIN); 252 253 /* Configure the record cache #1 */ 254 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) | 255 EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max); 256 writel(val, priv->base + EIP197_TRC_PARAMS2); 257 258 /* Configure the record cache #2 */ 259 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) | 260 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) | 261 EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz); 262 writel(val, priv->base + EIP197_TRC_PARAMS); 263 264 dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n", 265 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc); 266 return 0; 267} 268 269static void eip197_init_firmware(struct safexcel_crypto_priv *priv) 270{ 271 int pe, i; 272 u32 val; 273 274 for (pe = 0; pe < priv->config.pes; pe++) { 275 /* Configure the token FIFO's */ 276 writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe)); 277 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe)); 278 279 /* Clear the ICE scratchpad memory */ 280 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); 281 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER | 282 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN | 283 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS | 284 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS; 285 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe)); 286 287 /* clear the scratchpad RAM using 32 bit writes only */ 288 for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++) 289 writel(0, EIP197_PE(priv) + 290 EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2)); 291 292 /* Reset the IFPP engine to make its program mem accessible */ 293 writel(EIP197_PE_ICE_x_CTRL_SW_RESET | 294 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | 295 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, 296 EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe)); 297 298 /* Reset the IPUE engine to make its program mem accessible */ 299 writel(EIP197_PE_ICE_x_CTRL_SW_RESET | 300 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR | 301 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR, 302 EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe)); 303 304 /* Enable access to all IFPP program memories */ 305 writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN, 306 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); 307 308 /* bypass the OCE, if present */ 309 if (priv->flags & EIP197_OCE) 310 writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) + 311 EIP197_PE_DEBUG(pe)); 312 } 313 314} 315 316static int eip197_write_firmware(struct safexcel_crypto_priv *priv, 317 const struct firmware *fw) 318{ 319 const __be32 *data = (const __be32 *)fw->data; 320 int i; 321 322 /* Write the firmware */ 323 for (i = 0; i < fw->size / sizeof(u32); i++) 324 writel(be32_to_cpu(data[i]), 325 priv->base + EIP197_CLASSIFICATION_RAMS + 326 i * sizeof(__be32)); 327 328 /* Exclude final 2 NOPs from size */ 329 return i - EIP197_FW_TERMINAL_NOPS; 330} 331 332/* 333 * If FW is actual production firmware, then poll for its initialization 334 * to complete and check if it is good for the HW, otherwise just return OK. 335 */ 336static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp) 337{ 338 int pe, pollcnt; 339 u32 base, pollofs; 340 341 if (fpp) 342 pollofs = EIP197_FW_FPP_READY; 343 else 344 pollofs = EIP197_FW_PUE_READY; 345 346 for (pe = 0; pe < priv->config.pes; pe++) { 347 base = EIP197_PE_ICE_SCRATCH_RAM(pe); 348 pollcnt = EIP197_FW_START_POLLCNT; 349 while (pollcnt && 350 (readl_relaxed(EIP197_PE(priv) + base + 351 pollofs) != 1)) { 352 pollcnt--; 353 } 354 if (!pollcnt) { 355 dev_err(priv->dev, "FW(%d) for PE %d failed to start\n", 356 fpp, pe); 357 return false; 358 } 359 } 360 return true; 361} 362 363static bool eip197_start_firmware(struct safexcel_crypto_priv *priv, 364 int ipuesz, int ifppsz, int minifw) 365{ 366 int pe; 367 u32 val; 368 369 for (pe = 0; pe < priv->config.pes; pe++) { 370 /* Disable access to all program memory */ 371 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); 372 373 /* Start IFPP microengines */ 374 if (minifw) 375 val = 0; 376 else 377 val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) & 378 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) | 379 EIP197_PE_ICE_UENG_DEBUG_RESET; 380 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe)); 381 382 /* Start IPUE microengines */ 383 if (minifw) 384 val = 0; 385 else 386 val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) & 387 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) | 388 EIP197_PE_ICE_UENG_DEBUG_RESET; 389 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe)); 390 } 391 392 /* For miniFW startup, there is no initialization, so always succeed */ 393 if (minifw) 394 return true; 395 396 /* Wait until all the firmwares have properly started up */ 397 if (!poll_fw_ready(priv, 1)) 398 return false; 399 if (!poll_fw_ready(priv, 0)) 400 return false; 401 402 return true; 403} 404 405static int eip197_load_firmwares(struct safexcel_crypto_priv *priv) 406{ 407 const char *fw_name[] = {"ifpp.bin", "ipue.bin"}; 408 const struct firmware *fw[FW_NB]; 409 char fw_path[37], *dir = NULL; 410 int i, j, ret = 0, pe; 411 int ipuesz, ifppsz, minifw = 0; 412 413 if (priv->version == EIP197D_MRVL) 414 dir = "eip197d"; 415 else if (priv->version == EIP197B_MRVL || 416 priv->version == EIP197_DEVBRD) 417 dir = "eip197b"; 418 else 419 return -ENODEV; 420 421retry_fw: 422 for (i = 0; i < FW_NB; i++) { 423 snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]); 424 ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev); 425 if (ret) { 426 if (minifw || priv->version != EIP197B_MRVL) 427 goto release_fw; 428 429 /* Fallback to the old firmware location for the 430 * EIP197b. 431 */ 432 ret = firmware_request_nowarn(&fw[i], fw_name[i], 433 priv->dev); 434 if (ret) 435 goto release_fw; 436 } 437 } 438 439 eip197_init_firmware(priv); 440 441 ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]); 442 443 /* Enable access to IPUE program memories */ 444 for (pe = 0; pe < priv->config.pes; pe++) 445 writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN, 446 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe)); 447 448 ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]); 449 450 if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) { 451 dev_dbg(priv->dev, "Firmware loaded successfully\n"); 452 return 0; 453 } 454 455 ret = -ENODEV; 456 457release_fw: 458 for (j = 0; j < i; j++) 459 release_firmware(fw[j]); 460 461 if (!minifw) { 462 /* Retry with minifw path */ 463 dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n"); 464 dir = "eip197_minifw"; 465 minifw = 1; 466 goto retry_fw; 467 } 468 469 dev_dbg(priv->dev, "Firmware load failed.\n"); 470 471 return ret; 472} 473 474static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv) 475{ 476 u32 cd_size_rnd, val; 477 int i, cd_fetch_cnt; 478 479 cd_size_rnd = (priv->config.cd_size + 480 (BIT(priv->hwconfig.hwdataw) - 1)) >> 481 priv->hwconfig.hwdataw; 482 /* determine number of CD's we can fetch into the CD FIFO as 1 block */ 483 if (priv->flags & SAFEXCEL_HW_EIP197) { 484 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */ 485 cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd; 486 cd_fetch_cnt = min_t(uint, cd_fetch_cnt, 487 (priv->config.pes * EIP197_FETCH_DEPTH)); 488 } else { 489 /* for the EIP97, just fetch all that fits minus 1 */ 490 cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) / 491 cd_size_rnd) - 1; 492 } 493 /* 494 * Since we're using command desc's way larger than formally specified, 495 * we need to check whether we can fit even 1 for low-end EIP196's! 496 */ 497 if (!cd_fetch_cnt) { 498 dev_err(priv->dev, "Unable to fit even 1 command desc!\n"); 499 return -ENODEV; 500 } 501 502 for (i = 0; i < priv->config.rings; i++) { 503 /* ring base address */ 504 writel(lower_32_bits(priv->ring[i].cdr.base_dma), 505 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 506 writel(upper_32_bits(priv->ring[i].cdr.base_dma), 507 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 508 509 writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP | 510 (priv->config.cd_offset << 14) | priv->config.cd_size, 511 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); 512 writel(((cd_fetch_cnt * 513 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) | 514 (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))), 515 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); 516 517 /* Configure DMA tx control */ 518 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); 519 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); 520 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); 521 522 /* clear any pending interrupt */ 523 writel(GENMASK(5, 0), 524 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); 525 } 526 527 return 0; 528} 529 530static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv) 531{ 532 u32 rd_size_rnd, val; 533 int i, rd_fetch_cnt; 534 535 /* determine number of RD's we can fetch into the FIFO as one block */ 536 rd_size_rnd = (EIP197_RD64_FETCH_SIZE + 537 (BIT(priv->hwconfig.hwdataw) - 1)) >> 538 priv->hwconfig.hwdataw; 539 if (priv->flags & SAFEXCEL_HW_EIP197) { 540 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */ 541 rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd; 542 rd_fetch_cnt = min_t(uint, rd_fetch_cnt, 543 (priv->config.pes * EIP197_FETCH_DEPTH)); 544 } else { 545 /* for the EIP97, just fetch all that fits minus 1 */ 546 rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) / 547 rd_size_rnd) - 1; 548 } 549 550 for (i = 0; i < priv->config.rings; i++) { 551 /* ring base address */ 552 writel(lower_32_bits(priv->ring[i].rdr.base_dma), 553 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 554 writel(upper_32_bits(priv->ring[i].rdr.base_dma), 555 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 556 557 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) | 558 priv->config.rd_size, 559 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE); 560 561 writel(((rd_fetch_cnt * 562 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) | 563 (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))), 564 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); 565 566 /* Configure DMA tx control */ 567 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS); 568 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS); 569 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF; 570 writel(val, 571 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG); 572 573 /* clear any pending interrupt */ 574 writel(GENMASK(7, 0), 575 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); 576 577 /* enable ring interrupt */ 578 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); 579 val |= EIP197_RDR_IRQ(i); 580 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i)); 581 } 582 583 return 0; 584} 585 586static int safexcel_hw_init(struct safexcel_crypto_priv *priv) 587{ 588 u32 val; 589 int i, ret, pe, opbuflo, opbufhi; 590 591 dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n", 592 priv->config.pes, priv->config.rings); 593 594 /* 595 * For EIP197's only set maximum number of TX commands to 2^5 = 32 596 * Skip for the EIP97 as it does not have this field. 597 */ 598 if (priv->flags & SAFEXCEL_HW_EIP197) { 599 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 600 val |= EIP197_MST_CTRL_TX_MAX_CMD(5); 601 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 602 } 603 604 /* Configure wr/rd cache values */ 605 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | 606 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS), 607 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL); 608 609 /* Interrupts reset */ 610 611 /* Disable all global interrupts */ 612 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL); 613 614 /* Clear any pending interrupt */ 615 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); 616 617 /* Processing Engine configuration */ 618 for (pe = 0; pe < priv->config.pes; pe++) { 619 /* Data Fetch Engine configuration */ 620 621 /* Reset all DFE threads */ 622 writel(EIP197_DxE_THR_CTRL_RESET_PE, 623 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); 624 625 if (priv->flags & EIP197_PE_ARB) 626 /* Reset HIA input interface arbiter (if present) */ 627 writel(EIP197_HIA_RA_PE_CTRL_RESET, 628 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); 629 630 /* DMA transfer size to use */ 631 val = EIP197_HIA_DFE_CFG_DIS_DEBUG; 632 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) | 633 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9); 634 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | 635 EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7); 636 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS); 637 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS); 638 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe)); 639 640 /* Leave the DFE threads reset state */ 641 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); 642 643 /* Configure the processing engine thresholds */ 644 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | 645 EIP197_PE_IN_xBUF_THRES_MAX(9), 646 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe)); 647 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) | 648 EIP197_PE_IN_xBUF_THRES_MAX(7), 649 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe)); 650 651 if (priv->flags & SAFEXCEL_HW_EIP197) 652 /* enable HIA input interface arbiter and rings */ 653 writel(EIP197_HIA_RA_PE_CTRL_EN | 654 GENMASK(priv->config.rings - 1, 0), 655 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe)); 656 657 /* Data Store Engine configuration */ 658 659 /* Reset all DSE threads */ 660 writel(EIP197_DxE_THR_CTRL_RESET_PE, 661 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); 662 663 /* Wait for all DSE threads to complete */ 664 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) & 665 GENMASK(15, 12)) != GENMASK(15, 12)) 666 ; 667 668 /* DMA transfer size to use */ 669 if (priv->hwconfig.hwnumpes > 4) { 670 opbuflo = 9; 671 opbufhi = 10; 672 } else { 673 opbuflo = 7; 674 opbufhi = 8; 675 } 676 val = EIP197_HIA_DSE_CFG_DIS_DEBUG; 677 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) | 678 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi); 679 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS); 680 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE; 681 /* FIXME: instability issues can occur for EIP97 but disabling 682 * it impacts performance. 683 */ 684 if (priv->flags & SAFEXCEL_HW_EIP197) 685 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR; 686 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe)); 687 688 /* Leave the DSE threads reset state */ 689 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); 690 691 /* Configure the procesing engine thresholds */ 692 writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) | 693 EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi), 694 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe)); 695 696 /* Processing Engine configuration */ 697 698 /* Token & context configuration */ 699 val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES | 700 EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT | 701 EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT; 702 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe)); 703 704 /* H/W capabilities selection: just enable everything */ 705 writel(EIP197_FUNCTION_ALL, 706 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe)); 707 writel(EIP197_FUNCTION_ALL, 708 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe)); 709 } 710 711 /* Command Descriptor Rings prepare */ 712 for (i = 0; i < priv->config.rings; i++) { 713 /* Clear interrupts for this ring */ 714 writel(GENMASK(31, 0), 715 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i)); 716 717 /* Disable external triggering */ 718 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG); 719 720 /* Clear the pending prepared counter */ 721 writel(EIP197_xDR_PREP_CLR_COUNT, 722 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); 723 724 /* Clear the pending processed counter */ 725 writel(EIP197_xDR_PROC_CLR_COUNT, 726 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); 727 728 writel(0, 729 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); 730 writel(0, 731 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); 732 733 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset), 734 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); 735 } 736 737 /* Result Descriptor Ring prepare */ 738 for (i = 0; i < priv->config.rings; i++) { 739 /* Disable external triggering*/ 740 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG); 741 742 /* Clear the pending prepared counter */ 743 writel(EIP197_xDR_PREP_CLR_COUNT, 744 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT); 745 746 /* Clear the pending processed counter */ 747 writel(EIP197_xDR_PROC_CLR_COUNT, 748 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT); 749 750 writel(0, 751 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR); 752 writel(0, 753 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR); 754 755 /* Ring size */ 756 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset), 757 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE); 758 } 759 760 for (pe = 0; pe < priv->config.pes; pe++) { 761 /* Enable command descriptor rings */ 762 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), 763 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe)); 764 765 /* Enable result descriptor rings */ 766 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0), 767 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe)); 768 } 769 770 /* Clear any HIA interrupt */ 771 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK); 772 773 if (priv->flags & EIP197_SIMPLE_TRC) { 774 writel(EIP197_STRC_CONFIG_INIT | 775 EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) | 776 EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC), 777 priv->base + EIP197_STRC_CONFIG); 778 writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE, 779 EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0)); 780 } else if (priv->flags & SAFEXCEL_HW_EIP197) { 781 ret = eip197_trc_cache_init(priv); 782 if (ret) 783 return ret; 784 } 785 786 if (priv->flags & EIP197_ICE) { 787 ret = eip197_load_firmwares(priv); 788 if (ret) 789 return ret; 790 } 791 792 return safexcel_hw_setup_cdesc_rings(priv) ?: 793 safexcel_hw_setup_rdesc_rings(priv) ?: 794 0; 795} 796 797/* Called with ring's lock taken */ 798static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv, 799 int ring) 800{ 801 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ); 802 803 if (!coal) 804 return; 805 806 /* Configure when we want an interrupt */ 807 writel(EIP197_HIA_RDR_THRESH_PKT_MODE | 808 EIP197_HIA_RDR_THRESH_PROC_PKT(coal), 809 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH); 810} 811 812void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) 813{ 814 struct crypto_async_request *req, *backlog; 815 struct safexcel_context *ctx; 816 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; 817 818 /* If a request wasn't properly dequeued because of a lack of resources, 819 * proceeded it first, 820 */ 821 req = priv->ring[ring].req; 822 backlog = priv->ring[ring].backlog; 823 if (req) 824 goto handle_req; 825 826 while (true) { 827 spin_lock_bh(&priv->ring[ring].queue_lock); 828 backlog = crypto_get_backlog(&priv->ring[ring].queue); 829 req = crypto_dequeue_request(&priv->ring[ring].queue); 830 spin_unlock_bh(&priv->ring[ring].queue_lock); 831 832 if (!req) { 833 priv->ring[ring].req = NULL; 834 priv->ring[ring].backlog = NULL; 835 goto finalize; 836 } 837 838handle_req: 839 ctx = crypto_tfm_ctx(req->tfm); 840 ret = ctx->send(req, ring, &commands, &results); 841 if (ret) 842 goto request_failed; 843 844 if (backlog) 845 backlog->complete(backlog, -EINPROGRESS); 846 847 /* In case the send() helper did not issue any command to push 848 * to the engine because the input data was cached, continue to 849 * dequeue other requests as this is valid and not an error. 850 */ 851 if (!commands && !results) 852 continue; 853 854 cdesc += commands; 855 rdesc += results; 856 nreq++; 857 } 858 859request_failed: 860 /* Not enough resources to handle all the requests. Bail out and save 861 * the request and the backlog for the next dequeue call (per-ring). 862 */ 863 priv->ring[ring].req = req; 864 priv->ring[ring].backlog = backlog; 865 866finalize: 867 if (!nreq) 868 return; 869 870 spin_lock_bh(&priv->ring[ring].lock); 871 872 priv->ring[ring].requests += nreq; 873 874 if (!priv->ring[ring].busy) { 875 safexcel_try_push_requests(priv, ring); 876 priv->ring[ring].busy = true; 877 } 878 879 spin_unlock_bh(&priv->ring[ring].lock); 880 881 /* let the RDR know we have pending descriptors */ 882 writel((rdesc * priv->config.rd_offset), 883 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); 884 885 /* let the CDR know we have pending descriptors */ 886 writel((cdesc * priv->config.cd_offset), 887 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT); 888} 889 890inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, 891 void *rdp) 892{ 893 struct safexcel_result_desc *rdesc = rdp; 894 struct result_data_desc *result_data = rdp + priv->config.res_offset; 895 896 if (likely((!rdesc->last_seg) || /* Rest only valid if last seg! */ 897 ((!rdesc->descriptor_overflow) && 898 (!rdesc->buffer_overflow) && 899 (!result_data->error_code)))) 900 return 0; 901 902 if (rdesc->descriptor_overflow) 903 dev_err(priv->dev, "Descriptor overflow detected"); 904 905 if (rdesc->buffer_overflow) 906 dev_err(priv->dev, "Buffer overflow detected"); 907 908 if (result_data->error_code & 0x4066) { 909 /* Fatal error (bits 1,2,5,6 & 14) */ 910 dev_err(priv->dev, 911 "result descriptor error (%x)", 912 result_data->error_code); 913 914 return -EIO; 915 } else if (result_data->error_code & 916 (BIT(7) | BIT(4) | BIT(3) | BIT(0))) { 917 /* 918 * Give priority over authentication fails: 919 * Blocksize, length & overflow errors, 920 * something wrong with the input! 921 */ 922 return -EINVAL; 923 } else if (result_data->error_code & BIT(9)) { 924 /* Authentication failed */ 925 return -EBADMSG; 926 } 927 928 /* All other non-fatal errors */ 929 return -EINVAL; 930} 931 932inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv, 933 int ring, 934 struct safexcel_result_desc *rdesc, 935 struct crypto_async_request *req) 936{ 937 int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc); 938 939 priv->ring[ring].rdr_req[i] = req; 940} 941 942inline struct crypto_async_request * 943safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring) 944{ 945 int i = safexcel_ring_first_rdr_index(priv, ring); 946 947 return priv->ring[ring].rdr_req[i]; 948} 949 950void safexcel_complete(struct safexcel_crypto_priv *priv, int ring) 951{ 952 struct safexcel_command_desc *cdesc; 953 954 /* Acknowledge the command descriptors */ 955 do { 956 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr); 957 if (IS_ERR(cdesc)) { 958 dev_err(priv->dev, 959 "Could not retrieve the command descriptor\n"); 960 return; 961 } 962 } while (!cdesc->last_seg); 963} 964 965void safexcel_inv_complete(struct crypto_async_request *req, int error) 966{ 967 struct safexcel_inv_result *result = req->data; 968 969 if (error == -EINPROGRESS) 970 return; 971 972 result->error = error; 973 complete(&result->completion); 974} 975 976int safexcel_invalidate_cache(struct crypto_async_request *async, 977 struct safexcel_crypto_priv *priv, 978 dma_addr_t ctxr_dma, int ring) 979{ 980 struct safexcel_command_desc *cdesc; 981 struct safexcel_result_desc *rdesc; 982 struct safexcel_token *dmmy; 983 int ret = 0; 984 985 /* Prepare command descriptor */ 986 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma, 987 &dmmy); 988 if (IS_ERR(cdesc)) 989 return PTR_ERR(cdesc); 990 991 cdesc->control_data.type = EIP197_TYPE_EXTENDED; 992 cdesc->control_data.options = 0; 993 cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK; 994 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR; 995 996 /* Prepare result descriptor */ 997 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 998 999 if (IS_ERR(rdesc)) { 1000 ret = PTR_ERR(rdesc); 1001 goto cdesc_rollback; 1002 } 1003 1004 safexcel_rdr_req_set(priv, ring, rdesc, async); 1005 1006 return ret; 1007 1008cdesc_rollback: 1009 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 1010 1011 return ret; 1012} 1013 1014static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv, 1015 int ring) 1016{ 1017 struct crypto_async_request *req; 1018 struct safexcel_context *ctx; 1019 int ret, i, nreq, ndesc, tot_descs, handled = 0; 1020 bool should_complete; 1021 1022handle_results: 1023 tot_descs = 0; 1024 1025 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); 1026 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET; 1027 nreq &= EIP197_xDR_PROC_xD_PKT_MASK; 1028 if (!nreq) 1029 goto requests_left; 1030 1031 for (i = 0; i < nreq; i++) { 1032 req = safexcel_rdr_req_get(priv, ring); 1033 1034 ctx = crypto_tfm_ctx(req->tfm); 1035 ndesc = ctx->handle_result(priv, ring, req, 1036 &should_complete, &ret); 1037 if (ndesc < 0) { 1038 dev_err(priv->dev, "failed to handle result (%d)\n", 1039 ndesc); 1040 goto acknowledge; 1041 } 1042 1043 if (should_complete) { 1044 local_bh_disable(); 1045 req->complete(req, ret); 1046 local_bh_enable(); 1047 } 1048 1049 tot_descs += ndesc; 1050 handled++; 1051 } 1052 1053acknowledge: 1054 if (i) 1055 writel(EIP197_xDR_PROC_xD_PKT(i) | 1056 (tot_descs * priv->config.rd_offset), 1057 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); 1058 1059 /* If the number of requests overflowed the counter, try to proceed more 1060 * requests. 1061 */ 1062 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK) 1063 goto handle_results; 1064 1065requests_left: 1066 spin_lock_bh(&priv->ring[ring].lock); 1067 1068 priv->ring[ring].requests -= handled; 1069 safexcel_try_push_requests(priv, ring); 1070 1071 if (!priv->ring[ring].requests) 1072 priv->ring[ring].busy = false; 1073 1074 spin_unlock_bh(&priv->ring[ring].lock); 1075} 1076 1077static void safexcel_dequeue_work(struct work_struct *work) 1078{ 1079 struct safexcel_work_data *data = 1080 container_of(work, struct safexcel_work_data, work); 1081 1082 safexcel_dequeue(data->priv, data->ring); 1083} 1084 1085struct safexcel_ring_irq_data { 1086 struct safexcel_crypto_priv *priv; 1087 int ring; 1088}; 1089 1090static irqreturn_t safexcel_irq_ring(int irq, void *data) 1091{ 1092 struct safexcel_ring_irq_data *irq_data = data; 1093 struct safexcel_crypto_priv *priv = irq_data->priv; 1094 int ring = irq_data->ring, rc = IRQ_NONE; 1095 u32 status, stat; 1096 1097 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring)); 1098 if (!status) 1099 return rc; 1100 1101 /* RDR interrupts */ 1102 if (status & EIP197_RDR_IRQ(ring)) { 1103 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); 1104 1105 if (unlikely(stat & EIP197_xDR_ERR)) { 1106 /* 1107 * Fatal error, the RDR is unusable and must be 1108 * reinitialized. This should not happen under 1109 * normal circumstances. 1110 */ 1111 dev_err(priv->dev, "RDR: fatal error.\n"); 1112 } else if (likely(stat & EIP197_xDR_THRESH)) { 1113 rc = IRQ_WAKE_THREAD; 1114 } 1115 1116 /* ACK the interrupts */ 1117 writel(stat & 0xff, 1118 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT); 1119 } 1120 1121 /* ACK the interrupts */ 1122 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring)); 1123 1124 return rc; 1125} 1126 1127static irqreturn_t safexcel_irq_ring_thread(int irq, void *data) 1128{ 1129 struct safexcel_ring_irq_data *irq_data = data; 1130 struct safexcel_crypto_priv *priv = irq_data->priv; 1131 int ring = irq_data->ring; 1132 1133 safexcel_handle_result_descriptor(priv, ring); 1134 1135 queue_work(priv->ring[ring].workqueue, 1136 &priv->ring[ring].work_data.work); 1137 1138 return IRQ_HANDLED; 1139} 1140 1141static int safexcel_request_ring_irq(void *pdev, int irqid, 1142 int is_pci_dev, 1143 int ring_id, 1144 irq_handler_t handler, 1145 irq_handler_t threaded_handler, 1146 struct safexcel_ring_irq_data *ring_irq_priv) 1147{ 1148 int ret, irq, cpu; 1149 struct device *dev; 1150 1151 if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) { 1152 struct pci_dev *pci_pdev = pdev; 1153 1154 dev = &pci_pdev->dev; 1155 irq = pci_irq_vector(pci_pdev, irqid); 1156 if (irq < 0) { 1157 dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n", 1158 irqid, irq); 1159 return irq; 1160 } 1161 } else if (IS_ENABLED(CONFIG_OF)) { 1162 struct platform_device *plf_pdev = pdev; 1163 char irq_name[6] = {0}; /* "ringX\0" */ 1164 1165 snprintf(irq_name, 6, "ring%d", irqid); 1166 dev = &plf_pdev->dev; 1167 irq = platform_get_irq_byname(plf_pdev, irq_name); 1168 1169 if (irq < 0) { 1170 dev_err(dev, "unable to get IRQ '%s' (err %d)\n", 1171 irq_name, irq); 1172 return irq; 1173 } 1174 } else { 1175 return -ENXIO; 1176 } 1177 1178 ret = devm_request_threaded_irq(dev, irq, handler, 1179 threaded_handler, IRQF_ONESHOT, 1180 dev_name(dev), ring_irq_priv); 1181 if (ret) { 1182 dev_err(dev, "unable to request IRQ %d\n", irq); 1183 return ret; 1184 } 1185 1186 /* Set affinity */ 1187 cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE); 1188 irq_set_affinity_hint(irq, get_cpu_mask(cpu)); 1189 1190 return irq; 1191} 1192 1193static struct safexcel_alg_template *safexcel_algs[] = { 1194 &safexcel_alg_ecb_des, 1195 &safexcel_alg_cbc_des, 1196 &safexcel_alg_ecb_des3_ede, 1197 &safexcel_alg_cbc_des3_ede, 1198 &safexcel_alg_ecb_aes, 1199 &safexcel_alg_cbc_aes, 1200 &safexcel_alg_cfb_aes, 1201 &safexcel_alg_ofb_aes, 1202 &safexcel_alg_ctr_aes, 1203 &safexcel_alg_md5, 1204 &safexcel_alg_sha1, 1205 &safexcel_alg_sha224, 1206 &safexcel_alg_sha256, 1207 &safexcel_alg_sha384, 1208 &safexcel_alg_sha512, 1209 &safexcel_alg_hmac_md5, 1210 &safexcel_alg_hmac_sha1, 1211 &safexcel_alg_hmac_sha224, 1212 &safexcel_alg_hmac_sha256, 1213 &safexcel_alg_hmac_sha384, 1214 &safexcel_alg_hmac_sha512, 1215 &safexcel_alg_authenc_hmac_sha1_cbc_aes, 1216 &safexcel_alg_authenc_hmac_sha224_cbc_aes, 1217 &safexcel_alg_authenc_hmac_sha256_cbc_aes, 1218 &safexcel_alg_authenc_hmac_sha384_cbc_aes, 1219 &safexcel_alg_authenc_hmac_sha512_cbc_aes, 1220 &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede, 1221 &safexcel_alg_authenc_hmac_sha1_ctr_aes, 1222 &safexcel_alg_authenc_hmac_sha224_ctr_aes, 1223 &safexcel_alg_authenc_hmac_sha256_ctr_aes, 1224 &safexcel_alg_authenc_hmac_sha384_ctr_aes, 1225 &safexcel_alg_authenc_hmac_sha512_ctr_aes, 1226 &safexcel_alg_xts_aes, 1227 &safexcel_alg_gcm, 1228 &safexcel_alg_ccm, 1229 &safexcel_alg_crc32, 1230 &safexcel_alg_cbcmac, 1231 &safexcel_alg_xcbcmac, 1232 &safexcel_alg_cmac, 1233 &safexcel_alg_chacha20, 1234 &safexcel_alg_chachapoly, 1235 &safexcel_alg_chachapoly_esp, 1236 &safexcel_alg_sm3, 1237 &safexcel_alg_hmac_sm3, 1238 &safexcel_alg_ecb_sm4, 1239 &safexcel_alg_cbc_sm4, 1240 &safexcel_alg_ofb_sm4, 1241 &safexcel_alg_cfb_sm4, 1242 &safexcel_alg_ctr_sm4, 1243 &safexcel_alg_authenc_hmac_sha1_cbc_sm4, 1244 &safexcel_alg_authenc_hmac_sm3_cbc_sm4, 1245 &safexcel_alg_authenc_hmac_sha1_ctr_sm4, 1246 &safexcel_alg_authenc_hmac_sm3_ctr_sm4, 1247 &safexcel_alg_sha3_224, 1248 &safexcel_alg_sha3_256, 1249 &safexcel_alg_sha3_384, 1250 &safexcel_alg_sha3_512, 1251 &safexcel_alg_hmac_sha3_224, 1252 &safexcel_alg_hmac_sha3_256, 1253 &safexcel_alg_hmac_sha3_384, 1254 &safexcel_alg_hmac_sha3_512, 1255 &safexcel_alg_authenc_hmac_sha1_cbc_des, 1256 &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede, 1257 &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede, 1258 &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede, 1259 &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede, 1260 &safexcel_alg_authenc_hmac_sha256_cbc_des, 1261 &safexcel_alg_authenc_hmac_sha224_cbc_des, 1262 &safexcel_alg_authenc_hmac_sha512_cbc_des, 1263 &safexcel_alg_authenc_hmac_sha384_cbc_des, 1264 &safexcel_alg_rfc4106_gcm, 1265 &safexcel_alg_rfc4543_gcm, 1266 &safexcel_alg_rfc4309_ccm, 1267}; 1268 1269static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv) 1270{ 1271 int i, j, ret = 0; 1272 1273 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { 1274 safexcel_algs[i]->priv = priv; 1275 1276 /* Do we have all required base algorithms available? */ 1277 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) != 1278 safexcel_algs[i]->algo_mask) 1279 /* No, so don't register this ciphersuite */ 1280 continue; 1281 1282 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 1283 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher); 1284 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) 1285 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead); 1286 else 1287 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash); 1288 1289 if (ret) 1290 goto fail; 1291 } 1292 1293 return 0; 1294 1295fail: 1296 for (j = 0; j < i; j++) { 1297 /* Do we have all required base algorithms available? */ 1298 if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) != 1299 safexcel_algs[j]->algo_mask) 1300 /* No, so don't unregister this ciphersuite */ 1301 continue; 1302 1303 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 1304 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher); 1305 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD) 1306 crypto_unregister_aead(&safexcel_algs[j]->alg.aead); 1307 else 1308 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash); 1309 } 1310 1311 return ret; 1312} 1313 1314static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv) 1315{ 1316 int i; 1317 1318 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) { 1319 /* Do we have all required base algorithms available? */ 1320 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) != 1321 safexcel_algs[i]->algo_mask) 1322 /* No, so don't unregister this ciphersuite */ 1323 continue; 1324 1325 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER) 1326 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher); 1327 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD) 1328 crypto_unregister_aead(&safexcel_algs[i]->alg.aead); 1329 else 1330 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash); 1331 } 1332} 1333 1334static void safexcel_configure(struct safexcel_crypto_priv *priv) 1335{ 1336 u32 mask = BIT(priv->hwconfig.hwdataw) - 1; 1337 1338 priv->config.pes = priv->hwconfig.hwnumpes; 1339 priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings); 1340 /* Cannot currently support more rings than we have ring AICs! */ 1341 priv->config.rings = min_t(u32, priv->config.rings, 1342 priv->hwconfig.hwnumraic); 1343 1344 priv->config.cd_size = EIP197_CD64_FETCH_SIZE; 1345 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask; 1346 priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask; 1347 1348 /* res token is behind the descr, but ofs must be rounded to buswdth */ 1349 priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask; 1350 /* now the size of the descr is this 1st part plus the result struct */ 1351 priv->config.rd_size = priv->config.res_offset + 1352 EIP197_RD64_RESULT_SIZE; 1353 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask; 1354 1355 /* convert dwords to bytes */ 1356 priv->config.cd_offset *= sizeof(u32); 1357 priv->config.cdsh_offset *= sizeof(u32); 1358 priv->config.rd_offset *= sizeof(u32); 1359 priv->config.res_offset *= sizeof(u32); 1360} 1361 1362static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv) 1363{ 1364 struct safexcel_register_offsets *offsets = &priv->offsets; 1365 1366 if (priv->flags & SAFEXCEL_HW_EIP197) { 1367 offsets->hia_aic = EIP197_HIA_AIC_BASE; 1368 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE; 1369 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE; 1370 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE; 1371 offsets->hia_dfe = EIP197_HIA_DFE_BASE; 1372 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE; 1373 offsets->hia_dse = EIP197_HIA_DSE_BASE; 1374 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE; 1375 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE; 1376 offsets->pe = EIP197_PE_BASE; 1377 offsets->global = EIP197_GLOBAL_BASE; 1378 } else { 1379 offsets->hia_aic = EIP97_HIA_AIC_BASE; 1380 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE; 1381 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE; 1382 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE; 1383 offsets->hia_dfe = EIP97_HIA_DFE_BASE; 1384 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE; 1385 offsets->hia_dse = EIP97_HIA_DSE_BASE; 1386 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE; 1387 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE; 1388 offsets->pe = EIP97_PE_BASE; 1389 offsets->global = EIP97_GLOBAL_BASE; 1390 } 1391} 1392 1393/* 1394 * Generic part of probe routine, shared by platform and PCI driver 1395 * 1396 * Assumes IO resources have been mapped, private data mem has been allocated, 1397 * clocks have been enabled, device pointer has been assigned etc. 1398 * 1399 */ 1400static int safexcel_probe_generic(void *pdev, 1401 struct safexcel_crypto_priv *priv, 1402 int is_pci_dev) 1403{ 1404 struct device *dev = priv->dev; 1405 u32 peid, version, mask, val, hiaopt, hwopt, peopt; 1406 int i, ret, hwctg; 1407 1408 priv->context_pool = dmam_pool_create("safexcel-context", dev, 1409 sizeof(struct safexcel_context_record), 1410 1, 0); 1411 if (!priv->context_pool) 1412 return -ENOMEM; 1413 1414 /* 1415 * First try the EIP97 HIA version regs 1416 * For the EIP197, this is guaranteed to NOT return any of the test 1417 * values 1418 */ 1419 version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION); 1420 1421 mask = 0; /* do not swap */ 1422 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) { 1423 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version); 1424 } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) { 1425 /* read back byte-swapped, so complement byte swap bits */ 1426 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS; 1427 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version); 1428 } else { 1429 /* So it wasn't an EIP97 ... maybe it's an EIP197? */ 1430 version = readl(priv->base + EIP197_HIA_AIC_BASE + 1431 EIP197_HIA_VERSION); 1432 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) { 1433 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version); 1434 priv->flags |= SAFEXCEL_HW_EIP197; 1435 } else if (EIP197_REG_HI16(version) == 1436 EIP197_HIA_VERSION_BE) { 1437 /* read back byte-swapped, so complement swap bits */ 1438 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS; 1439 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version); 1440 priv->flags |= SAFEXCEL_HW_EIP197; 1441 } else { 1442 return -ENODEV; 1443 } 1444 } 1445 1446 /* Now initialize the reg offsets based on the probing info so far */ 1447 safexcel_init_register_offsets(priv); 1448 1449 /* 1450 * If the version was read byte-swapped, we need to flip the device 1451 * swapping Keep in mind here, though, that what we write will also be 1452 * byte-swapped ... 1453 */ 1454 if (mask) { 1455 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 1456 val = val ^ (mask >> 24); /* toggle byte swap bits */ 1457 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL); 1458 } 1459 1460 /* 1461 * We're not done probing yet! We may fall through to here if no HIA 1462 * was found at all. So, with the endianness presumably correct now and 1463 * the offsets setup, *really* probe for the EIP97/EIP197. 1464 */ 1465 version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION); 1466 if (((priv->flags & SAFEXCEL_HW_EIP197) && 1467 (EIP197_REG_LO16(version) != EIP197_VERSION_LE) && 1468 (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) || 1469 ((!(priv->flags & SAFEXCEL_HW_EIP197) && 1470 (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) { 1471 /* 1472 * We did not find the device that matched our initial probing 1473 * (or our initial probing failed) Report appropriate error. 1474 */ 1475 dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n", 1476 version); 1477 return -ENODEV; 1478 } 1479 1480 priv->hwconfig.hwver = EIP197_VERSION_MASK(version); 1481 hwctg = version >> 28; 1482 peid = version & 255; 1483 1484 /* Detect EIP206 processing pipe */ 1485 version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0)); 1486 if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) { 1487 dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid); 1488 return -ENODEV; 1489 } 1490 priv->hwconfig.ppver = EIP197_VERSION_MASK(version); 1491 1492 /* Detect EIP96 packet engine and version */ 1493 version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0)); 1494 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { 1495 dev_err(dev, "EIP%d: EIP96 not detected.\n", peid); 1496 return -ENODEV; 1497 } 1498 priv->hwconfig.pever = EIP197_VERSION_MASK(version); 1499 1500 hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS); 1501 hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS); 1502 1503 priv->hwconfig.icever = 0; 1504 priv->hwconfig.ocever = 0; 1505 priv->hwconfig.psever = 0; 1506 if (priv->flags & SAFEXCEL_HW_EIP197) { 1507 /* EIP197 */ 1508 peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0)); 1509 1510 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & 1511 EIP197_HWDATAW_MASK; 1512 priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) & 1513 EIP197_CFSIZE_MASK) + 1514 EIP197_CFSIZE_ADJUST; 1515 priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) & 1516 EIP197_RFSIZE_MASK) + 1517 EIP197_RFSIZE_ADJUST; 1518 priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) & 1519 EIP197_N_PES_MASK; 1520 priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & 1521 EIP197_N_RINGS_MASK; 1522 if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB) 1523 priv->flags |= EIP197_PE_ARB; 1524 if (EIP206_OPT_ICE_TYPE(peopt) == 1) { 1525 priv->flags |= EIP197_ICE; 1526 /* Detect ICE EIP207 class. engine and version */ 1527 version = readl(EIP197_PE(priv) + 1528 EIP197_PE_ICE_VERSION(0)); 1529 if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { 1530 dev_err(dev, "EIP%d: ICE EIP207 not detected.\n", 1531 peid); 1532 return -ENODEV; 1533 } 1534 priv->hwconfig.icever = EIP197_VERSION_MASK(version); 1535 } 1536 if (EIP206_OPT_OCE_TYPE(peopt) == 1) { 1537 priv->flags |= EIP197_OCE; 1538 /* Detect EIP96PP packet stream editor and version */ 1539 version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0)); 1540 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) { 1541 dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid); 1542 return -ENODEV; 1543 } 1544 priv->hwconfig.psever = EIP197_VERSION_MASK(version); 1545 /* Detect OCE EIP207 class. engine and version */ 1546 version = readl(EIP197_PE(priv) + 1547 EIP197_PE_ICE_VERSION(0)); 1548 if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) { 1549 dev_err(dev, "EIP%d: OCE EIP207 not detected.\n", 1550 peid); 1551 return -ENODEV; 1552 } 1553 priv->hwconfig.ocever = EIP197_VERSION_MASK(version); 1554 } 1555 /* If not a full TRC, then assume simple TRC */ 1556 if (!(hwopt & EIP197_OPT_HAS_TRC)) 1557 priv->flags |= EIP197_SIMPLE_TRC; 1558 /* EIP197 always has SOME form of TRC */ 1559 priv->flags |= EIP197_TRC_CACHE; 1560 } else { 1561 /* EIP97 */ 1562 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) & 1563 EIP97_HWDATAW_MASK; 1564 priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) & 1565 EIP97_CFSIZE_MASK; 1566 priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) & 1567 EIP97_RFSIZE_MASK; 1568 priv->hwconfig.hwnumpes = 1; /* by definition */ 1569 priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) & 1570 EIP197_N_RINGS_MASK; 1571 } 1572 1573 /* Scan for ring AIC's */ 1574 for (i = 0; i < EIP197_MAX_RING_AIC; i++) { 1575 version = readl(EIP197_HIA_AIC_R(priv) + 1576 EIP197_HIA_AIC_R_VERSION(i)); 1577 if (EIP197_REG_LO16(version) != EIP201_VERSION_LE) 1578 break; 1579 } 1580 priv->hwconfig.hwnumraic = i; 1581 /* Low-end EIP196 may not have any ring AIC's ... */ 1582 if (!priv->hwconfig.hwnumraic) { 1583 dev_err(priv->dev, "No ring interrupt controller present!\n"); 1584 return -ENODEV; 1585 } 1586 1587 /* Get supported algorithms from EIP96 transform engine */ 1588 priv->hwconfig.algo_flags = readl(EIP197_PE(priv) + 1589 EIP197_PE_EIP96_OPTIONS(0)); 1590 1591 /* Print single info line describing what we just detected */ 1592 dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n", 1593 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes, 1594 priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic, 1595 priv->hwconfig.hiaver, priv->hwconfig.hwdataw, 1596 priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize, 1597 priv->hwconfig.ppver, priv->hwconfig.pever, 1598 priv->hwconfig.algo_flags, priv->hwconfig.icever, 1599 priv->hwconfig.ocever, priv->hwconfig.psever); 1600 1601 safexcel_configure(priv); 1602 1603 if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) { 1604 /* 1605 * Request MSI vectors for global + 1 per ring - 1606 * or just 1 for older dev images 1607 */ 1608 struct pci_dev *pci_pdev = pdev; 1609 1610 ret = pci_alloc_irq_vectors(pci_pdev, 1611 priv->config.rings + 1, 1612 priv->config.rings + 1, 1613 PCI_IRQ_MSI | PCI_IRQ_MSIX); 1614 if (ret < 0) { 1615 dev_err(dev, "Failed to allocate PCI MSI interrupts\n"); 1616 return ret; 1617 } 1618 } 1619 1620 /* Register the ring IRQ handlers and configure the rings */ 1621 priv->ring = devm_kcalloc(dev, priv->config.rings, 1622 sizeof(*priv->ring), 1623 GFP_KERNEL); 1624 if (!priv->ring) 1625 return -ENOMEM; 1626 1627 for (i = 0; i < priv->config.rings; i++) { 1628 char wq_name[9] = {0}; 1629 int irq; 1630 struct safexcel_ring_irq_data *ring_irq; 1631 1632 ret = safexcel_init_ring_descriptors(priv, 1633 &priv->ring[i].cdr, 1634 &priv->ring[i].rdr); 1635 if (ret) { 1636 dev_err(dev, "Failed to initialize rings\n"); 1637 goto err_cleanup_rings; 1638 } 1639 1640 priv->ring[i].rdr_req = devm_kcalloc(dev, 1641 EIP197_DEFAULT_RING_SIZE, 1642 sizeof(*priv->ring[i].rdr_req), 1643 GFP_KERNEL); 1644 if (!priv->ring[i].rdr_req) { 1645 ret = -ENOMEM; 1646 goto err_cleanup_rings; 1647 } 1648 1649 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL); 1650 if (!ring_irq) { 1651 ret = -ENOMEM; 1652 goto err_cleanup_rings; 1653 } 1654 1655 ring_irq->priv = priv; 1656 ring_irq->ring = i; 1657 1658 irq = safexcel_request_ring_irq(pdev, 1659 EIP197_IRQ_NUMBER(i, is_pci_dev), 1660 is_pci_dev, 1661 i, 1662 safexcel_irq_ring, 1663 safexcel_irq_ring_thread, 1664 ring_irq); 1665 if (irq < 0) { 1666 dev_err(dev, "Failed to get IRQ ID for ring %d\n", i); 1667 ret = irq; 1668 goto err_cleanup_rings; 1669 } 1670 1671 priv->ring[i].irq = irq; 1672 priv->ring[i].work_data.priv = priv; 1673 priv->ring[i].work_data.ring = i; 1674 INIT_WORK(&priv->ring[i].work_data.work, 1675 safexcel_dequeue_work); 1676 1677 snprintf(wq_name, 9, "wq_ring%d", i); 1678 priv->ring[i].workqueue = 1679 create_singlethread_workqueue(wq_name); 1680 if (!priv->ring[i].workqueue) { 1681 ret = -ENOMEM; 1682 goto err_cleanup_rings; 1683 } 1684 1685 priv->ring[i].requests = 0; 1686 priv->ring[i].busy = false; 1687 1688 crypto_init_queue(&priv->ring[i].queue, 1689 EIP197_DEFAULT_RING_SIZE); 1690 1691 spin_lock_init(&priv->ring[i].lock); 1692 spin_lock_init(&priv->ring[i].queue_lock); 1693 } 1694 1695 atomic_set(&priv->ring_used, 0); 1696 1697 ret = safexcel_hw_init(priv); 1698 if (ret) { 1699 dev_err(dev, "HW init failed (%d)\n", ret); 1700 goto err_cleanup_rings; 1701 } 1702 1703 ret = safexcel_register_algorithms(priv); 1704 if (ret) { 1705 dev_err(dev, "Failed to register algorithms (%d)\n", ret); 1706 goto err_cleanup_rings; 1707 } 1708 1709 return 0; 1710 1711err_cleanup_rings: 1712 for (i = 0; i < priv->config.rings; i++) { 1713 if (priv->ring[i].irq) 1714 irq_set_affinity_hint(priv->ring[i].irq, NULL); 1715 if (priv->ring[i].workqueue) 1716 destroy_workqueue(priv->ring[i].workqueue); 1717 } 1718 1719 return ret; 1720} 1721 1722static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv) 1723{ 1724 int i; 1725 1726 for (i = 0; i < priv->config.rings; i++) { 1727 /* clear any pending interrupt */ 1728 writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT); 1729 writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT); 1730 1731 /* Reset the CDR base address */ 1732 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 1733 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 1734 1735 /* Reset the RDR base address */ 1736 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO); 1737 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI); 1738 } 1739} 1740 1741/* for Device Tree platform driver */ 1742 1743static int safexcel_probe(struct platform_device *pdev) 1744{ 1745 struct device *dev = &pdev->dev; 1746 struct safexcel_crypto_priv *priv; 1747 int ret; 1748 1749 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 1750 if (!priv) 1751 return -ENOMEM; 1752 1753 priv->dev = dev; 1754 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev); 1755 1756 platform_set_drvdata(pdev, priv); 1757 1758 priv->base = devm_platform_ioremap_resource(pdev, 0); 1759 if (IS_ERR(priv->base)) { 1760 dev_err(dev, "failed to get resource\n"); 1761 return PTR_ERR(priv->base); 1762 } 1763 1764 priv->clk = devm_clk_get(&pdev->dev, NULL); 1765 ret = PTR_ERR_OR_ZERO(priv->clk); 1766 /* The clock isn't mandatory */ 1767 if (ret != -ENOENT) { 1768 if (ret) 1769 return ret; 1770 1771 ret = clk_prepare_enable(priv->clk); 1772 if (ret) { 1773 dev_err(dev, "unable to enable clk (%d)\n", ret); 1774 return ret; 1775 } 1776 } 1777 1778 priv->reg_clk = devm_clk_get(&pdev->dev, "reg"); 1779 ret = PTR_ERR_OR_ZERO(priv->reg_clk); 1780 /* The clock isn't mandatory */ 1781 if (ret != -ENOENT) { 1782 if (ret) 1783 goto err_core_clk; 1784 1785 ret = clk_prepare_enable(priv->reg_clk); 1786 if (ret) { 1787 dev_err(dev, "unable to enable reg clk (%d)\n", ret); 1788 goto err_core_clk; 1789 } 1790 } 1791 1792 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 1793 if (ret) 1794 goto err_reg_clk; 1795 1796 /* Generic EIP97/EIP197 device probing */ 1797 ret = safexcel_probe_generic(pdev, priv, 0); 1798 if (ret) 1799 goto err_reg_clk; 1800 1801 return 0; 1802 1803err_reg_clk: 1804 clk_disable_unprepare(priv->reg_clk); 1805err_core_clk: 1806 clk_disable_unprepare(priv->clk); 1807 return ret; 1808} 1809 1810static int safexcel_remove(struct platform_device *pdev) 1811{ 1812 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev); 1813 int i; 1814 1815 safexcel_unregister_algorithms(priv); 1816 safexcel_hw_reset_rings(priv); 1817 1818 clk_disable_unprepare(priv->reg_clk); 1819 clk_disable_unprepare(priv->clk); 1820 1821 for (i = 0; i < priv->config.rings; i++) { 1822 irq_set_affinity_hint(priv->ring[i].irq, NULL); 1823 destroy_workqueue(priv->ring[i].workqueue); 1824 } 1825 1826 return 0; 1827} 1828 1829static const struct of_device_id safexcel_of_match_table[] = { 1830 { 1831 .compatible = "inside-secure,safexcel-eip97ies", 1832 .data = (void *)EIP97IES_MRVL, 1833 }, 1834 { 1835 .compatible = "inside-secure,safexcel-eip197b", 1836 .data = (void *)EIP197B_MRVL, 1837 }, 1838 { 1839 .compatible = "inside-secure,safexcel-eip197d", 1840 .data = (void *)EIP197D_MRVL, 1841 }, 1842 /* For backward compatibility and intended for generic use */ 1843 { 1844 .compatible = "inside-secure,safexcel-eip97", 1845 .data = (void *)EIP97IES_MRVL, 1846 }, 1847 { 1848 .compatible = "inside-secure,safexcel-eip197", 1849 .data = (void *)EIP197B_MRVL, 1850 }, 1851 {}, 1852}; 1853 1854MODULE_DEVICE_TABLE(of, safexcel_of_match_table); 1855 1856static struct platform_driver crypto_safexcel = { 1857 .probe = safexcel_probe, 1858 .remove = safexcel_remove, 1859 .driver = { 1860 .name = "crypto-safexcel", 1861 .of_match_table = safexcel_of_match_table, 1862 }, 1863}; 1864 1865/* PCIE devices - i.e. Inside Secure development boards */ 1866 1867static int safexcel_pci_probe(struct pci_dev *pdev, 1868 const struct pci_device_id *ent) 1869{ 1870 struct device *dev = &pdev->dev; 1871 struct safexcel_crypto_priv *priv; 1872 void __iomem *pciebase; 1873 int rc; 1874 u32 val; 1875 1876 dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n", 1877 ent->vendor, ent->device, ent->subvendor, 1878 ent->subdevice, ent->driver_data); 1879 1880 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 1881 if (!priv) 1882 return -ENOMEM; 1883 1884 priv->dev = dev; 1885 priv->version = (enum safexcel_eip_version)ent->driver_data; 1886 1887 pci_set_drvdata(pdev, priv); 1888 1889 /* enable the device */ 1890 rc = pcim_enable_device(pdev); 1891 if (rc) { 1892 dev_err(dev, "Failed to enable PCI device\n"); 1893 return rc; 1894 } 1895 1896 /* take ownership of PCI BAR0 */ 1897 rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel"); 1898 if (rc) { 1899 dev_err(dev, "Failed to map IO region for BAR0\n"); 1900 return rc; 1901 } 1902 priv->base = pcim_iomap_table(pdev)[0]; 1903 1904 if (priv->version == EIP197_DEVBRD) { 1905 dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n"); 1906 1907 rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel"); 1908 if (rc) { 1909 dev_err(dev, "Failed to map IO region for BAR4\n"); 1910 return rc; 1911 } 1912 1913 pciebase = pcim_iomap_table(pdev)[2]; 1914 val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR); 1915 if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) { 1916 dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n", 1917 (val & 0xff)); 1918 1919 /* Setup MSI identity map mapping */ 1920 writel(EIP197_XLX_USER_VECT_LUT0_IDENT, 1921 pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR); 1922 writel(EIP197_XLX_USER_VECT_LUT1_IDENT, 1923 pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR); 1924 writel(EIP197_XLX_USER_VECT_LUT2_IDENT, 1925 pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR); 1926 writel(EIP197_XLX_USER_VECT_LUT3_IDENT, 1927 pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR); 1928 1929 /* Enable all device interrupts */ 1930 writel(GENMASK(31, 0), 1931 pciebase + EIP197_XLX_USER_INT_ENB_MSK); 1932 } else { 1933 dev_err(dev, "Unrecognised IRQ block identifier %x\n", 1934 val); 1935 return -ENODEV; 1936 } 1937 1938 /* HW reset FPGA dev board */ 1939 /* assert reset */ 1940 writel(1, priv->base + EIP197_XLX_GPIO_BASE); 1941 wmb(); /* maintain strict ordering for accesses here */ 1942 /* deassert reset */ 1943 writel(0, priv->base + EIP197_XLX_GPIO_BASE); 1944 wmb(); /* maintain strict ordering for accesses here */ 1945 } 1946 1947 /* enable bus mastering */ 1948 pci_set_master(pdev); 1949 1950 /* Generic EIP97/EIP197 device probing */ 1951 rc = safexcel_probe_generic(pdev, priv, 1); 1952 return rc; 1953} 1954 1955static void safexcel_pci_remove(struct pci_dev *pdev) 1956{ 1957 struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev); 1958 int i; 1959 1960 safexcel_unregister_algorithms(priv); 1961 1962 for (i = 0; i < priv->config.rings; i++) 1963 destroy_workqueue(priv->ring[i].workqueue); 1964 1965 safexcel_hw_reset_rings(priv); 1966} 1967 1968static const struct pci_device_id safexcel_pci_ids[] = { 1969 { 1970 PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038, 1971 0x16ae, 0xc522), 1972 .driver_data = EIP197_DEVBRD, 1973 }, 1974 {}, 1975}; 1976 1977MODULE_DEVICE_TABLE(pci, safexcel_pci_ids); 1978 1979static struct pci_driver safexcel_pci_driver = { 1980 .name = "crypto-safexcel", 1981 .id_table = safexcel_pci_ids, 1982 .probe = safexcel_pci_probe, 1983 .remove = safexcel_pci_remove, 1984}; 1985 1986static int __init safexcel_init(void) 1987{ 1988 int ret; 1989 1990 /* Register PCI driver */ 1991 ret = pci_register_driver(&safexcel_pci_driver); 1992 1993 /* Register platform driver */ 1994 if (IS_ENABLED(CONFIG_OF) && !ret) { 1995 ret = platform_driver_register(&crypto_safexcel); 1996 if (ret) 1997 pci_unregister_driver(&safexcel_pci_driver); 1998 } 1999 2000 return ret; 2001} 2002 2003static void __exit safexcel_exit(void) 2004{ 2005 /* Unregister platform driver */ 2006 if (IS_ENABLED(CONFIG_OF)) 2007 platform_driver_unregister(&crypto_safexcel); 2008 2009 /* Unregister PCI driver if successfully registered before */ 2010 pci_unregister_driver(&safexcel_pci_driver); 2011} 2012 2013module_init(safexcel_init); 2014module_exit(safexcel_exit); 2015 2016MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>"); 2017MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>"); 2018MODULE_AUTHOR("Igal Liberman <igall@marvell.com>"); 2019MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197"); 2020MODULE_LICENSE("GPL v2"); 2021