1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Freescale i.MX23/i.MX28 Data Co-Processor driver 4 * 5 * Copyright (C) 2013 Marek Vasut <marex@denx.de> 6 */ 7 8#include <linux/dma-mapping.h> 9#include <linux/interrupt.h> 10#include <linux/io.h> 11#include <linux/kernel.h> 12#include <linux/kthread.h> 13#include <linux/module.h> 14#include <linux/of.h> 15#include <linux/platform_device.h> 16#include <linux/stmp_device.h> 17#include <linux/clk.h> 18 19#include <crypto/aes.h> 20#include <crypto/sha.h> 21#include <crypto/internal/hash.h> 22#include <crypto/internal/skcipher.h> 23#include <crypto/scatterwalk.h> 24 25#define DCP_MAX_CHANS 4 26#define DCP_BUF_SZ PAGE_SIZE 27#define DCP_SHA_PAY_SZ 64 28 29#define DCP_ALIGNMENT 64 30 31/* 32 * Null hashes to align with hw behavior on imx6sl and ull 33 * these are flipped for consistency with hw output 34 */ 35static const uint8_t sha1_null_hash[] = 36 "\x09\x07\xd8\xaf\x90\x18\x60\x95\xef\xbf" 37 "\x55\x32\x0d\x4b\x6b\x5e\xee\xa3\x39\xda"; 38 39static const uint8_t sha256_null_hash[] = 40 "\x55\xb8\x52\x78\x1b\x99\x95\xa4" 41 "\x4c\x93\x9b\x64\xe4\x41\xae\x27" 42 "\x24\xb9\x6f\x99\xc8\xf4\xfb\x9a" 43 "\x14\x1c\xfc\x98\x42\xc4\xb0\xe3"; 44 45/* DCP DMA descriptor. */ 46struct dcp_dma_desc { 47 uint32_t next_cmd_addr; 48 uint32_t control0; 49 uint32_t control1; 50 uint32_t source; 51 uint32_t destination; 52 uint32_t size; 53 uint32_t payload; 54 uint32_t status; 55}; 56 57/* Coherent aligned block for bounce buffering. */ 58struct dcp_coherent_block { 59 uint8_t aes_in_buf[DCP_BUF_SZ]; 60 uint8_t aes_out_buf[DCP_BUF_SZ]; 61 uint8_t sha_in_buf[DCP_BUF_SZ]; 62 uint8_t sha_out_buf[DCP_SHA_PAY_SZ]; 63 64 uint8_t aes_key[2 * AES_KEYSIZE_128]; 65 66 struct dcp_dma_desc desc[DCP_MAX_CHANS]; 67}; 68 69struct dcp { 70 struct device *dev; 71 void __iomem *base; 72 73 uint32_t caps; 74 75 struct dcp_coherent_block *coh; 76 77 struct completion completion[DCP_MAX_CHANS]; 78 spinlock_t lock[DCP_MAX_CHANS]; 79 struct task_struct *thread[DCP_MAX_CHANS]; 80 struct crypto_queue queue[DCP_MAX_CHANS]; 81 struct clk *dcp_clk; 82}; 83 84enum dcp_chan { 85 DCP_CHAN_HASH_SHA = 0, 86 DCP_CHAN_CRYPTO = 2, 87}; 88 89struct dcp_async_ctx { 90 /* Common context */ 91 enum dcp_chan chan; 92 uint32_t fill; 93 94 /* SHA Hash-specific context */ 95 struct mutex mutex; 96 uint32_t alg; 97 unsigned int hot:1; 98 99 /* Crypto-specific context */ 100 struct crypto_skcipher *fallback; 101 unsigned int key_len; 102 uint8_t key[AES_KEYSIZE_128]; 103}; 104 105struct dcp_aes_req_ctx { 106 unsigned int enc:1; 107 unsigned int ecb:1; 108 struct skcipher_request fallback_req; // keep at the end 109}; 110 111struct dcp_sha_req_ctx { 112 unsigned int init:1; 113 unsigned int fini:1; 114}; 115 116struct dcp_export_state { 117 struct dcp_sha_req_ctx req_ctx; 118 struct dcp_async_ctx async_ctx; 119}; 120 121/* 122 * There can even be only one instance of the MXS DCP due to the 123 * design of Linux Crypto API. 124 */ 125static struct dcp *global_sdcp; 126 127/* DCP register layout. */ 128#define MXS_DCP_CTRL 0x00 129#define MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES (1 << 23) 130#define MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING (1 << 22) 131 132#define MXS_DCP_STAT 0x10 133#define MXS_DCP_STAT_CLR 0x18 134#define MXS_DCP_STAT_IRQ_MASK 0xf 135 136#define MXS_DCP_CHANNELCTRL 0x20 137#define MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK 0xff 138 139#define MXS_DCP_CAPABILITY1 0x40 140#define MXS_DCP_CAPABILITY1_SHA256 (4 << 16) 141#define MXS_DCP_CAPABILITY1_SHA1 (1 << 16) 142#define MXS_DCP_CAPABILITY1_AES128 (1 << 0) 143 144#define MXS_DCP_CONTEXT 0x50 145 146#define MXS_DCP_CH_N_CMDPTR(n) (0x100 + ((n) * 0x40)) 147 148#define MXS_DCP_CH_N_SEMA(n) (0x110 + ((n) * 0x40)) 149 150#define MXS_DCP_CH_N_STAT(n) (0x120 + ((n) * 0x40)) 151#define MXS_DCP_CH_N_STAT_CLR(n) (0x128 + ((n) * 0x40)) 152 153/* DMA descriptor bits. */ 154#define MXS_DCP_CONTROL0_HASH_TERM (1 << 13) 155#define MXS_DCP_CONTROL0_HASH_INIT (1 << 12) 156#define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11) 157#define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8) 158#define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9) 159#define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6) 160#define MXS_DCP_CONTROL0_ENABLE_CIPHER (1 << 5) 161#define MXS_DCP_CONTROL0_DECR_SEMAPHORE (1 << 1) 162#define MXS_DCP_CONTROL0_INTERRUPT (1 << 0) 163 164#define MXS_DCP_CONTROL1_HASH_SELECT_SHA256 (2 << 16) 165#define MXS_DCP_CONTROL1_HASH_SELECT_SHA1 (0 << 16) 166#define MXS_DCP_CONTROL1_CIPHER_MODE_CBC (1 << 4) 167#define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4) 168#define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0) 169 170static int mxs_dcp_start_dma(struct dcp_async_ctx *actx) 171{ 172 int dma_err; 173 struct dcp *sdcp = global_sdcp; 174 const int chan = actx->chan; 175 uint32_t stat; 176 unsigned long ret; 177 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; 178 dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc), 179 DMA_TO_DEVICE); 180 181 dma_err = dma_mapping_error(sdcp->dev, desc_phys); 182 if (dma_err) 183 return dma_err; 184 185 reinit_completion(&sdcp->completion[chan]); 186 187 /* Clear status register. */ 188 writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(chan)); 189 190 /* Load the DMA descriptor. */ 191 writel(desc_phys, sdcp->base + MXS_DCP_CH_N_CMDPTR(chan)); 192 193 /* Increment the semaphore to start the DMA transfer. */ 194 writel(1, sdcp->base + MXS_DCP_CH_N_SEMA(chan)); 195 196 ret = wait_for_completion_timeout(&sdcp->completion[chan], 197 msecs_to_jiffies(1000)); 198 if (!ret) { 199 dev_err(sdcp->dev, "Channel %i timeout (DCP_STAT=0x%08x)\n", 200 chan, readl(sdcp->base + MXS_DCP_STAT)); 201 return -ETIMEDOUT; 202 } 203 204 stat = readl(sdcp->base + MXS_DCP_CH_N_STAT(chan)); 205 if (stat & 0xff) { 206 dev_err(sdcp->dev, "Channel %i error (CH_STAT=0x%08x)\n", 207 chan, stat); 208 return -EINVAL; 209 } 210 211 dma_unmap_single(sdcp->dev, desc_phys, sizeof(*desc), DMA_TO_DEVICE); 212 213 return 0; 214} 215 216/* 217 * Encryption (AES128) 218 */ 219static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, 220 struct skcipher_request *req, int init) 221{ 222 dma_addr_t key_phys, src_phys, dst_phys; 223 struct dcp *sdcp = global_sdcp; 224 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; 225 struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); 226 int ret; 227 228 key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, 229 2 * AES_KEYSIZE_128, DMA_TO_DEVICE); 230 ret = dma_mapping_error(sdcp->dev, key_phys); 231 if (ret) 232 return ret; 233 234 src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf, 235 DCP_BUF_SZ, DMA_TO_DEVICE); 236 ret = dma_mapping_error(sdcp->dev, src_phys); 237 if (ret) 238 goto err_src; 239 240 dst_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_out_buf, 241 DCP_BUF_SZ, DMA_FROM_DEVICE); 242 ret = dma_mapping_error(sdcp->dev, dst_phys); 243 if (ret) 244 goto err_dst; 245 246 if (actx->fill % AES_BLOCK_SIZE) { 247 dev_err(sdcp->dev, "Invalid block size!\n"); 248 ret = -EINVAL; 249 goto aes_done_run; 250 } 251 252 /* Fill in the DMA descriptor. */ 253 desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | 254 MXS_DCP_CONTROL0_INTERRUPT | 255 MXS_DCP_CONTROL0_ENABLE_CIPHER; 256 257 /* Payload contains the key. */ 258 desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; 259 260 if (rctx->enc) 261 desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT; 262 if (init) 263 desc->control0 |= MXS_DCP_CONTROL0_CIPHER_INIT; 264 265 desc->control1 = MXS_DCP_CONTROL1_CIPHER_SELECT_AES128; 266 267 if (rctx->ecb) 268 desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_ECB; 269 else 270 desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_CBC; 271 272 desc->next_cmd_addr = 0; 273 desc->source = src_phys; 274 desc->destination = dst_phys; 275 desc->size = actx->fill; 276 desc->payload = key_phys; 277 desc->status = 0; 278 279 ret = mxs_dcp_start_dma(actx); 280 281aes_done_run: 282 dma_unmap_single(sdcp->dev, dst_phys, DCP_BUF_SZ, DMA_FROM_DEVICE); 283err_dst: 284 dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE); 285err_src: 286 dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, 287 DMA_TO_DEVICE); 288 289 return ret; 290} 291 292static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq) 293{ 294 struct dcp *sdcp = global_sdcp; 295 296 struct skcipher_request *req = skcipher_request_cast(arq); 297 struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); 298 struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); 299 300 struct scatterlist *dst = req->dst; 301 struct scatterlist *src = req->src; 302 int dst_nents = sg_nents(dst); 303 304 const int out_off = DCP_BUF_SZ; 305 uint8_t *in_buf = sdcp->coh->aes_in_buf; 306 uint8_t *out_buf = sdcp->coh->aes_out_buf; 307 308 uint32_t dst_off = 0; 309 uint8_t *src_buf = NULL; 310 uint32_t last_out_len = 0; 311 312 uint8_t *key = sdcp->coh->aes_key; 313 314 int ret = 0; 315 unsigned int i, len, clen, tlen = 0; 316 int init = 0; 317 bool limit_hit = false; 318 319 actx->fill = 0; 320 321 /* Copy the key from the temporary location. */ 322 memcpy(key, actx->key, actx->key_len); 323 324 if (!rctx->ecb) { 325 /* Copy the CBC IV just past the key. */ 326 memcpy(key + AES_KEYSIZE_128, req->iv, AES_KEYSIZE_128); 327 /* CBC needs the INIT set. */ 328 init = 1; 329 } else { 330 memset(key + AES_KEYSIZE_128, 0, AES_KEYSIZE_128); 331 } 332 333 for_each_sg(req->src, src, sg_nents(req->src), i) { 334 src_buf = sg_virt(src); 335 len = sg_dma_len(src); 336 tlen += len; 337 limit_hit = tlen > req->cryptlen; 338 339 if (limit_hit) 340 len = req->cryptlen - (tlen - len); 341 342 do { 343 if (actx->fill + len > out_off) 344 clen = out_off - actx->fill; 345 else 346 clen = len; 347 348 memcpy(in_buf + actx->fill, src_buf, clen); 349 len -= clen; 350 src_buf += clen; 351 actx->fill += clen; 352 353 /* 354 * If we filled the buffer or this is the last SG, 355 * submit the buffer. 356 */ 357 if (actx->fill == out_off || sg_is_last(src) || 358 limit_hit) { 359 ret = mxs_dcp_run_aes(actx, req, init); 360 if (ret) 361 return ret; 362 init = 0; 363 364 sg_pcopy_from_buffer(dst, dst_nents, out_buf, 365 actx->fill, dst_off); 366 dst_off += actx->fill; 367 last_out_len = actx->fill; 368 actx->fill = 0; 369 } 370 } while (len); 371 372 if (limit_hit) 373 break; 374 } 375 376 /* Copy the IV for CBC for chaining */ 377 if (!rctx->ecb) { 378 if (rctx->enc) 379 memcpy(req->iv, out_buf+(last_out_len-AES_BLOCK_SIZE), 380 AES_BLOCK_SIZE); 381 else 382 memcpy(req->iv, in_buf+(last_out_len-AES_BLOCK_SIZE), 383 AES_BLOCK_SIZE); 384 } 385 386 return ret; 387} 388 389static int dcp_chan_thread_aes(void *data) 390{ 391 struct dcp *sdcp = global_sdcp; 392 const int chan = DCP_CHAN_CRYPTO; 393 394 struct crypto_async_request *backlog; 395 struct crypto_async_request *arq; 396 397 int ret; 398 399 while (!kthread_should_stop()) { 400 set_current_state(TASK_INTERRUPTIBLE); 401 402 spin_lock(&sdcp->lock[chan]); 403 backlog = crypto_get_backlog(&sdcp->queue[chan]); 404 arq = crypto_dequeue_request(&sdcp->queue[chan]); 405 spin_unlock(&sdcp->lock[chan]); 406 407 if (!backlog && !arq) { 408 schedule(); 409 continue; 410 } 411 412 set_current_state(TASK_RUNNING); 413 414 if (backlog) 415 backlog->complete(backlog, -EINPROGRESS); 416 417 if (arq) { 418 ret = mxs_dcp_aes_block_crypt(arq); 419 arq->complete(arq, ret); 420 } 421 } 422 423 return 0; 424} 425 426static int mxs_dcp_block_fallback(struct skcipher_request *req, int enc) 427{ 428 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 429 struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); 430 struct dcp_async_ctx *ctx = crypto_skcipher_ctx(tfm); 431 int ret; 432 433 skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); 434 skcipher_request_set_callback(&rctx->fallback_req, req->base.flags, 435 req->base.complete, req->base.data); 436 skcipher_request_set_crypt(&rctx->fallback_req, req->src, req->dst, 437 req->cryptlen, req->iv); 438 439 if (enc) 440 ret = crypto_skcipher_encrypt(&rctx->fallback_req); 441 else 442 ret = crypto_skcipher_decrypt(&rctx->fallback_req); 443 444 return ret; 445} 446 447static int mxs_dcp_aes_enqueue(struct skcipher_request *req, int enc, int ecb) 448{ 449 struct dcp *sdcp = global_sdcp; 450 struct crypto_async_request *arq = &req->base; 451 struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); 452 struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); 453 int ret; 454 455 if (unlikely(actx->key_len != AES_KEYSIZE_128)) 456 return mxs_dcp_block_fallback(req, enc); 457 458 rctx->enc = enc; 459 rctx->ecb = ecb; 460 actx->chan = DCP_CHAN_CRYPTO; 461 462 spin_lock(&sdcp->lock[actx->chan]); 463 ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); 464 spin_unlock(&sdcp->lock[actx->chan]); 465 466 wake_up_process(sdcp->thread[actx->chan]); 467 468 return ret; 469} 470 471static int mxs_dcp_aes_ecb_decrypt(struct skcipher_request *req) 472{ 473 return mxs_dcp_aes_enqueue(req, 0, 1); 474} 475 476static int mxs_dcp_aes_ecb_encrypt(struct skcipher_request *req) 477{ 478 return mxs_dcp_aes_enqueue(req, 1, 1); 479} 480 481static int mxs_dcp_aes_cbc_decrypt(struct skcipher_request *req) 482{ 483 return mxs_dcp_aes_enqueue(req, 0, 0); 484} 485 486static int mxs_dcp_aes_cbc_encrypt(struct skcipher_request *req) 487{ 488 return mxs_dcp_aes_enqueue(req, 1, 0); 489} 490 491static int mxs_dcp_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 492 unsigned int len) 493{ 494 struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); 495 496 /* 497 * AES 128 is supposed by the hardware, store key into temporary 498 * buffer and exit. We must use the temporary buffer here, since 499 * there can still be an operation in progress. 500 */ 501 actx->key_len = len; 502 if (len == AES_KEYSIZE_128) { 503 memcpy(actx->key, key, len); 504 return 0; 505 } 506 507 /* 508 * If the requested AES key size is not supported by the hardware, 509 * but is supported by in-kernel software implementation, we use 510 * software fallback. 511 */ 512 crypto_skcipher_clear_flags(actx->fallback, CRYPTO_TFM_REQ_MASK); 513 crypto_skcipher_set_flags(actx->fallback, 514 tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); 515 return crypto_skcipher_setkey(actx->fallback, key, len); 516} 517 518static int mxs_dcp_aes_fallback_init_tfm(struct crypto_skcipher *tfm) 519{ 520 const char *name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); 521 struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); 522 struct crypto_skcipher *blk; 523 524 blk = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 525 if (IS_ERR(blk)) 526 return PTR_ERR(blk); 527 528 actx->fallback = blk; 529 crypto_skcipher_set_reqsize(tfm, sizeof(struct dcp_aes_req_ctx) + 530 crypto_skcipher_reqsize(blk)); 531 return 0; 532} 533 534static void mxs_dcp_aes_fallback_exit_tfm(struct crypto_skcipher *tfm) 535{ 536 struct dcp_async_ctx *actx = crypto_skcipher_ctx(tfm); 537 538 crypto_free_skcipher(actx->fallback); 539} 540 541/* 542 * Hashing (SHA1/SHA256) 543 */ 544static int mxs_dcp_run_sha(struct ahash_request *req) 545{ 546 struct dcp *sdcp = global_sdcp; 547 int ret; 548 549 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 550 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); 551 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); 552 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; 553 554 dma_addr_t digest_phys = 0; 555 dma_addr_t buf_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_in_buf, 556 DCP_BUF_SZ, DMA_TO_DEVICE); 557 558 ret = dma_mapping_error(sdcp->dev, buf_phys); 559 if (ret) 560 return ret; 561 562 /* Fill in the DMA descriptor. */ 563 desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | 564 MXS_DCP_CONTROL0_INTERRUPT | 565 MXS_DCP_CONTROL0_ENABLE_HASH; 566 if (rctx->init) 567 desc->control0 |= MXS_DCP_CONTROL0_HASH_INIT; 568 569 desc->control1 = actx->alg; 570 desc->next_cmd_addr = 0; 571 desc->source = buf_phys; 572 desc->destination = 0; 573 desc->size = actx->fill; 574 desc->payload = 0; 575 desc->status = 0; 576 577 /* 578 * Align driver with hw behavior when generating null hashes 579 */ 580 if (rctx->init && rctx->fini && desc->size == 0) { 581 struct hash_alg_common *halg = crypto_hash_alg_common(tfm); 582 const uint8_t *sha_buf = 583 (actx->alg == MXS_DCP_CONTROL1_HASH_SELECT_SHA1) ? 584 sha1_null_hash : sha256_null_hash; 585 memcpy(sdcp->coh->sha_out_buf, sha_buf, halg->digestsize); 586 ret = 0; 587 goto done_run; 588 } 589 590 /* Set HASH_TERM bit for last transfer block. */ 591 if (rctx->fini) { 592 digest_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_out_buf, 593 DCP_SHA_PAY_SZ, DMA_FROM_DEVICE); 594 ret = dma_mapping_error(sdcp->dev, digest_phys); 595 if (ret) 596 goto done_run; 597 598 desc->control0 |= MXS_DCP_CONTROL0_HASH_TERM; 599 desc->payload = digest_phys; 600 } 601 602 ret = mxs_dcp_start_dma(actx); 603 604 if (rctx->fini) 605 dma_unmap_single(sdcp->dev, digest_phys, DCP_SHA_PAY_SZ, 606 DMA_FROM_DEVICE); 607 608done_run: 609 dma_unmap_single(sdcp->dev, buf_phys, DCP_BUF_SZ, DMA_TO_DEVICE); 610 611 return ret; 612} 613 614static int dcp_sha_req_to_buf(struct crypto_async_request *arq) 615{ 616 struct dcp *sdcp = global_sdcp; 617 618 struct ahash_request *req = ahash_request_cast(arq); 619 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 620 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); 621 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); 622 struct hash_alg_common *halg = crypto_hash_alg_common(tfm); 623 624 uint8_t *in_buf = sdcp->coh->sha_in_buf; 625 uint8_t *out_buf = sdcp->coh->sha_out_buf; 626 627 struct scatterlist *src; 628 629 unsigned int i, len, clen, oft = 0; 630 int ret; 631 632 int fin = rctx->fini; 633 if (fin) 634 rctx->fini = 0; 635 636 src = req->src; 637 len = req->nbytes; 638 639 while (len) { 640 if (actx->fill + len > DCP_BUF_SZ) 641 clen = DCP_BUF_SZ - actx->fill; 642 else 643 clen = len; 644 645 scatterwalk_map_and_copy(in_buf + actx->fill, src, oft, clen, 646 0); 647 648 len -= clen; 649 oft += clen; 650 actx->fill += clen; 651 652 /* 653 * If we filled the buffer and still have some 654 * more data, submit the buffer. 655 */ 656 if (len && actx->fill == DCP_BUF_SZ) { 657 ret = mxs_dcp_run_sha(req); 658 if (ret) 659 return ret; 660 actx->fill = 0; 661 rctx->init = 0; 662 } 663 } 664 665 if (fin) { 666 rctx->fini = 1; 667 668 /* Submit whatever is left. */ 669 if (!req->result) 670 return -EINVAL; 671 672 ret = mxs_dcp_run_sha(req); 673 if (ret) 674 return ret; 675 676 actx->fill = 0; 677 678 /* For some reason the result is flipped */ 679 for (i = 0; i < halg->digestsize; i++) 680 req->result[i] = out_buf[halg->digestsize - i - 1]; 681 } 682 683 return 0; 684} 685 686static int dcp_chan_thread_sha(void *data) 687{ 688 struct dcp *sdcp = global_sdcp; 689 const int chan = DCP_CHAN_HASH_SHA; 690 691 struct crypto_async_request *backlog; 692 struct crypto_async_request *arq; 693 int ret; 694 695 while (!kthread_should_stop()) { 696 set_current_state(TASK_INTERRUPTIBLE); 697 698 spin_lock(&sdcp->lock[chan]); 699 backlog = crypto_get_backlog(&sdcp->queue[chan]); 700 arq = crypto_dequeue_request(&sdcp->queue[chan]); 701 spin_unlock(&sdcp->lock[chan]); 702 703 if (!backlog && !arq) { 704 schedule(); 705 continue; 706 } 707 708 set_current_state(TASK_RUNNING); 709 710 if (backlog) 711 backlog->complete(backlog, -EINPROGRESS); 712 713 if (arq) { 714 ret = dcp_sha_req_to_buf(arq); 715 arq->complete(arq, ret); 716 } 717 } 718 719 return 0; 720} 721 722static int dcp_sha_init(struct ahash_request *req) 723{ 724 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 725 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); 726 727 struct hash_alg_common *halg = crypto_hash_alg_common(tfm); 728 729 /* 730 * Start hashing session. The code below only inits the 731 * hashing session context, nothing more. 732 */ 733 memset(actx, 0, sizeof(*actx)); 734 735 if (strcmp(halg->base.cra_name, "sha1") == 0) 736 actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA1; 737 else 738 actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA256; 739 740 actx->fill = 0; 741 actx->hot = 0; 742 actx->chan = DCP_CHAN_HASH_SHA; 743 744 mutex_init(&actx->mutex); 745 746 return 0; 747} 748 749static int dcp_sha_update_fx(struct ahash_request *req, int fini) 750{ 751 struct dcp *sdcp = global_sdcp; 752 753 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); 754 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 755 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); 756 757 int ret; 758 759 /* 760 * Ignore requests that have no data in them and are not 761 * the trailing requests in the stream of requests. 762 */ 763 if (!req->nbytes && !fini) 764 return 0; 765 766 mutex_lock(&actx->mutex); 767 768 rctx->fini = fini; 769 770 if (!actx->hot) { 771 actx->hot = 1; 772 rctx->init = 1; 773 } 774 775 spin_lock(&sdcp->lock[actx->chan]); 776 ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); 777 spin_unlock(&sdcp->lock[actx->chan]); 778 779 wake_up_process(sdcp->thread[actx->chan]); 780 mutex_unlock(&actx->mutex); 781 782 return ret; 783} 784 785static int dcp_sha_update(struct ahash_request *req) 786{ 787 return dcp_sha_update_fx(req, 0); 788} 789 790static int dcp_sha_final(struct ahash_request *req) 791{ 792 ahash_request_set_crypt(req, NULL, req->result, 0); 793 req->nbytes = 0; 794 return dcp_sha_update_fx(req, 1); 795} 796 797static int dcp_sha_finup(struct ahash_request *req) 798{ 799 return dcp_sha_update_fx(req, 1); 800} 801 802static int dcp_sha_digest(struct ahash_request *req) 803{ 804 int ret; 805 806 ret = dcp_sha_init(req); 807 if (ret) 808 return ret; 809 810 return dcp_sha_finup(req); 811} 812 813static int dcp_sha_import(struct ahash_request *req, const void *in) 814{ 815 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); 816 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 817 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); 818 const struct dcp_export_state *export = in; 819 820 memset(rctx, 0, sizeof(struct dcp_sha_req_ctx)); 821 memset(actx, 0, sizeof(struct dcp_async_ctx)); 822 memcpy(rctx, &export->req_ctx, sizeof(struct dcp_sha_req_ctx)); 823 memcpy(actx, &export->async_ctx, sizeof(struct dcp_async_ctx)); 824 825 return 0; 826} 827 828static int dcp_sha_export(struct ahash_request *req, void *out) 829{ 830 struct dcp_sha_req_ctx *rctx_state = ahash_request_ctx(req); 831 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 832 struct dcp_async_ctx *actx_state = crypto_ahash_ctx(tfm); 833 struct dcp_export_state *export = out; 834 835 memcpy(&export->req_ctx, rctx_state, sizeof(struct dcp_sha_req_ctx)); 836 memcpy(&export->async_ctx, actx_state, sizeof(struct dcp_async_ctx)); 837 838 return 0; 839} 840 841static int dcp_sha_cra_init(struct crypto_tfm *tfm) 842{ 843 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 844 sizeof(struct dcp_sha_req_ctx)); 845 return 0; 846} 847 848static void dcp_sha_cra_exit(struct crypto_tfm *tfm) 849{ 850} 851 852/* AES 128 ECB and AES 128 CBC */ 853static struct skcipher_alg dcp_aes_algs[] = { 854 { 855 .base.cra_name = "ecb(aes)", 856 .base.cra_driver_name = "ecb-aes-dcp", 857 .base.cra_priority = 400, 858 .base.cra_alignmask = 15, 859 .base.cra_flags = CRYPTO_ALG_ASYNC | 860 CRYPTO_ALG_NEED_FALLBACK, 861 .base.cra_blocksize = AES_BLOCK_SIZE, 862 .base.cra_ctxsize = sizeof(struct dcp_async_ctx), 863 .base.cra_module = THIS_MODULE, 864 865 .min_keysize = AES_MIN_KEY_SIZE, 866 .max_keysize = AES_MAX_KEY_SIZE, 867 .setkey = mxs_dcp_aes_setkey, 868 .encrypt = mxs_dcp_aes_ecb_encrypt, 869 .decrypt = mxs_dcp_aes_ecb_decrypt, 870 .init = mxs_dcp_aes_fallback_init_tfm, 871 .exit = mxs_dcp_aes_fallback_exit_tfm, 872 }, { 873 .base.cra_name = "cbc(aes)", 874 .base.cra_driver_name = "cbc-aes-dcp", 875 .base.cra_priority = 400, 876 .base.cra_alignmask = 15, 877 .base.cra_flags = CRYPTO_ALG_ASYNC | 878 CRYPTO_ALG_NEED_FALLBACK, 879 .base.cra_blocksize = AES_BLOCK_SIZE, 880 .base.cra_ctxsize = sizeof(struct dcp_async_ctx), 881 .base.cra_module = THIS_MODULE, 882 883 .min_keysize = AES_MIN_KEY_SIZE, 884 .max_keysize = AES_MAX_KEY_SIZE, 885 .setkey = mxs_dcp_aes_setkey, 886 .encrypt = mxs_dcp_aes_cbc_encrypt, 887 .decrypt = mxs_dcp_aes_cbc_decrypt, 888 .ivsize = AES_BLOCK_SIZE, 889 .init = mxs_dcp_aes_fallback_init_tfm, 890 .exit = mxs_dcp_aes_fallback_exit_tfm, 891 }, 892}; 893 894/* SHA1 */ 895static struct ahash_alg dcp_sha1_alg = { 896 .init = dcp_sha_init, 897 .update = dcp_sha_update, 898 .final = dcp_sha_final, 899 .finup = dcp_sha_finup, 900 .digest = dcp_sha_digest, 901 .import = dcp_sha_import, 902 .export = dcp_sha_export, 903 .halg = { 904 .digestsize = SHA1_DIGEST_SIZE, 905 .statesize = sizeof(struct dcp_export_state), 906 .base = { 907 .cra_name = "sha1", 908 .cra_driver_name = "sha1-dcp", 909 .cra_priority = 400, 910 .cra_alignmask = 63, 911 .cra_flags = CRYPTO_ALG_ASYNC, 912 .cra_blocksize = SHA1_BLOCK_SIZE, 913 .cra_ctxsize = sizeof(struct dcp_async_ctx), 914 .cra_module = THIS_MODULE, 915 .cra_init = dcp_sha_cra_init, 916 .cra_exit = dcp_sha_cra_exit, 917 }, 918 }, 919}; 920 921/* SHA256 */ 922static struct ahash_alg dcp_sha256_alg = { 923 .init = dcp_sha_init, 924 .update = dcp_sha_update, 925 .final = dcp_sha_final, 926 .finup = dcp_sha_finup, 927 .digest = dcp_sha_digest, 928 .import = dcp_sha_import, 929 .export = dcp_sha_export, 930 .halg = { 931 .digestsize = SHA256_DIGEST_SIZE, 932 .statesize = sizeof(struct dcp_export_state), 933 .base = { 934 .cra_name = "sha256", 935 .cra_driver_name = "sha256-dcp", 936 .cra_priority = 400, 937 .cra_alignmask = 63, 938 .cra_flags = CRYPTO_ALG_ASYNC, 939 .cra_blocksize = SHA256_BLOCK_SIZE, 940 .cra_ctxsize = sizeof(struct dcp_async_ctx), 941 .cra_module = THIS_MODULE, 942 .cra_init = dcp_sha_cra_init, 943 .cra_exit = dcp_sha_cra_exit, 944 }, 945 }, 946}; 947 948static irqreturn_t mxs_dcp_irq(int irq, void *context) 949{ 950 struct dcp *sdcp = context; 951 uint32_t stat; 952 int i; 953 954 stat = readl(sdcp->base + MXS_DCP_STAT); 955 stat &= MXS_DCP_STAT_IRQ_MASK; 956 if (!stat) 957 return IRQ_NONE; 958 959 /* Clear the interrupts. */ 960 writel(stat, sdcp->base + MXS_DCP_STAT_CLR); 961 962 /* Complete the DMA requests that finished. */ 963 for (i = 0; i < DCP_MAX_CHANS; i++) 964 if (stat & (1 << i)) 965 complete(&sdcp->completion[i]); 966 967 return IRQ_HANDLED; 968} 969 970static int mxs_dcp_probe(struct platform_device *pdev) 971{ 972 struct device *dev = &pdev->dev; 973 struct dcp *sdcp = NULL; 974 int i, ret; 975 int dcp_vmi_irq, dcp_irq; 976 977 if (global_sdcp) { 978 dev_err(dev, "Only one DCP instance allowed!\n"); 979 return -ENODEV; 980 } 981 982 dcp_vmi_irq = platform_get_irq(pdev, 0); 983 if (dcp_vmi_irq < 0) 984 return dcp_vmi_irq; 985 986 dcp_irq = platform_get_irq(pdev, 1); 987 if (dcp_irq < 0) 988 return dcp_irq; 989 990 sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL); 991 if (!sdcp) 992 return -ENOMEM; 993 994 sdcp->dev = dev; 995 sdcp->base = devm_platform_ioremap_resource(pdev, 0); 996 if (IS_ERR(sdcp->base)) 997 return PTR_ERR(sdcp->base); 998 999 1000 ret = devm_request_irq(dev, dcp_vmi_irq, mxs_dcp_irq, 0, 1001 "dcp-vmi-irq", sdcp); 1002 if (ret) { 1003 dev_err(dev, "Failed to claim DCP VMI IRQ!\n"); 1004 return ret; 1005 } 1006 1007 ret = devm_request_irq(dev, dcp_irq, mxs_dcp_irq, 0, 1008 "dcp-irq", sdcp); 1009 if (ret) { 1010 dev_err(dev, "Failed to claim DCP IRQ!\n"); 1011 return ret; 1012 } 1013 1014 /* Allocate coherent helper block. */ 1015 sdcp->coh = devm_kzalloc(dev, sizeof(*sdcp->coh) + DCP_ALIGNMENT, 1016 GFP_KERNEL); 1017 if (!sdcp->coh) 1018 return -ENOMEM; 1019 1020 /* Re-align the structure so it fits the DCP constraints. */ 1021 sdcp->coh = PTR_ALIGN(sdcp->coh, DCP_ALIGNMENT); 1022 1023 /* DCP clock is optional, only used on some SOCs */ 1024 sdcp->dcp_clk = devm_clk_get(dev, "dcp"); 1025 if (IS_ERR(sdcp->dcp_clk)) { 1026 if (sdcp->dcp_clk != ERR_PTR(-ENOENT)) 1027 return PTR_ERR(sdcp->dcp_clk); 1028 sdcp->dcp_clk = NULL; 1029 } 1030 ret = clk_prepare_enable(sdcp->dcp_clk); 1031 if (ret) 1032 return ret; 1033 1034 /* Restart the DCP block. */ 1035 ret = stmp_reset_block(sdcp->base); 1036 if (ret) { 1037 dev_err(dev, "Failed reset\n"); 1038 goto err_disable_unprepare_clk; 1039 } 1040 1041 /* Initialize control register. */ 1042 writel(MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES | 1043 MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING | 0xf, 1044 sdcp->base + MXS_DCP_CTRL); 1045 1046 /* Enable all DCP DMA channels. */ 1047 writel(MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK, 1048 sdcp->base + MXS_DCP_CHANNELCTRL); 1049 1050 /* 1051 * We do not enable context switching. Give the context buffer a 1052 * pointer to an illegal address so if context switching is 1053 * inadvertantly enabled, the DCP will return an error instead of 1054 * trashing good memory. The DCP DMA cannot access ROM, so any ROM 1055 * address will do. 1056 */ 1057 writel(0xffff0000, sdcp->base + MXS_DCP_CONTEXT); 1058 for (i = 0; i < DCP_MAX_CHANS; i++) 1059 writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(i)); 1060 writel(0xffffffff, sdcp->base + MXS_DCP_STAT_CLR); 1061 1062 global_sdcp = sdcp; 1063 1064 platform_set_drvdata(pdev, sdcp); 1065 1066 for (i = 0; i < DCP_MAX_CHANS; i++) { 1067 spin_lock_init(&sdcp->lock[i]); 1068 init_completion(&sdcp->completion[i]); 1069 crypto_init_queue(&sdcp->queue[i], 50); 1070 } 1071 1072 /* Create the SHA and AES handler threads. */ 1073 sdcp->thread[DCP_CHAN_HASH_SHA] = kthread_run(dcp_chan_thread_sha, 1074 NULL, "mxs_dcp_chan/sha"); 1075 if (IS_ERR(sdcp->thread[DCP_CHAN_HASH_SHA])) { 1076 dev_err(dev, "Error starting SHA thread!\n"); 1077 ret = PTR_ERR(sdcp->thread[DCP_CHAN_HASH_SHA]); 1078 goto err_disable_unprepare_clk; 1079 } 1080 1081 sdcp->thread[DCP_CHAN_CRYPTO] = kthread_run(dcp_chan_thread_aes, 1082 NULL, "mxs_dcp_chan/aes"); 1083 if (IS_ERR(sdcp->thread[DCP_CHAN_CRYPTO])) { 1084 dev_err(dev, "Error starting SHA thread!\n"); 1085 ret = PTR_ERR(sdcp->thread[DCP_CHAN_CRYPTO]); 1086 goto err_destroy_sha_thread; 1087 } 1088 1089 /* Register the various crypto algorithms. */ 1090 sdcp->caps = readl(sdcp->base + MXS_DCP_CAPABILITY1); 1091 1092 if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) { 1093 ret = crypto_register_skciphers(dcp_aes_algs, 1094 ARRAY_SIZE(dcp_aes_algs)); 1095 if (ret) { 1096 /* Failed to register algorithm. */ 1097 dev_err(dev, "Failed to register AES crypto!\n"); 1098 goto err_destroy_aes_thread; 1099 } 1100 } 1101 1102 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) { 1103 ret = crypto_register_ahash(&dcp_sha1_alg); 1104 if (ret) { 1105 dev_err(dev, "Failed to register %s hash!\n", 1106 dcp_sha1_alg.halg.base.cra_name); 1107 goto err_unregister_aes; 1108 } 1109 } 1110 1111 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) { 1112 ret = crypto_register_ahash(&dcp_sha256_alg); 1113 if (ret) { 1114 dev_err(dev, "Failed to register %s hash!\n", 1115 dcp_sha256_alg.halg.base.cra_name); 1116 goto err_unregister_sha1; 1117 } 1118 } 1119 1120 return 0; 1121 1122err_unregister_sha1: 1123 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) 1124 crypto_unregister_ahash(&dcp_sha1_alg); 1125 1126err_unregister_aes: 1127 if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) 1128 crypto_unregister_skciphers(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); 1129 1130err_destroy_aes_thread: 1131 kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); 1132 1133err_destroy_sha_thread: 1134 kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); 1135 1136err_disable_unprepare_clk: 1137 clk_disable_unprepare(sdcp->dcp_clk); 1138 1139 return ret; 1140} 1141 1142static int mxs_dcp_remove(struct platform_device *pdev) 1143{ 1144 struct dcp *sdcp = platform_get_drvdata(pdev); 1145 1146 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) 1147 crypto_unregister_ahash(&dcp_sha256_alg); 1148 1149 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) 1150 crypto_unregister_ahash(&dcp_sha1_alg); 1151 1152 if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) 1153 crypto_unregister_skciphers(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); 1154 1155 kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); 1156 kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); 1157 1158 clk_disable_unprepare(sdcp->dcp_clk); 1159 1160 platform_set_drvdata(pdev, NULL); 1161 1162 global_sdcp = NULL; 1163 1164 return 0; 1165} 1166 1167static const struct of_device_id mxs_dcp_dt_ids[] = { 1168 { .compatible = "fsl,imx23-dcp", .data = NULL, }, 1169 { .compatible = "fsl,imx28-dcp", .data = NULL, }, 1170 { /* sentinel */ } 1171}; 1172 1173MODULE_DEVICE_TABLE(of, mxs_dcp_dt_ids); 1174 1175static struct platform_driver mxs_dcp_driver = { 1176 .probe = mxs_dcp_probe, 1177 .remove = mxs_dcp_remove, 1178 .driver = { 1179 .name = "mxs-dcp", 1180 .of_match_table = mxs_dcp_dt_ids, 1181 }, 1182}; 1183 1184module_platform_driver(mxs_dcp_driver); 1185 1186MODULE_AUTHOR("Marek Vasut <marex@denx.de>"); 1187MODULE_DESCRIPTION("Freescale MXS DCP Driver"); 1188MODULE_LICENSE("GPL"); 1189MODULE_ALIAS("platform:mxs-dcp"); 1190