1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Cryptographic API. 4 * 5 * Support for ATMEL AES HW acceleration. 6 * 7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL 8 * Author: Nicolas Royer <nicolas@eukrea.com> 9 * 10 * Some ideas are from omap-aes.c driver. 11 */ 12 13 14#include <linux/kernel.h> 15#include <linux/module.h> 16#include <linux/slab.h> 17#include <linux/err.h> 18#include <linux/clk.h> 19#include <linux/io.h> 20#include <linux/hw_random.h> 21#include <linux/platform_device.h> 22 23#include <linux/device.h> 24#include <linux/dmaengine.h> 25#include <linux/init.h> 26#include <linux/errno.h> 27#include <linux/interrupt.h> 28#include <linux/irq.h> 29#include <linux/scatterlist.h> 30#include <linux/dma-mapping.h> 31#include <linux/of_device.h> 32#include <linux/delay.h> 33#include <linux/crypto.h> 34#include <crypto/scatterwalk.h> 35#include <crypto/algapi.h> 36#include <crypto/aes.h> 37#include <crypto/gcm.h> 38#include <crypto/xts.h> 39#include <crypto/internal/aead.h> 40#include <crypto/internal/skcipher.h> 41#include "atmel-aes-regs.h" 42#include "atmel-authenc.h" 43 44#define ATMEL_AES_PRIORITY 300 45 46#define ATMEL_AES_BUFFER_ORDER 2 47#define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) 48 49#define CFB8_BLOCK_SIZE 1 50#define CFB16_BLOCK_SIZE 2 51#define CFB32_BLOCK_SIZE 4 52#define CFB64_BLOCK_SIZE 8 53 54#define SIZE_IN_WORDS(x) ((x) >> 2) 55 56/* AES flags */ 57/* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ 58#define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC 59#define AES_FLAGS_GTAGEN AES_MR_GTAGEN 60#define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) 61#define AES_FLAGS_ECB AES_MR_OPMOD_ECB 62#define AES_FLAGS_CBC AES_MR_OPMOD_CBC 63#define AES_FLAGS_OFB AES_MR_OPMOD_OFB 64#define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b) 65#define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b) 66#define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b) 67#define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b) 68#define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b) 69#define AES_FLAGS_CTR AES_MR_OPMOD_CTR 70#define AES_FLAGS_GCM AES_MR_OPMOD_GCM 71#define AES_FLAGS_XTS AES_MR_OPMOD_XTS 72 73#define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \ 74 AES_FLAGS_ENCRYPT | \ 75 AES_FLAGS_GTAGEN) 76 77#define AES_FLAGS_BUSY BIT(3) 78#define AES_FLAGS_DUMP_REG BIT(4) 79#define AES_FLAGS_OWN_SHA BIT(5) 80 81#define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY 82 83#define ATMEL_AES_QUEUE_LENGTH 50 84 85#define ATMEL_AES_DMA_THRESHOLD 256 86 87 88struct atmel_aes_caps { 89 bool has_dualbuff; 90 bool has_cfb64; 91 bool has_gcm; 92 bool has_xts; 93 bool has_authenc; 94 u32 max_burst_size; 95}; 96 97struct atmel_aes_dev; 98 99 100typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *); 101 102 103struct atmel_aes_base_ctx { 104 struct atmel_aes_dev *dd; 105 atmel_aes_fn_t start; 106 int keylen; 107 u32 key[AES_KEYSIZE_256 / sizeof(u32)]; 108 u16 block_size; 109 bool is_aead; 110}; 111 112struct atmel_aes_ctx { 113 struct atmel_aes_base_ctx base; 114}; 115 116struct atmel_aes_ctr_ctx { 117 struct atmel_aes_base_ctx base; 118 119 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 120 size_t offset; 121 struct scatterlist src[2]; 122 struct scatterlist dst[2]; 123 u32 blocks; 124}; 125 126struct atmel_aes_gcm_ctx { 127 struct atmel_aes_base_ctx base; 128 129 struct scatterlist src[2]; 130 struct scatterlist dst[2]; 131 132 __be32 j0[AES_BLOCK_SIZE / sizeof(u32)]; 133 u32 tag[AES_BLOCK_SIZE / sizeof(u32)]; 134 __be32 ghash[AES_BLOCK_SIZE / sizeof(u32)]; 135 size_t textlen; 136 137 const __be32 *ghash_in; 138 __be32 *ghash_out; 139 atmel_aes_fn_t ghash_resume; 140}; 141 142struct atmel_aes_xts_ctx { 143 struct atmel_aes_base_ctx base; 144 145 u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; 146}; 147 148#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 149struct atmel_aes_authenc_ctx { 150 struct atmel_aes_base_ctx base; 151 struct atmel_sha_authenc_ctx *auth; 152}; 153#endif 154 155struct atmel_aes_reqctx { 156 unsigned long mode; 157 u8 lastc[AES_BLOCK_SIZE]; 158}; 159 160#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 161struct atmel_aes_authenc_reqctx { 162 struct atmel_aes_reqctx base; 163 164 struct scatterlist src[2]; 165 struct scatterlist dst[2]; 166 size_t textlen; 167 u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)]; 168 169 /* auth_req MUST be place last. */ 170 struct ahash_request auth_req; 171}; 172#endif 173 174struct atmel_aes_dma { 175 struct dma_chan *chan; 176 struct scatterlist *sg; 177 int nents; 178 unsigned int remainder; 179 unsigned int sg_len; 180}; 181 182struct atmel_aes_dev { 183 struct list_head list; 184 unsigned long phys_base; 185 void __iomem *io_base; 186 187 struct crypto_async_request *areq; 188 struct atmel_aes_base_ctx *ctx; 189 190 bool is_async; 191 atmel_aes_fn_t resume; 192 atmel_aes_fn_t cpu_transfer_complete; 193 194 struct device *dev; 195 struct clk *iclk; 196 int irq; 197 198 unsigned long flags; 199 200 spinlock_t lock; 201 struct crypto_queue queue; 202 203 struct tasklet_struct done_task; 204 struct tasklet_struct queue_task; 205 206 size_t total; 207 size_t datalen; 208 u32 *data; 209 210 struct atmel_aes_dma src; 211 struct atmel_aes_dma dst; 212 213 size_t buflen; 214 void *buf; 215 struct scatterlist aligned_sg; 216 struct scatterlist *real_dst; 217 218 struct atmel_aes_caps caps; 219 220 u32 hw_version; 221}; 222 223struct atmel_aes_drv { 224 struct list_head dev_list; 225 spinlock_t lock; 226}; 227 228static struct atmel_aes_drv atmel_aes = { 229 .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list), 230 .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), 231}; 232 233#ifdef VERBOSE_DEBUG 234static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) 235{ 236 switch (offset) { 237 case AES_CR: 238 return "CR"; 239 240 case AES_MR: 241 return "MR"; 242 243 case AES_ISR: 244 return "ISR"; 245 246 case AES_IMR: 247 return "IMR"; 248 249 case AES_IER: 250 return "IER"; 251 252 case AES_IDR: 253 return "IDR"; 254 255 case AES_KEYWR(0): 256 case AES_KEYWR(1): 257 case AES_KEYWR(2): 258 case AES_KEYWR(3): 259 case AES_KEYWR(4): 260 case AES_KEYWR(5): 261 case AES_KEYWR(6): 262 case AES_KEYWR(7): 263 snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); 264 break; 265 266 case AES_IDATAR(0): 267 case AES_IDATAR(1): 268 case AES_IDATAR(2): 269 case AES_IDATAR(3): 270 snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); 271 break; 272 273 case AES_ODATAR(0): 274 case AES_ODATAR(1): 275 case AES_ODATAR(2): 276 case AES_ODATAR(3): 277 snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); 278 break; 279 280 case AES_IVR(0): 281 case AES_IVR(1): 282 case AES_IVR(2): 283 case AES_IVR(3): 284 snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); 285 break; 286 287 case AES_AADLENR: 288 return "AADLENR"; 289 290 case AES_CLENR: 291 return "CLENR"; 292 293 case AES_GHASHR(0): 294 case AES_GHASHR(1): 295 case AES_GHASHR(2): 296 case AES_GHASHR(3): 297 snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); 298 break; 299 300 case AES_TAGR(0): 301 case AES_TAGR(1): 302 case AES_TAGR(2): 303 case AES_TAGR(3): 304 snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); 305 break; 306 307 case AES_CTRR: 308 return "CTRR"; 309 310 case AES_GCMHR(0): 311 case AES_GCMHR(1): 312 case AES_GCMHR(2): 313 case AES_GCMHR(3): 314 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); 315 break; 316 317 case AES_EMR: 318 return "EMR"; 319 320 case AES_TWR(0): 321 case AES_TWR(1): 322 case AES_TWR(2): 323 case AES_TWR(3): 324 snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2); 325 break; 326 327 case AES_ALPHAR(0): 328 case AES_ALPHAR(1): 329 case AES_ALPHAR(2): 330 case AES_ALPHAR(3): 331 snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2); 332 break; 333 334 default: 335 snprintf(tmp, sz, "0x%02x", offset); 336 break; 337 } 338 339 return tmp; 340} 341#endif /* VERBOSE_DEBUG */ 342 343/* Shared functions */ 344 345static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) 346{ 347 u32 value = readl_relaxed(dd->io_base + offset); 348 349#ifdef VERBOSE_DEBUG 350 if (dd->flags & AES_FLAGS_DUMP_REG) { 351 char tmp[16]; 352 353 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, 354 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 355 } 356#endif /* VERBOSE_DEBUG */ 357 358 return value; 359} 360 361static inline void atmel_aes_write(struct atmel_aes_dev *dd, 362 u32 offset, u32 value) 363{ 364#ifdef VERBOSE_DEBUG 365 if (dd->flags & AES_FLAGS_DUMP_REG) { 366 char tmp[16]; 367 368 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, 369 atmel_aes_reg_name(offset, tmp, sizeof(tmp))); 370 } 371#endif /* VERBOSE_DEBUG */ 372 373 writel_relaxed(value, dd->io_base + offset); 374} 375 376static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, 377 u32 *value, int count) 378{ 379 for (; count--; value++, offset += 4) 380 *value = atmel_aes_read(dd, offset); 381} 382 383static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, 384 const u32 *value, int count) 385{ 386 for (; count--; value++, offset += 4) 387 atmel_aes_write(dd, offset, *value); 388} 389 390static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, 391 void *value) 392{ 393 atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 394} 395 396static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, 397 const void *value) 398{ 399 atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); 400} 401 402static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, 403 atmel_aes_fn_t resume) 404{ 405 u32 isr = atmel_aes_read(dd, AES_ISR); 406 407 if (unlikely(isr & AES_INT_DATARDY)) 408 return resume(dd); 409 410 dd->resume = resume; 411 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 412 return -EINPROGRESS; 413} 414 415static inline size_t atmel_aes_padlen(size_t len, size_t block_size) 416{ 417 len &= block_size - 1; 418 return len ? block_size - len : 0; 419} 420 421static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_base_ctx *ctx) 422{ 423 struct atmel_aes_dev *aes_dd = NULL; 424 struct atmel_aes_dev *tmp; 425 426 spin_lock_bh(&atmel_aes.lock); 427 if (!ctx->dd) { 428 list_for_each_entry(tmp, &atmel_aes.dev_list, list) { 429 aes_dd = tmp; 430 break; 431 } 432 ctx->dd = aes_dd; 433 } else { 434 aes_dd = ctx->dd; 435 } 436 437 spin_unlock_bh(&atmel_aes.lock); 438 439 return aes_dd; 440} 441 442static int atmel_aes_hw_init(struct atmel_aes_dev *dd) 443{ 444 int err; 445 446 err = clk_enable(dd->iclk); 447 if (err) 448 return err; 449 450 atmel_aes_write(dd, AES_CR, AES_CR_SWRST); 451 atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); 452 453 return 0; 454} 455 456static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) 457{ 458 return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; 459} 460 461static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) 462{ 463 int err; 464 465 err = atmel_aes_hw_init(dd); 466 if (err) 467 return err; 468 469 dd->hw_version = atmel_aes_get_version(dd); 470 471 dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); 472 473 clk_disable(dd->iclk); 474 return 0; 475} 476 477static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, 478 const struct atmel_aes_reqctx *rctx) 479{ 480 /* Clear all but persistent flags and set request flags. */ 481 dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode; 482} 483 484static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) 485{ 486 return (dd->flags & AES_FLAGS_ENCRYPT); 487} 488 489#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 490static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); 491#endif 492 493static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd) 494{ 495 struct skcipher_request *req = skcipher_request_cast(dd->areq); 496 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 497 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 498 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 499 500 if (req->cryptlen < ivsize) 501 return; 502 503 if (rctx->mode & AES_FLAGS_ENCRYPT) { 504 scatterwalk_map_and_copy(req->iv, req->dst, 505 req->cryptlen - ivsize, ivsize, 0); 506 } else { 507 if (req->src == req->dst) 508 memcpy(req->iv, rctx->lastc, ivsize); 509 else 510 scatterwalk_map_and_copy(req->iv, req->src, 511 req->cryptlen - ivsize, 512 ivsize, 0); 513 } 514} 515 516static inline struct atmel_aes_ctr_ctx * 517atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) 518{ 519 return container_of(ctx, struct atmel_aes_ctr_ctx, base); 520} 521 522static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd) 523{ 524 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 525 struct skcipher_request *req = skcipher_request_cast(dd->areq); 526 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 527 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 528 int i; 529 530 /* 531 * The CTR transfer works in fragments of data of maximum 1 MByte 532 * because of the 16 bit CTR counter embedded in the IP. When reaching 533 * here, ctx->blocks contains the number of blocks of the last fragment 534 * processed, there is no need to explicit cast it to u16. 535 */ 536 for (i = 0; i < ctx->blocks; i++) 537 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 538 539 memcpy(req->iv, ctx->iv, ivsize); 540} 541 542static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) 543{ 544 struct skcipher_request *req = skcipher_request_cast(dd->areq); 545 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 546 547#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 548 if (dd->ctx->is_aead) 549 atmel_aes_authenc_complete(dd, err); 550#endif 551 552 clk_disable(dd->iclk); 553 dd->flags &= ~AES_FLAGS_BUSY; 554 555 if (!err && !dd->ctx->is_aead && 556 (rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB) { 557 if ((rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_CTR) 558 atmel_aes_set_iv_as_last_ciphertext_block(dd); 559 else 560 atmel_aes_ctr_update_req_iv(dd); 561 } 562 563 if (dd->is_async) 564 dd->areq->complete(dd->areq, err); 565 566 tasklet_schedule(&dd->queue_task); 567 568 return err; 569} 570 571static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma, 572 const __be32 *iv, const u32 *key, int keylen) 573{ 574 u32 valmr = 0; 575 576 /* MR register must be set before IV registers */ 577 if (keylen == AES_KEYSIZE_128) 578 valmr |= AES_MR_KEYSIZE_128; 579 else if (keylen == AES_KEYSIZE_192) 580 valmr |= AES_MR_KEYSIZE_192; 581 else 582 valmr |= AES_MR_KEYSIZE_256; 583 584 valmr |= dd->flags & AES_FLAGS_MODE_MASK; 585 586 if (use_dma) { 587 valmr |= AES_MR_SMOD_IDATAR0; 588 if (dd->caps.has_dualbuff) 589 valmr |= AES_MR_DUALBUFF; 590 } else { 591 valmr |= AES_MR_SMOD_AUTO; 592 } 593 594 atmel_aes_write(dd, AES_MR, valmr); 595 596 atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen)); 597 598 if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB) 599 atmel_aes_write_block(dd, AES_IVR(0), iv); 600} 601 602static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, 603 const __be32 *iv) 604 605{ 606 atmel_aes_write_ctrl_key(dd, use_dma, iv, 607 dd->ctx->key, dd->ctx->keylen); 608} 609 610/* CPU transfer */ 611 612static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) 613{ 614 int err = 0; 615 u32 isr; 616 617 for (;;) { 618 atmel_aes_read_block(dd, AES_ODATAR(0), dd->data); 619 dd->data += 4; 620 dd->datalen -= AES_BLOCK_SIZE; 621 622 if (dd->datalen < AES_BLOCK_SIZE) 623 break; 624 625 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 626 627 isr = atmel_aes_read(dd, AES_ISR); 628 if (!(isr & AES_INT_DATARDY)) { 629 dd->resume = atmel_aes_cpu_transfer; 630 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 631 return -EINPROGRESS; 632 } 633 } 634 635 if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 636 dd->buf, dd->total)) 637 err = -EINVAL; 638 639 if (err) 640 return atmel_aes_complete(dd, err); 641 642 return dd->cpu_transfer_complete(dd); 643} 644 645static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, 646 struct scatterlist *src, 647 struct scatterlist *dst, 648 size_t len, 649 atmel_aes_fn_t resume) 650{ 651 size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE); 652 653 if (unlikely(len == 0)) 654 return -EINVAL; 655 656 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 657 658 dd->total = len; 659 dd->real_dst = dst; 660 dd->cpu_transfer_complete = resume; 661 dd->datalen = len + padlen; 662 dd->data = (u32 *)dd->buf; 663 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 664 return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer); 665} 666 667 668/* DMA transfer */ 669 670static void atmel_aes_dma_callback(void *data); 671 672static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, 673 struct scatterlist *sg, 674 size_t len, 675 struct atmel_aes_dma *dma) 676{ 677 int nents; 678 679 if (!IS_ALIGNED(len, dd->ctx->block_size)) 680 return false; 681 682 for (nents = 0; sg; sg = sg_next(sg), ++nents) { 683 if (!IS_ALIGNED(sg->offset, sizeof(u32))) 684 return false; 685 686 if (len <= sg->length) { 687 if (!IS_ALIGNED(len, dd->ctx->block_size)) 688 return false; 689 690 dma->nents = nents+1; 691 dma->remainder = sg->length - len; 692 sg->length = len; 693 return true; 694 } 695 696 if (!IS_ALIGNED(sg->length, dd->ctx->block_size)) 697 return false; 698 699 len -= sg->length; 700 } 701 702 return false; 703} 704 705static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) 706{ 707 struct scatterlist *sg = dma->sg; 708 int nents = dma->nents; 709 710 if (!dma->remainder) 711 return; 712 713 while (--nents > 0 && sg) 714 sg = sg_next(sg); 715 716 if (!sg) 717 return; 718 719 sg->length += dma->remainder; 720} 721 722static int atmel_aes_map(struct atmel_aes_dev *dd, 723 struct scatterlist *src, 724 struct scatterlist *dst, 725 size_t len) 726{ 727 bool src_aligned, dst_aligned; 728 size_t padlen; 729 730 dd->total = len; 731 dd->src.sg = src; 732 dd->dst.sg = dst; 733 dd->real_dst = dst; 734 735 src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src); 736 if (src == dst) 737 dst_aligned = src_aligned; 738 else 739 dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst); 740 if (!src_aligned || !dst_aligned) { 741 padlen = atmel_aes_padlen(len, dd->ctx->block_size); 742 743 if (dd->buflen < len + padlen) 744 return -ENOMEM; 745 746 if (!src_aligned) { 747 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); 748 dd->src.sg = &dd->aligned_sg; 749 dd->src.nents = 1; 750 dd->src.remainder = 0; 751 } 752 753 if (!dst_aligned) { 754 dd->dst.sg = &dd->aligned_sg; 755 dd->dst.nents = 1; 756 dd->dst.remainder = 0; 757 } 758 759 sg_init_table(&dd->aligned_sg, 1); 760 sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen); 761 } 762 763 if (dd->src.sg == dd->dst.sg) { 764 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 765 DMA_BIDIRECTIONAL); 766 dd->dst.sg_len = dd->src.sg_len; 767 if (!dd->src.sg_len) 768 return -EFAULT; 769 } else { 770 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, 771 DMA_TO_DEVICE); 772 if (!dd->src.sg_len) 773 return -EFAULT; 774 775 dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents, 776 DMA_FROM_DEVICE); 777 if (!dd->dst.sg_len) { 778 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 779 DMA_TO_DEVICE); 780 return -EFAULT; 781 } 782 } 783 784 return 0; 785} 786 787static void atmel_aes_unmap(struct atmel_aes_dev *dd) 788{ 789 if (dd->src.sg == dd->dst.sg) { 790 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 791 DMA_BIDIRECTIONAL); 792 793 if (dd->src.sg != &dd->aligned_sg) 794 atmel_aes_restore_sg(&dd->src); 795 } else { 796 dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents, 797 DMA_FROM_DEVICE); 798 799 if (dd->dst.sg != &dd->aligned_sg) 800 atmel_aes_restore_sg(&dd->dst); 801 802 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, 803 DMA_TO_DEVICE); 804 805 if (dd->src.sg != &dd->aligned_sg) 806 atmel_aes_restore_sg(&dd->src); 807 } 808 809 if (dd->dst.sg == &dd->aligned_sg) 810 sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), 811 dd->buf, dd->total); 812} 813 814static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, 815 enum dma_slave_buswidth addr_width, 816 enum dma_transfer_direction dir, 817 u32 maxburst) 818{ 819 struct dma_async_tx_descriptor *desc; 820 struct dma_slave_config config; 821 dma_async_tx_callback callback; 822 struct atmel_aes_dma *dma; 823 int err; 824 825 memset(&config, 0, sizeof(config)); 826 config.src_addr_width = addr_width; 827 config.dst_addr_width = addr_width; 828 config.src_maxburst = maxburst; 829 config.dst_maxburst = maxburst; 830 831 switch (dir) { 832 case DMA_MEM_TO_DEV: 833 dma = &dd->src; 834 callback = NULL; 835 config.dst_addr = dd->phys_base + AES_IDATAR(0); 836 break; 837 838 case DMA_DEV_TO_MEM: 839 dma = &dd->dst; 840 callback = atmel_aes_dma_callback; 841 config.src_addr = dd->phys_base + AES_ODATAR(0); 842 break; 843 844 default: 845 return -EINVAL; 846 } 847 848 err = dmaengine_slave_config(dma->chan, &config); 849 if (err) 850 return err; 851 852 desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir, 853 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 854 if (!desc) 855 return -ENOMEM; 856 857 desc->callback = callback; 858 desc->callback_param = dd; 859 dmaengine_submit(desc); 860 dma_async_issue_pending(dma->chan); 861 862 return 0; 863} 864 865static int atmel_aes_dma_start(struct atmel_aes_dev *dd, 866 struct scatterlist *src, 867 struct scatterlist *dst, 868 size_t len, 869 atmel_aes_fn_t resume) 870{ 871 enum dma_slave_buswidth addr_width; 872 u32 maxburst; 873 int err; 874 875 switch (dd->ctx->block_size) { 876 case CFB8_BLOCK_SIZE: 877 addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; 878 maxburst = 1; 879 break; 880 881 case CFB16_BLOCK_SIZE: 882 addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; 883 maxburst = 1; 884 break; 885 886 case CFB32_BLOCK_SIZE: 887 case CFB64_BLOCK_SIZE: 888 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 889 maxburst = 1; 890 break; 891 892 case AES_BLOCK_SIZE: 893 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 894 maxburst = dd->caps.max_burst_size; 895 break; 896 897 default: 898 err = -EINVAL; 899 goto exit; 900 } 901 902 err = atmel_aes_map(dd, src, dst, len); 903 if (err) 904 goto exit; 905 906 dd->resume = resume; 907 908 /* Set output DMA transfer first */ 909 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM, 910 maxburst); 911 if (err) 912 goto unmap; 913 914 /* Then set input DMA transfer */ 915 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV, 916 maxburst); 917 if (err) 918 goto output_transfer_stop; 919 920 return -EINPROGRESS; 921 922output_transfer_stop: 923 dmaengine_terminate_sync(dd->dst.chan); 924unmap: 925 atmel_aes_unmap(dd); 926exit: 927 return atmel_aes_complete(dd, err); 928} 929 930static void atmel_aes_dma_callback(void *data) 931{ 932 struct atmel_aes_dev *dd = data; 933 934 atmel_aes_unmap(dd); 935 dd->is_async = true; 936 (void)dd->resume(dd); 937} 938 939static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, 940 struct crypto_async_request *new_areq) 941{ 942 struct crypto_async_request *areq, *backlog; 943 struct atmel_aes_base_ctx *ctx; 944 unsigned long flags; 945 bool start_async; 946 int err, ret = 0; 947 948 spin_lock_irqsave(&dd->lock, flags); 949 if (new_areq) 950 ret = crypto_enqueue_request(&dd->queue, new_areq); 951 if (dd->flags & AES_FLAGS_BUSY) { 952 spin_unlock_irqrestore(&dd->lock, flags); 953 return ret; 954 } 955 backlog = crypto_get_backlog(&dd->queue); 956 areq = crypto_dequeue_request(&dd->queue); 957 if (areq) 958 dd->flags |= AES_FLAGS_BUSY; 959 spin_unlock_irqrestore(&dd->lock, flags); 960 961 if (!areq) 962 return ret; 963 964 if (backlog) 965 backlog->complete(backlog, -EINPROGRESS); 966 967 ctx = crypto_tfm_ctx(areq->tfm); 968 969 dd->areq = areq; 970 dd->ctx = ctx; 971 start_async = (areq != new_areq); 972 dd->is_async = start_async; 973 974 /* WARNING: ctx->start() MAY change dd->is_async. */ 975 err = ctx->start(dd); 976 return (start_async) ? ret : err; 977} 978 979 980/* AES async block ciphers */ 981 982static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) 983{ 984 return atmel_aes_complete(dd, 0); 985} 986 987static int atmel_aes_start(struct atmel_aes_dev *dd) 988{ 989 struct skcipher_request *req = skcipher_request_cast(dd->areq); 990 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 991 bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD || 992 dd->ctx->block_size != AES_BLOCK_SIZE); 993 int err; 994 995 atmel_aes_set_mode(dd, rctx); 996 997 err = atmel_aes_hw_init(dd); 998 if (err) 999 return atmel_aes_complete(dd, err); 1000 1001 atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv); 1002 if (use_dma) 1003 return atmel_aes_dma_start(dd, req->src, req->dst, 1004 req->cryptlen, 1005 atmel_aes_transfer_complete); 1006 1007 return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, 1008 atmel_aes_transfer_complete); 1009} 1010 1011static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) 1012{ 1013 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 1014 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1015 struct scatterlist *src, *dst; 1016 size_t datalen; 1017 u32 ctr; 1018 u16 start, end; 1019 bool use_dma, fragmented = false; 1020 1021 /* Check for transfer completion. */ 1022 ctx->offset += dd->total; 1023 if (ctx->offset >= req->cryptlen) 1024 return atmel_aes_transfer_complete(dd); 1025 1026 /* Compute data length. */ 1027 datalen = req->cryptlen - ctx->offset; 1028 ctx->blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); 1029 ctr = be32_to_cpu(ctx->iv[3]); 1030 1031 /* Check 16bit counter overflow. */ 1032 start = ctr & 0xffff; 1033 end = start + ctx->blocks - 1; 1034 1035 if (ctx->blocks >> 16 || end < start) { 1036 ctr |= 0xffff; 1037 datalen = AES_BLOCK_SIZE * (0x10000 - start); 1038 fragmented = true; 1039 } 1040 1041 use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); 1042 1043 /* Jump to offset. */ 1044 src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset); 1045 dst = ((req->src == req->dst) ? src : 1046 scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset)); 1047 1048 /* Configure hardware. */ 1049 atmel_aes_write_ctrl(dd, use_dma, ctx->iv); 1050 if (unlikely(fragmented)) { 1051 /* 1052 * Increment the counter manually to cope with the hardware 1053 * counter overflow. 1054 */ 1055 ctx->iv[3] = cpu_to_be32(ctr); 1056 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); 1057 } 1058 1059 if (use_dma) 1060 return atmel_aes_dma_start(dd, src, dst, datalen, 1061 atmel_aes_ctr_transfer); 1062 1063 return atmel_aes_cpu_start(dd, src, dst, datalen, 1064 atmel_aes_ctr_transfer); 1065} 1066 1067static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) 1068{ 1069 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); 1070 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1071 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1072 int err; 1073 1074 atmel_aes_set_mode(dd, rctx); 1075 1076 err = atmel_aes_hw_init(dd); 1077 if (err) 1078 return atmel_aes_complete(dd, err); 1079 1080 memcpy(ctx->iv, req->iv, AES_BLOCK_SIZE); 1081 ctx->offset = 0; 1082 dd->total = 0; 1083 return atmel_aes_ctr_transfer(dd); 1084} 1085 1086static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) 1087{ 1088 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1089 struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); 1090 struct atmel_aes_reqctx *rctx; 1091 struct atmel_aes_dev *dd; 1092 1093 switch (mode & AES_FLAGS_OPMODE_MASK) { 1094 case AES_FLAGS_CFB8: 1095 ctx->block_size = CFB8_BLOCK_SIZE; 1096 break; 1097 1098 case AES_FLAGS_CFB16: 1099 ctx->block_size = CFB16_BLOCK_SIZE; 1100 break; 1101 1102 case AES_FLAGS_CFB32: 1103 ctx->block_size = CFB32_BLOCK_SIZE; 1104 break; 1105 1106 case AES_FLAGS_CFB64: 1107 ctx->block_size = CFB64_BLOCK_SIZE; 1108 break; 1109 1110 default: 1111 ctx->block_size = AES_BLOCK_SIZE; 1112 break; 1113 } 1114 ctx->is_aead = false; 1115 1116 dd = atmel_aes_find_dev(ctx); 1117 if (!dd) 1118 return -ENODEV; 1119 1120 rctx = skcipher_request_ctx(req); 1121 rctx->mode = mode; 1122 1123 if ((mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB && 1124 !(mode & AES_FLAGS_ENCRYPT) && req->src == req->dst) { 1125 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1126 1127 if (req->cryptlen >= ivsize) 1128 scatterwalk_map_and_copy(rctx->lastc, req->src, 1129 req->cryptlen - ivsize, 1130 ivsize, 0); 1131 } 1132 1133 return atmel_aes_handle_queue(dd, &req->base); 1134} 1135 1136static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 1137 unsigned int keylen) 1138{ 1139 struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); 1140 1141 if (keylen != AES_KEYSIZE_128 && 1142 keylen != AES_KEYSIZE_192 && 1143 keylen != AES_KEYSIZE_256) 1144 return -EINVAL; 1145 1146 memcpy(ctx->key, key, keylen); 1147 ctx->keylen = keylen; 1148 1149 return 0; 1150} 1151 1152static int atmel_aes_ecb_encrypt(struct skcipher_request *req) 1153{ 1154 return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1155} 1156 1157static int atmel_aes_ecb_decrypt(struct skcipher_request *req) 1158{ 1159 return atmel_aes_crypt(req, AES_FLAGS_ECB); 1160} 1161 1162static int atmel_aes_cbc_encrypt(struct skcipher_request *req) 1163{ 1164 return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 1165} 1166 1167static int atmel_aes_cbc_decrypt(struct skcipher_request *req) 1168{ 1169 return atmel_aes_crypt(req, AES_FLAGS_CBC); 1170} 1171 1172static int atmel_aes_ofb_encrypt(struct skcipher_request *req) 1173{ 1174 return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT); 1175} 1176 1177static int atmel_aes_ofb_decrypt(struct skcipher_request *req) 1178{ 1179 return atmel_aes_crypt(req, AES_FLAGS_OFB); 1180} 1181 1182static int atmel_aes_cfb_encrypt(struct skcipher_request *req) 1183{ 1184 return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT); 1185} 1186 1187static int atmel_aes_cfb_decrypt(struct skcipher_request *req) 1188{ 1189 return atmel_aes_crypt(req, AES_FLAGS_CFB128); 1190} 1191 1192static int atmel_aes_cfb64_encrypt(struct skcipher_request *req) 1193{ 1194 return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT); 1195} 1196 1197static int atmel_aes_cfb64_decrypt(struct skcipher_request *req) 1198{ 1199 return atmel_aes_crypt(req, AES_FLAGS_CFB64); 1200} 1201 1202static int atmel_aes_cfb32_encrypt(struct skcipher_request *req) 1203{ 1204 return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT); 1205} 1206 1207static int atmel_aes_cfb32_decrypt(struct skcipher_request *req) 1208{ 1209 return atmel_aes_crypt(req, AES_FLAGS_CFB32); 1210} 1211 1212static int atmel_aes_cfb16_encrypt(struct skcipher_request *req) 1213{ 1214 return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT); 1215} 1216 1217static int atmel_aes_cfb16_decrypt(struct skcipher_request *req) 1218{ 1219 return atmel_aes_crypt(req, AES_FLAGS_CFB16); 1220} 1221 1222static int atmel_aes_cfb8_encrypt(struct skcipher_request *req) 1223{ 1224 return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT); 1225} 1226 1227static int atmel_aes_cfb8_decrypt(struct skcipher_request *req) 1228{ 1229 return atmel_aes_crypt(req, AES_FLAGS_CFB8); 1230} 1231 1232static int atmel_aes_ctr_encrypt(struct skcipher_request *req) 1233{ 1234 return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); 1235} 1236 1237static int atmel_aes_ctr_decrypt(struct skcipher_request *req) 1238{ 1239 return atmel_aes_crypt(req, AES_FLAGS_CTR); 1240} 1241 1242static int atmel_aes_init_tfm(struct crypto_skcipher *tfm) 1243{ 1244 struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 1245 1246 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1247 ctx->base.start = atmel_aes_start; 1248 1249 return 0; 1250} 1251 1252static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm) 1253{ 1254 struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm); 1255 1256 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1257 ctx->base.start = atmel_aes_ctr_start; 1258 1259 return 0; 1260} 1261 1262static struct skcipher_alg aes_algs[] = { 1263{ 1264 .base.cra_name = "ecb(aes)", 1265 .base.cra_driver_name = "atmel-ecb-aes", 1266 .base.cra_blocksize = AES_BLOCK_SIZE, 1267 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1268 1269 .init = atmel_aes_init_tfm, 1270 .min_keysize = AES_MIN_KEY_SIZE, 1271 .max_keysize = AES_MAX_KEY_SIZE, 1272 .setkey = atmel_aes_setkey, 1273 .encrypt = atmel_aes_ecb_encrypt, 1274 .decrypt = atmel_aes_ecb_decrypt, 1275}, 1276{ 1277 .base.cra_name = "cbc(aes)", 1278 .base.cra_driver_name = "atmel-cbc-aes", 1279 .base.cra_blocksize = AES_BLOCK_SIZE, 1280 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1281 1282 .init = atmel_aes_init_tfm, 1283 .min_keysize = AES_MIN_KEY_SIZE, 1284 .max_keysize = AES_MAX_KEY_SIZE, 1285 .setkey = atmel_aes_setkey, 1286 .encrypt = atmel_aes_cbc_encrypt, 1287 .decrypt = atmel_aes_cbc_decrypt, 1288 .ivsize = AES_BLOCK_SIZE, 1289}, 1290{ 1291 .base.cra_name = "ofb(aes)", 1292 .base.cra_driver_name = "atmel-ofb-aes", 1293 .base.cra_blocksize = AES_BLOCK_SIZE, 1294 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1295 1296 .init = atmel_aes_init_tfm, 1297 .min_keysize = AES_MIN_KEY_SIZE, 1298 .max_keysize = AES_MAX_KEY_SIZE, 1299 .setkey = atmel_aes_setkey, 1300 .encrypt = atmel_aes_ofb_encrypt, 1301 .decrypt = atmel_aes_ofb_decrypt, 1302 .ivsize = AES_BLOCK_SIZE, 1303}, 1304{ 1305 .base.cra_name = "cfb(aes)", 1306 .base.cra_driver_name = "atmel-cfb-aes", 1307 .base.cra_blocksize = AES_BLOCK_SIZE, 1308 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1309 1310 .init = atmel_aes_init_tfm, 1311 .min_keysize = AES_MIN_KEY_SIZE, 1312 .max_keysize = AES_MAX_KEY_SIZE, 1313 .setkey = atmel_aes_setkey, 1314 .encrypt = atmel_aes_cfb_encrypt, 1315 .decrypt = atmel_aes_cfb_decrypt, 1316 .ivsize = AES_BLOCK_SIZE, 1317}, 1318{ 1319 .base.cra_name = "cfb32(aes)", 1320 .base.cra_driver_name = "atmel-cfb32-aes", 1321 .base.cra_blocksize = CFB32_BLOCK_SIZE, 1322 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1323 1324 .init = atmel_aes_init_tfm, 1325 .min_keysize = AES_MIN_KEY_SIZE, 1326 .max_keysize = AES_MAX_KEY_SIZE, 1327 .setkey = atmel_aes_setkey, 1328 .encrypt = atmel_aes_cfb32_encrypt, 1329 .decrypt = atmel_aes_cfb32_decrypt, 1330 .ivsize = AES_BLOCK_SIZE, 1331}, 1332{ 1333 .base.cra_name = "cfb16(aes)", 1334 .base.cra_driver_name = "atmel-cfb16-aes", 1335 .base.cra_blocksize = CFB16_BLOCK_SIZE, 1336 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1337 1338 .init = atmel_aes_init_tfm, 1339 .min_keysize = AES_MIN_KEY_SIZE, 1340 .max_keysize = AES_MAX_KEY_SIZE, 1341 .setkey = atmel_aes_setkey, 1342 .encrypt = atmel_aes_cfb16_encrypt, 1343 .decrypt = atmel_aes_cfb16_decrypt, 1344 .ivsize = AES_BLOCK_SIZE, 1345}, 1346{ 1347 .base.cra_name = "cfb8(aes)", 1348 .base.cra_driver_name = "atmel-cfb8-aes", 1349 .base.cra_blocksize = CFB8_BLOCK_SIZE, 1350 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1351 1352 .init = atmel_aes_init_tfm, 1353 .min_keysize = AES_MIN_KEY_SIZE, 1354 .max_keysize = AES_MAX_KEY_SIZE, 1355 .setkey = atmel_aes_setkey, 1356 .encrypt = atmel_aes_cfb8_encrypt, 1357 .decrypt = atmel_aes_cfb8_decrypt, 1358 .ivsize = AES_BLOCK_SIZE, 1359}, 1360{ 1361 .base.cra_name = "ctr(aes)", 1362 .base.cra_driver_name = "atmel-ctr-aes", 1363 .base.cra_blocksize = 1, 1364 .base.cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx), 1365 1366 .init = atmel_aes_ctr_init_tfm, 1367 .min_keysize = AES_MIN_KEY_SIZE, 1368 .max_keysize = AES_MAX_KEY_SIZE, 1369 .setkey = atmel_aes_setkey, 1370 .encrypt = atmel_aes_ctr_encrypt, 1371 .decrypt = atmel_aes_ctr_decrypt, 1372 .ivsize = AES_BLOCK_SIZE, 1373}, 1374}; 1375 1376static struct skcipher_alg aes_cfb64_alg = { 1377 .base.cra_name = "cfb64(aes)", 1378 .base.cra_driver_name = "atmel-cfb64-aes", 1379 .base.cra_blocksize = CFB64_BLOCK_SIZE, 1380 .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), 1381 1382 .init = atmel_aes_init_tfm, 1383 .min_keysize = AES_MIN_KEY_SIZE, 1384 .max_keysize = AES_MAX_KEY_SIZE, 1385 .setkey = atmel_aes_setkey, 1386 .encrypt = atmel_aes_cfb64_encrypt, 1387 .decrypt = atmel_aes_cfb64_decrypt, 1388 .ivsize = AES_BLOCK_SIZE, 1389}; 1390 1391 1392/* gcm aead functions */ 1393 1394static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1395 const u32 *data, size_t datalen, 1396 const __be32 *ghash_in, __be32 *ghash_out, 1397 atmel_aes_fn_t resume); 1398static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); 1399static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); 1400 1401static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); 1402static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); 1403static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); 1404static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); 1405static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); 1406static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); 1407static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); 1408 1409static inline struct atmel_aes_gcm_ctx * 1410atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) 1411{ 1412 return container_of(ctx, struct atmel_aes_gcm_ctx, base); 1413} 1414 1415static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, 1416 const u32 *data, size_t datalen, 1417 const __be32 *ghash_in, __be32 *ghash_out, 1418 atmel_aes_fn_t resume) 1419{ 1420 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1421 1422 dd->data = (u32 *)data; 1423 dd->datalen = datalen; 1424 ctx->ghash_in = ghash_in; 1425 ctx->ghash_out = ghash_out; 1426 ctx->ghash_resume = resume; 1427 1428 atmel_aes_write_ctrl(dd, false, NULL); 1429 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init); 1430} 1431 1432static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) 1433{ 1434 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1435 1436 /* Set the data length. */ 1437 atmel_aes_write(dd, AES_AADLENR, dd->total); 1438 atmel_aes_write(dd, AES_CLENR, 0); 1439 1440 /* If needed, overwrite the GCM Intermediate Hash Word Registers */ 1441 if (ctx->ghash_in) 1442 atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in); 1443 1444 return atmel_aes_gcm_ghash_finalize(dd); 1445} 1446 1447static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) 1448{ 1449 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1450 u32 isr; 1451 1452 /* Write data into the Input Data Registers. */ 1453 while (dd->datalen > 0) { 1454 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1455 dd->data += 4; 1456 dd->datalen -= AES_BLOCK_SIZE; 1457 1458 isr = atmel_aes_read(dd, AES_ISR); 1459 if (!(isr & AES_INT_DATARDY)) { 1460 dd->resume = atmel_aes_gcm_ghash_finalize; 1461 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1462 return -EINPROGRESS; 1463 } 1464 } 1465 1466 /* Read the computed hash from GHASHRx. */ 1467 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out); 1468 1469 return ctx->ghash_resume(dd); 1470} 1471 1472 1473static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) 1474{ 1475 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1476 struct aead_request *req = aead_request_cast(dd->areq); 1477 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1478 struct atmel_aes_reqctx *rctx = aead_request_ctx(req); 1479 size_t ivsize = crypto_aead_ivsize(tfm); 1480 size_t datalen, padlen; 1481 const void *iv = req->iv; 1482 u8 *data = dd->buf; 1483 int err; 1484 1485 atmel_aes_set_mode(dd, rctx); 1486 1487 err = atmel_aes_hw_init(dd); 1488 if (err) 1489 return atmel_aes_complete(dd, err); 1490 1491 if (likely(ivsize == GCM_AES_IV_SIZE)) { 1492 memcpy(ctx->j0, iv, ivsize); 1493 ctx->j0[3] = cpu_to_be32(1); 1494 return atmel_aes_gcm_process(dd); 1495 } 1496 1497 padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE); 1498 datalen = ivsize + padlen + AES_BLOCK_SIZE; 1499 if (datalen > dd->buflen) 1500 return atmel_aes_complete(dd, -EINVAL); 1501 1502 memcpy(data, iv, ivsize); 1503 memset(data + ivsize, 0, padlen + sizeof(u64)); 1504 ((__be64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8); 1505 1506 return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen, 1507 NULL, ctx->j0, atmel_aes_gcm_process); 1508} 1509 1510static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) 1511{ 1512 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1513 struct aead_request *req = aead_request_cast(dd->areq); 1514 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1515 bool enc = atmel_aes_is_encrypt(dd); 1516 u32 authsize; 1517 1518 /* Compute text length. */ 1519 authsize = crypto_aead_authsize(tfm); 1520 ctx->textlen = req->cryptlen - (enc ? 0 : authsize); 1521 1522 /* 1523 * According to tcrypt test suite, the GCM Automatic Tag Generation 1524 * fails when both the message and its associated data are empty. 1525 */ 1526 if (likely(req->assoclen != 0 || ctx->textlen != 0)) 1527 dd->flags |= AES_FLAGS_GTAGEN; 1528 1529 atmel_aes_write_ctrl(dd, false, NULL); 1530 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length); 1531} 1532 1533static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) 1534{ 1535 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1536 struct aead_request *req = aead_request_cast(dd->areq); 1537 __be32 j0_lsw, *j0 = ctx->j0; 1538 size_t padlen; 1539 1540 /* Write incr32(J0) into IV. */ 1541 j0_lsw = j0[3]; 1542 be32_add_cpu(&j0[3], 1); 1543 atmel_aes_write_block(dd, AES_IVR(0), j0); 1544 j0[3] = j0_lsw; 1545 1546 /* Set aad and text lengths. */ 1547 atmel_aes_write(dd, AES_AADLENR, req->assoclen); 1548 atmel_aes_write(dd, AES_CLENR, ctx->textlen); 1549 1550 /* Check whether AAD are present. */ 1551 if (unlikely(req->assoclen == 0)) { 1552 dd->datalen = 0; 1553 return atmel_aes_gcm_data(dd); 1554 } 1555 1556 /* Copy assoc data and add padding. */ 1557 padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE); 1558 if (unlikely(req->assoclen + padlen > dd->buflen)) 1559 return atmel_aes_complete(dd, -EINVAL); 1560 sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen); 1561 1562 /* Write assoc data into the Input Data register. */ 1563 dd->data = (u32 *)dd->buf; 1564 dd->datalen = req->assoclen + padlen; 1565 return atmel_aes_gcm_data(dd); 1566} 1567 1568static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) 1569{ 1570 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1571 struct aead_request *req = aead_request_cast(dd->areq); 1572 bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD); 1573 struct scatterlist *src, *dst; 1574 u32 isr, mr; 1575 1576 /* Write AAD first. */ 1577 while (dd->datalen > 0) { 1578 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); 1579 dd->data += 4; 1580 dd->datalen -= AES_BLOCK_SIZE; 1581 1582 isr = atmel_aes_read(dd, AES_ISR); 1583 if (!(isr & AES_INT_DATARDY)) { 1584 dd->resume = atmel_aes_gcm_data; 1585 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); 1586 return -EINPROGRESS; 1587 } 1588 } 1589 1590 /* GMAC only. */ 1591 if (unlikely(ctx->textlen == 0)) 1592 return atmel_aes_gcm_tag_init(dd); 1593 1594 /* Prepare src and dst scatter lists to transfer cipher/plain texts */ 1595 src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen); 1596 dst = ((req->src == req->dst) ? src : 1597 scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen)); 1598 1599 if (use_dma) { 1600 /* Update the Mode Register for DMA transfers. */ 1601 mr = atmel_aes_read(dd, AES_MR); 1602 mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF); 1603 mr |= AES_MR_SMOD_IDATAR0; 1604 if (dd->caps.has_dualbuff) 1605 mr |= AES_MR_DUALBUFF; 1606 atmel_aes_write(dd, AES_MR, mr); 1607 1608 return atmel_aes_dma_start(dd, src, dst, ctx->textlen, 1609 atmel_aes_gcm_tag_init); 1610 } 1611 1612 return atmel_aes_cpu_start(dd, src, dst, ctx->textlen, 1613 atmel_aes_gcm_tag_init); 1614} 1615 1616static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) 1617{ 1618 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1619 struct aead_request *req = aead_request_cast(dd->areq); 1620 __be64 *data = dd->buf; 1621 1622 if (likely(dd->flags & AES_FLAGS_GTAGEN)) { 1623 if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) { 1624 dd->resume = atmel_aes_gcm_tag_init; 1625 atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY); 1626 return -EINPROGRESS; 1627 } 1628 1629 return atmel_aes_gcm_finalize(dd); 1630 } 1631 1632 /* Read the GCM Intermediate Hash Word Registers. */ 1633 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash); 1634 1635 data[0] = cpu_to_be64(req->assoclen * 8); 1636 data[1] = cpu_to_be64(ctx->textlen * 8); 1637 1638 return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE, 1639 ctx->ghash, ctx->ghash, atmel_aes_gcm_tag); 1640} 1641 1642static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) 1643{ 1644 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1645 unsigned long flags; 1646 1647 /* 1648 * Change mode to CTR to complete the tag generation. 1649 * Use J0 as Initialization Vector. 1650 */ 1651 flags = dd->flags; 1652 dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN); 1653 dd->flags |= AES_FLAGS_CTR; 1654 atmel_aes_write_ctrl(dd, false, ctx->j0); 1655 dd->flags = flags; 1656 1657 atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash); 1658 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize); 1659} 1660 1661static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) 1662{ 1663 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); 1664 struct aead_request *req = aead_request_cast(dd->areq); 1665 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1666 bool enc = atmel_aes_is_encrypt(dd); 1667 u32 offset, authsize, itag[4], *otag = ctx->tag; 1668 int err; 1669 1670 /* Read the computed tag. */ 1671 if (likely(dd->flags & AES_FLAGS_GTAGEN)) 1672 atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag); 1673 else 1674 atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag); 1675 1676 offset = req->assoclen + ctx->textlen; 1677 authsize = crypto_aead_authsize(tfm); 1678 if (enc) { 1679 scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1); 1680 err = 0; 1681 } else { 1682 scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0); 1683 err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0; 1684 } 1685 1686 return atmel_aes_complete(dd, err); 1687} 1688 1689static int atmel_aes_gcm_crypt(struct aead_request *req, 1690 unsigned long mode) 1691{ 1692 struct atmel_aes_base_ctx *ctx; 1693 struct atmel_aes_reqctx *rctx; 1694 struct atmel_aes_dev *dd; 1695 1696 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); 1697 ctx->block_size = AES_BLOCK_SIZE; 1698 ctx->is_aead = true; 1699 1700 dd = atmel_aes_find_dev(ctx); 1701 if (!dd) 1702 return -ENODEV; 1703 1704 rctx = aead_request_ctx(req); 1705 rctx->mode = AES_FLAGS_GCM | mode; 1706 1707 return atmel_aes_handle_queue(dd, &req->base); 1708} 1709 1710static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, 1711 unsigned int keylen) 1712{ 1713 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 1714 1715 if (keylen != AES_KEYSIZE_256 && 1716 keylen != AES_KEYSIZE_192 && 1717 keylen != AES_KEYSIZE_128) 1718 return -EINVAL; 1719 1720 memcpy(ctx->key, key, keylen); 1721 ctx->keylen = keylen; 1722 1723 return 0; 1724} 1725 1726static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, 1727 unsigned int authsize) 1728{ 1729 return crypto_gcm_check_authsize(authsize); 1730} 1731 1732static int atmel_aes_gcm_encrypt(struct aead_request *req) 1733{ 1734 return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); 1735} 1736 1737static int atmel_aes_gcm_decrypt(struct aead_request *req) 1738{ 1739 return atmel_aes_gcm_crypt(req, 0); 1740} 1741 1742static int atmel_aes_gcm_init(struct crypto_aead *tfm) 1743{ 1744 struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); 1745 1746 crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1747 ctx->base.start = atmel_aes_gcm_start; 1748 1749 return 0; 1750} 1751 1752static struct aead_alg aes_gcm_alg = { 1753 .setkey = atmel_aes_gcm_setkey, 1754 .setauthsize = atmel_aes_gcm_setauthsize, 1755 .encrypt = atmel_aes_gcm_encrypt, 1756 .decrypt = atmel_aes_gcm_decrypt, 1757 .init = atmel_aes_gcm_init, 1758 .ivsize = GCM_AES_IV_SIZE, 1759 .maxauthsize = AES_BLOCK_SIZE, 1760 1761 .base = { 1762 .cra_name = "gcm(aes)", 1763 .cra_driver_name = "atmel-gcm-aes", 1764 .cra_blocksize = 1, 1765 .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx), 1766 }, 1767}; 1768 1769 1770/* xts functions */ 1771 1772static inline struct atmel_aes_xts_ctx * 1773atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx) 1774{ 1775 return container_of(ctx, struct atmel_aes_xts_ctx, base); 1776} 1777 1778static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd); 1779 1780static int atmel_aes_xts_start(struct atmel_aes_dev *dd) 1781{ 1782 struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx); 1783 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1784 struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req); 1785 unsigned long flags; 1786 int err; 1787 1788 atmel_aes_set_mode(dd, rctx); 1789 1790 err = atmel_aes_hw_init(dd); 1791 if (err) 1792 return atmel_aes_complete(dd, err); 1793 1794 /* Compute the tweak value from req->iv with ecb(aes). */ 1795 flags = dd->flags; 1796 dd->flags &= ~AES_FLAGS_MODE_MASK; 1797 dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); 1798 atmel_aes_write_ctrl_key(dd, false, NULL, 1799 ctx->key2, ctx->base.keylen); 1800 dd->flags = flags; 1801 1802 atmel_aes_write_block(dd, AES_IDATAR(0), req->iv); 1803 return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data); 1804} 1805 1806static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd) 1807{ 1808 struct skcipher_request *req = skcipher_request_cast(dd->areq); 1809 bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD); 1810 u32 tweak[AES_BLOCK_SIZE / sizeof(u32)]; 1811 static const __le32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), }; 1812 u8 *tweak_bytes = (u8 *)tweak; 1813 int i; 1814 1815 /* Read the computed ciphered tweak value. */ 1816 atmel_aes_read_block(dd, AES_ODATAR(0), tweak); 1817 /* 1818 * Hardware quirk: 1819 * the order of the ciphered tweak bytes need to be reversed before 1820 * writing them into the ODATARx registers. 1821 */ 1822 for (i = 0; i < AES_BLOCK_SIZE/2; ++i) { 1823 u8 tmp = tweak_bytes[AES_BLOCK_SIZE - 1 - i]; 1824 1825 tweak_bytes[AES_BLOCK_SIZE - 1 - i] = tweak_bytes[i]; 1826 tweak_bytes[i] = tmp; 1827 } 1828 1829 /* Process the data. */ 1830 atmel_aes_write_ctrl(dd, use_dma, NULL); 1831 atmel_aes_write_block(dd, AES_TWR(0), tweak); 1832 atmel_aes_write_block(dd, AES_ALPHAR(0), one); 1833 if (use_dma) 1834 return atmel_aes_dma_start(dd, req->src, req->dst, 1835 req->cryptlen, 1836 atmel_aes_transfer_complete); 1837 1838 return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen, 1839 atmel_aes_transfer_complete); 1840} 1841 1842static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, 1843 unsigned int keylen) 1844{ 1845 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1846 int err; 1847 1848 err = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen); 1849 if (err) 1850 return err; 1851 1852 memcpy(ctx->base.key, key, keylen/2); 1853 memcpy(ctx->key2, key + keylen/2, keylen/2); 1854 ctx->base.keylen = keylen/2; 1855 1856 return 0; 1857} 1858 1859static int atmel_aes_xts_encrypt(struct skcipher_request *req) 1860{ 1861 return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT); 1862} 1863 1864static int atmel_aes_xts_decrypt(struct skcipher_request *req) 1865{ 1866 return atmel_aes_crypt(req, AES_FLAGS_XTS); 1867} 1868 1869static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm) 1870{ 1871 struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 1872 1873 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); 1874 ctx->base.start = atmel_aes_xts_start; 1875 1876 return 0; 1877} 1878 1879static struct skcipher_alg aes_xts_alg = { 1880 .base.cra_name = "xts(aes)", 1881 .base.cra_driver_name = "atmel-xts-aes", 1882 .base.cra_blocksize = AES_BLOCK_SIZE, 1883 .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx), 1884 1885 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1886 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1887 .ivsize = AES_BLOCK_SIZE, 1888 .setkey = atmel_aes_xts_setkey, 1889 .encrypt = atmel_aes_xts_encrypt, 1890 .decrypt = atmel_aes_xts_decrypt, 1891 .init = atmel_aes_xts_init_tfm, 1892}; 1893 1894#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 1895/* authenc aead functions */ 1896 1897static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); 1898static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1899 bool is_async); 1900static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1901 bool is_async); 1902static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd); 1903static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 1904 bool is_async); 1905 1906static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err) 1907{ 1908 struct aead_request *req = aead_request_cast(dd->areq); 1909 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1910 1911 if (err && (dd->flags & AES_FLAGS_OWN_SHA)) 1912 atmel_sha_authenc_abort(&rctx->auth_req); 1913 dd->flags &= ~AES_FLAGS_OWN_SHA; 1914} 1915 1916static int atmel_aes_authenc_start(struct atmel_aes_dev *dd) 1917{ 1918 struct aead_request *req = aead_request_cast(dd->areq); 1919 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1920 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1921 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 1922 int err; 1923 1924 atmel_aes_set_mode(dd, &rctx->base); 1925 1926 err = atmel_aes_hw_init(dd); 1927 if (err) 1928 return atmel_aes_complete(dd, err); 1929 1930 return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth, 1931 atmel_aes_authenc_init, dd); 1932} 1933 1934static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err, 1935 bool is_async) 1936{ 1937 struct aead_request *req = aead_request_cast(dd->areq); 1938 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1939 1940 if (is_async) 1941 dd->is_async = true; 1942 if (err) 1943 return atmel_aes_complete(dd, err); 1944 1945 /* If here, we've got the ownership of the SHA device. */ 1946 dd->flags |= AES_FLAGS_OWN_SHA; 1947 1948 /* Configure the SHA device. */ 1949 return atmel_sha_authenc_init(&rctx->auth_req, 1950 req->src, req->assoclen, 1951 rctx->textlen, 1952 atmel_aes_authenc_transfer, dd); 1953} 1954 1955static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err, 1956 bool is_async) 1957{ 1958 struct aead_request *req = aead_request_cast(dd->areq); 1959 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 1960 bool enc = atmel_aes_is_encrypt(dd); 1961 struct scatterlist *src, *dst; 1962 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; 1963 u32 emr; 1964 1965 if (is_async) 1966 dd->is_async = true; 1967 if (err) 1968 return atmel_aes_complete(dd, err); 1969 1970 /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */ 1971 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); 1972 dst = src; 1973 1974 if (req->src != req->dst) 1975 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); 1976 1977 /* Configure the AES device. */ 1978 memcpy(iv, req->iv, sizeof(iv)); 1979 1980 /* 1981 * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to 1982 * 'true' even if the data transfer is actually performed by the CPU (so 1983 * not by the DMA) because we must force the AES_MR_SMOD bitfield to the 1984 * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD 1985 * must be set to *_MR_SMOD_IDATAR0. 1986 */ 1987 atmel_aes_write_ctrl(dd, true, iv); 1988 emr = AES_EMR_PLIPEN; 1989 if (!enc) 1990 emr |= AES_EMR_PLIPD; 1991 atmel_aes_write(dd, AES_EMR, emr); 1992 1993 /* Transfer data. */ 1994 return atmel_aes_dma_start(dd, src, dst, rctx->textlen, 1995 atmel_aes_authenc_digest); 1996} 1997 1998static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd) 1999{ 2000 struct aead_request *req = aead_request_cast(dd->areq); 2001 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2002 2003 /* atmel_sha_authenc_final() releases the SHA device. */ 2004 dd->flags &= ~AES_FLAGS_OWN_SHA; 2005 return atmel_sha_authenc_final(&rctx->auth_req, 2006 rctx->digest, sizeof(rctx->digest), 2007 atmel_aes_authenc_final, dd); 2008} 2009 2010static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err, 2011 bool is_async) 2012{ 2013 struct aead_request *req = aead_request_cast(dd->areq); 2014 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2015 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2016 bool enc = atmel_aes_is_encrypt(dd); 2017 u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest; 2018 u32 offs, authsize; 2019 2020 if (is_async) 2021 dd->is_async = true; 2022 if (err) 2023 goto complete; 2024 2025 offs = req->assoclen + rctx->textlen; 2026 authsize = crypto_aead_authsize(tfm); 2027 if (enc) { 2028 scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1); 2029 } else { 2030 scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0); 2031 if (crypto_memneq(idigest, odigest, authsize)) 2032 err = -EBADMSG; 2033 } 2034 2035complete: 2036 return atmel_aes_complete(dd, err); 2037} 2038 2039static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key, 2040 unsigned int keylen) 2041{ 2042 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 2043 struct crypto_authenc_keys keys; 2044 int err; 2045 2046 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 2047 goto badkey; 2048 2049 if (keys.enckeylen > sizeof(ctx->base.key)) 2050 goto badkey; 2051 2052 /* Save auth key. */ 2053 err = atmel_sha_authenc_setkey(ctx->auth, 2054 keys.authkey, keys.authkeylen, 2055 crypto_aead_get_flags(tfm)); 2056 if (err) { 2057 memzero_explicit(&keys, sizeof(keys)); 2058 return err; 2059 } 2060 2061 /* Save enc key. */ 2062 ctx->base.keylen = keys.enckeylen; 2063 memcpy(ctx->base.key, keys.enckey, keys.enckeylen); 2064 2065 memzero_explicit(&keys, sizeof(keys)); 2066 return 0; 2067 2068badkey: 2069 memzero_explicit(&keys, sizeof(keys)); 2070 return -EINVAL; 2071} 2072 2073static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm, 2074 unsigned long auth_mode) 2075{ 2076 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 2077 unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize(); 2078 2079 ctx->auth = atmel_sha_authenc_spawn(auth_mode); 2080 if (IS_ERR(ctx->auth)) 2081 return PTR_ERR(ctx->auth); 2082 2083 crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) + 2084 auth_reqsize)); 2085 ctx->base.start = atmel_aes_authenc_start; 2086 2087 return 0; 2088} 2089 2090static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm) 2091{ 2092 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1); 2093} 2094 2095static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm) 2096{ 2097 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224); 2098} 2099 2100static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm) 2101{ 2102 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256); 2103} 2104 2105static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm) 2106{ 2107 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384); 2108} 2109 2110static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm) 2111{ 2112 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512); 2113} 2114 2115static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm) 2116{ 2117 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm); 2118 2119 atmel_sha_authenc_free(ctx->auth); 2120} 2121 2122static int atmel_aes_authenc_crypt(struct aead_request *req, 2123 unsigned long mode) 2124{ 2125 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req); 2126 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2127 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); 2128 u32 authsize = crypto_aead_authsize(tfm); 2129 bool enc = (mode & AES_FLAGS_ENCRYPT); 2130 struct atmel_aes_dev *dd; 2131 2132 /* Compute text length. */ 2133 if (!enc && req->cryptlen < authsize) 2134 return -EINVAL; 2135 rctx->textlen = req->cryptlen - (enc ? 0 : authsize); 2136 2137 /* 2138 * Currently, empty messages are not supported yet: 2139 * the SHA auto-padding can be used only on non-empty messages. 2140 * Hence a special case needs to be implemented for empty message. 2141 */ 2142 if (!rctx->textlen && !req->assoclen) 2143 return -EINVAL; 2144 2145 rctx->base.mode = mode; 2146 ctx->block_size = AES_BLOCK_SIZE; 2147 ctx->is_aead = true; 2148 2149 dd = atmel_aes_find_dev(ctx); 2150 if (!dd) 2151 return -ENODEV; 2152 2153 return atmel_aes_handle_queue(dd, &req->base); 2154} 2155 2156static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req) 2157{ 2158 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); 2159} 2160 2161static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req) 2162{ 2163 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC); 2164} 2165 2166static struct aead_alg aes_authenc_algs[] = { 2167{ 2168 .setkey = atmel_aes_authenc_setkey, 2169 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2170 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2171 .init = atmel_aes_authenc_hmac_sha1_init_tfm, 2172 .exit = atmel_aes_authenc_exit_tfm, 2173 .ivsize = AES_BLOCK_SIZE, 2174 .maxauthsize = SHA1_DIGEST_SIZE, 2175 2176 .base = { 2177 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2178 .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes", 2179 .cra_blocksize = AES_BLOCK_SIZE, 2180 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2181 }, 2182}, 2183{ 2184 .setkey = atmel_aes_authenc_setkey, 2185 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2186 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2187 .init = atmel_aes_authenc_hmac_sha224_init_tfm, 2188 .exit = atmel_aes_authenc_exit_tfm, 2189 .ivsize = AES_BLOCK_SIZE, 2190 .maxauthsize = SHA224_DIGEST_SIZE, 2191 2192 .base = { 2193 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2194 .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes", 2195 .cra_blocksize = AES_BLOCK_SIZE, 2196 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2197 }, 2198}, 2199{ 2200 .setkey = atmel_aes_authenc_setkey, 2201 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2202 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2203 .init = atmel_aes_authenc_hmac_sha256_init_tfm, 2204 .exit = atmel_aes_authenc_exit_tfm, 2205 .ivsize = AES_BLOCK_SIZE, 2206 .maxauthsize = SHA256_DIGEST_SIZE, 2207 2208 .base = { 2209 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2210 .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes", 2211 .cra_blocksize = AES_BLOCK_SIZE, 2212 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2213 }, 2214}, 2215{ 2216 .setkey = atmel_aes_authenc_setkey, 2217 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2218 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2219 .init = atmel_aes_authenc_hmac_sha384_init_tfm, 2220 .exit = atmel_aes_authenc_exit_tfm, 2221 .ivsize = AES_BLOCK_SIZE, 2222 .maxauthsize = SHA384_DIGEST_SIZE, 2223 2224 .base = { 2225 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2226 .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes", 2227 .cra_blocksize = AES_BLOCK_SIZE, 2228 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2229 }, 2230}, 2231{ 2232 .setkey = atmel_aes_authenc_setkey, 2233 .encrypt = atmel_aes_authenc_cbc_aes_encrypt, 2234 .decrypt = atmel_aes_authenc_cbc_aes_decrypt, 2235 .init = atmel_aes_authenc_hmac_sha512_init_tfm, 2236 .exit = atmel_aes_authenc_exit_tfm, 2237 .ivsize = AES_BLOCK_SIZE, 2238 .maxauthsize = SHA512_DIGEST_SIZE, 2239 2240 .base = { 2241 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2242 .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes", 2243 .cra_blocksize = AES_BLOCK_SIZE, 2244 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx), 2245 }, 2246}, 2247}; 2248#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */ 2249 2250/* Probe functions */ 2251 2252static int atmel_aes_buff_init(struct atmel_aes_dev *dd) 2253{ 2254 dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER); 2255 dd->buflen = ATMEL_AES_BUFFER_SIZE; 2256 dd->buflen &= ~(AES_BLOCK_SIZE - 1); 2257 2258 if (!dd->buf) { 2259 dev_err(dd->dev, "unable to alloc pages.\n"); 2260 return -ENOMEM; 2261 } 2262 2263 return 0; 2264} 2265 2266static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) 2267{ 2268 free_page((unsigned long)dd->buf); 2269} 2270 2271static int atmel_aes_dma_init(struct atmel_aes_dev *dd) 2272{ 2273 int ret; 2274 2275 /* Try to grab 2 DMA channels */ 2276 dd->src.chan = dma_request_chan(dd->dev, "tx"); 2277 if (IS_ERR(dd->src.chan)) { 2278 ret = PTR_ERR(dd->src.chan); 2279 goto err_dma_in; 2280 } 2281 2282 dd->dst.chan = dma_request_chan(dd->dev, "rx"); 2283 if (IS_ERR(dd->dst.chan)) { 2284 ret = PTR_ERR(dd->dst.chan); 2285 goto err_dma_out; 2286 } 2287 2288 return 0; 2289 2290err_dma_out: 2291 dma_release_channel(dd->src.chan); 2292err_dma_in: 2293 dev_err(dd->dev, "no DMA channel available\n"); 2294 return ret; 2295} 2296 2297static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) 2298{ 2299 dma_release_channel(dd->dst.chan); 2300 dma_release_channel(dd->src.chan); 2301} 2302 2303static void atmel_aes_queue_task(unsigned long data) 2304{ 2305 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2306 2307 atmel_aes_handle_queue(dd, NULL); 2308} 2309 2310static void atmel_aes_done_task(unsigned long data) 2311{ 2312 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; 2313 2314 dd->is_async = true; 2315 (void)dd->resume(dd); 2316} 2317 2318static irqreturn_t atmel_aes_irq(int irq, void *dev_id) 2319{ 2320 struct atmel_aes_dev *aes_dd = dev_id; 2321 u32 reg; 2322 2323 reg = atmel_aes_read(aes_dd, AES_ISR); 2324 if (reg & atmel_aes_read(aes_dd, AES_IMR)) { 2325 atmel_aes_write(aes_dd, AES_IDR, reg); 2326 if (AES_FLAGS_BUSY & aes_dd->flags) 2327 tasklet_schedule(&aes_dd->done_task); 2328 else 2329 dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n"); 2330 return IRQ_HANDLED; 2331 } 2332 2333 return IRQ_NONE; 2334} 2335 2336static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) 2337{ 2338 int i; 2339 2340#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2341 if (dd->caps.has_authenc) 2342 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) 2343 crypto_unregister_aead(&aes_authenc_algs[i]); 2344#endif 2345 2346 if (dd->caps.has_xts) 2347 crypto_unregister_skcipher(&aes_xts_alg); 2348 2349 if (dd->caps.has_gcm) 2350 crypto_unregister_aead(&aes_gcm_alg); 2351 2352 if (dd->caps.has_cfb64) 2353 crypto_unregister_skcipher(&aes_cfb64_alg); 2354 2355 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) 2356 crypto_unregister_skcipher(&aes_algs[i]); 2357} 2358 2359static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) 2360{ 2361 alg->cra_flags = CRYPTO_ALG_ASYNC; 2362 alg->cra_alignmask = 0xf; 2363 alg->cra_priority = ATMEL_AES_PRIORITY; 2364 alg->cra_module = THIS_MODULE; 2365} 2366 2367static int atmel_aes_register_algs(struct atmel_aes_dev *dd) 2368{ 2369 int err, i, j; 2370 2371 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 2372 atmel_aes_crypto_alg_init(&aes_algs[i].base); 2373 2374 err = crypto_register_skcipher(&aes_algs[i]); 2375 if (err) 2376 goto err_aes_algs; 2377 } 2378 2379 if (dd->caps.has_cfb64) { 2380 atmel_aes_crypto_alg_init(&aes_cfb64_alg.base); 2381 2382 err = crypto_register_skcipher(&aes_cfb64_alg); 2383 if (err) 2384 goto err_aes_cfb64_alg; 2385 } 2386 2387 if (dd->caps.has_gcm) { 2388 atmel_aes_crypto_alg_init(&aes_gcm_alg.base); 2389 2390 err = crypto_register_aead(&aes_gcm_alg); 2391 if (err) 2392 goto err_aes_gcm_alg; 2393 } 2394 2395 if (dd->caps.has_xts) { 2396 atmel_aes_crypto_alg_init(&aes_xts_alg.base); 2397 2398 err = crypto_register_skcipher(&aes_xts_alg); 2399 if (err) 2400 goto err_aes_xts_alg; 2401 } 2402 2403#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2404 if (dd->caps.has_authenc) { 2405 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) { 2406 atmel_aes_crypto_alg_init(&aes_authenc_algs[i].base); 2407 2408 err = crypto_register_aead(&aes_authenc_algs[i]); 2409 if (err) 2410 goto err_aes_authenc_alg; 2411 } 2412 } 2413#endif 2414 2415 return 0; 2416 2417#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2418 /* i = ARRAY_SIZE(aes_authenc_algs); */ 2419err_aes_authenc_alg: 2420 for (j = 0; j < i; j++) 2421 crypto_unregister_aead(&aes_authenc_algs[j]); 2422 crypto_unregister_skcipher(&aes_xts_alg); 2423#endif 2424err_aes_xts_alg: 2425 crypto_unregister_aead(&aes_gcm_alg); 2426err_aes_gcm_alg: 2427 crypto_unregister_skcipher(&aes_cfb64_alg); 2428err_aes_cfb64_alg: 2429 i = ARRAY_SIZE(aes_algs); 2430err_aes_algs: 2431 for (j = 0; j < i; j++) 2432 crypto_unregister_skcipher(&aes_algs[j]); 2433 2434 return err; 2435} 2436 2437static void atmel_aes_get_cap(struct atmel_aes_dev *dd) 2438{ 2439 dd->caps.has_dualbuff = 0; 2440 dd->caps.has_cfb64 = 0; 2441 dd->caps.has_gcm = 0; 2442 dd->caps.has_xts = 0; 2443 dd->caps.has_authenc = 0; 2444 dd->caps.max_burst_size = 1; 2445 2446 /* keep only major version number */ 2447 switch (dd->hw_version & 0xff0) { 2448 case 0x500: 2449 dd->caps.has_dualbuff = 1; 2450 dd->caps.has_cfb64 = 1; 2451 dd->caps.has_gcm = 1; 2452 dd->caps.has_xts = 1; 2453 dd->caps.has_authenc = 1; 2454 dd->caps.max_burst_size = 4; 2455 break; 2456 case 0x200: 2457 dd->caps.has_dualbuff = 1; 2458 dd->caps.has_cfb64 = 1; 2459 dd->caps.has_gcm = 1; 2460 dd->caps.max_burst_size = 4; 2461 break; 2462 case 0x130: 2463 dd->caps.has_dualbuff = 1; 2464 dd->caps.has_cfb64 = 1; 2465 dd->caps.max_burst_size = 4; 2466 break; 2467 case 0x120: 2468 break; 2469 default: 2470 dev_warn(dd->dev, 2471 "Unmanaged aes version, set minimum capabilities\n"); 2472 break; 2473 } 2474} 2475 2476#if defined(CONFIG_OF) 2477static const struct of_device_id atmel_aes_dt_ids[] = { 2478 { .compatible = "atmel,at91sam9g46-aes" }, 2479 { /* sentinel */ } 2480}; 2481MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); 2482#endif 2483 2484static int atmel_aes_probe(struct platform_device *pdev) 2485{ 2486 struct atmel_aes_dev *aes_dd; 2487 struct device *dev = &pdev->dev; 2488 struct resource *aes_res; 2489 int err; 2490 2491 aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL); 2492 if (!aes_dd) 2493 return -ENOMEM; 2494 2495 aes_dd->dev = dev; 2496 2497 platform_set_drvdata(pdev, aes_dd); 2498 2499 INIT_LIST_HEAD(&aes_dd->list); 2500 spin_lock_init(&aes_dd->lock); 2501 2502 tasklet_init(&aes_dd->done_task, atmel_aes_done_task, 2503 (unsigned long)aes_dd); 2504 tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task, 2505 (unsigned long)aes_dd); 2506 2507 crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); 2508 2509 /* Get the base address */ 2510 aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 2511 if (!aes_res) { 2512 dev_err(dev, "no MEM resource info\n"); 2513 err = -ENODEV; 2514 goto err_tasklet_kill; 2515 } 2516 aes_dd->phys_base = aes_res->start; 2517 2518 /* Get the IRQ */ 2519 aes_dd->irq = platform_get_irq(pdev, 0); 2520 if (aes_dd->irq < 0) { 2521 err = aes_dd->irq; 2522 goto err_tasklet_kill; 2523 } 2524 2525 err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq, 2526 IRQF_SHARED, "atmel-aes", aes_dd); 2527 if (err) { 2528 dev_err(dev, "unable to request aes irq.\n"); 2529 goto err_tasklet_kill; 2530 } 2531 2532 /* Initializing the clock */ 2533 aes_dd->iclk = devm_clk_get(&pdev->dev, "aes_clk"); 2534 if (IS_ERR(aes_dd->iclk)) { 2535 dev_err(dev, "clock initialization failed.\n"); 2536 err = PTR_ERR(aes_dd->iclk); 2537 goto err_tasklet_kill; 2538 } 2539 2540 aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res); 2541 if (IS_ERR(aes_dd->io_base)) { 2542 dev_err(dev, "can't ioremap\n"); 2543 err = PTR_ERR(aes_dd->io_base); 2544 goto err_tasklet_kill; 2545 } 2546 2547 err = clk_prepare(aes_dd->iclk); 2548 if (err) 2549 goto err_tasklet_kill; 2550 2551 err = atmel_aes_hw_version_init(aes_dd); 2552 if (err) 2553 goto err_iclk_unprepare; 2554 2555 atmel_aes_get_cap(aes_dd); 2556 2557#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2558 if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) { 2559 err = -EPROBE_DEFER; 2560 goto err_iclk_unprepare; 2561 } 2562#endif 2563 2564 err = atmel_aes_buff_init(aes_dd); 2565 if (err) 2566 goto err_iclk_unprepare; 2567 2568 err = atmel_aes_dma_init(aes_dd); 2569 if (err) 2570 goto err_buff_cleanup; 2571 2572 spin_lock(&atmel_aes.lock); 2573 list_add_tail(&aes_dd->list, &atmel_aes.dev_list); 2574 spin_unlock(&atmel_aes.lock); 2575 2576 err = atmel_aes_register_algs(aes_dd); 2577 if (err) 2578 goto err_algs; 2579 2580 dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", 2581 dma_chan_name(aes_dd->src.chan), 2582 dma_chan_name(aes_dd->dst.chan)); 2583 2584 return 0; 2585 2586err_algs: 2587 spin_lock(&atmel_aes.lock); 2588 list_del(&aes_dd->list); 2589 spin_unlock(&atmel_aes.lock); 2590 atmel_aes_dma_cleanup(aes_dd); 2591err_buff_cleanup: 2592 atmel_aes_buff_cleanup(aes_dd); 2593err_iclk_unprepare: 2594 clk_unprepare(aes_dd->iclk); 2595err_tasklet_kill: 2596 tasklet_kill(&aes_dd->done_task); 2597 tasklet_kill(&aes_dd->queue_task); 2598 2599 return err; 2600} 2601 2602static int atmel_aes_remove(struct platform_device *pdev) 2603{ 2604 struct atmel_aes_dev *aes_dd; 2605 2606 aes_dd = platform_get_drvdata(pdev); 2607 if (!aes_dd) 2608 return -ENODEV; 2609 spin_lock(&atmel_aes.lock); 2610 list_del(&aes_dd->list); 2611 spin_unlock(&atmel_aes.lock); 2612 2613 atmel_aes_unregister_algs(aes_dd); 2614 2615 tasklet_kill(&aes_dd->done_task); 2616 tasklet_kill(&aes_dd->queue_task); 2617 2618 atmel_aes_dma_cleanup(aes_dd); 2619 atmel_aes_buff_cleanup(aes_dd); 2620 2621 clk_unprepare(aes_dd->iclk); 2622 2623 return 0; 2624} 2625 2626static struct platform_driver atmel_aes_driver = { 2627 .probe = atmel_aes_probe, 2628 .remove = atmel_aes_remove, 2629 .driver = { 2630 .name = "atmel_aes", 2631 .of_match_table = of_match_ptr(atmel_aes_dt_ids), 2632 }, 2633}; 2634 2635module_platform_driver(atmel_aes_driver); 2636 2637MODULE_DESCRIPTION("Atmel AES hw acceleration support."); 2638MODULE_LICENSE("GPL v2"); 2639MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique"); 2640