1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru> 4 * All rights reserved. 5 */ 6 7#include <linux/kernel.h> 8#include <linux/module.h> 9#include <linux/moduleparam.h> 10#include <linux/mod_devicetable.h> 11#include <linux/interrupt.h> 12#include <linux/pci.h> 13#include <linux/slab.h> 14#include <linux/delay.h> 15#include <linux/mm.h> 16#include <linux/dma-mapping.h> 17#include <linux/scatterlist.h> 18#include <linux/highmem.h> 19#include <linux/crypto.h> 20#include <linux/hw_random.h> 21#include <linux/ktime.h> 22 23#include <crypto/algapi.h> 24#include <crypto/internal/des.h> 25#include <crypto/internal/skcipher.h> 26 27static char hifn_pll_ref[sizeof("extNNN")] = "ext"; 28module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); 29MODULE_PARM_DESC(hifn_pll_ref, 30 "PLL reference clock (pci[freq] or ext[freq], default ext)"); 31 32static atomic_t hifn_dev_number; 33 34#define ACRYPTO_OP_DECRYPT 0 35#define ACRYPTO_OP_ENCRYPT 1 36#define ACRYPTO_OP_HMAC 2 37#define ACRYPTO_OP_RNG 3 38 39#define ACRYPTO_MODE_ECB 0 40#define ACRYPTO_MODE_CBC 1 41#define ACRYPTO_MODE_CFB 2 42#define ACRYPTO_MODE_OFB 3 43 44#define ACRYPTO_TYPE_AES_128 0 45#define ACRYPTO_TYPE_AES_192 1 46#define ACRYPTO_TYPE_AES_256 2 47#define ACRYPTO_TYPE_3DES 3 48#define ACRYPTO_TYPE_DES 4 49 50#define PCI_VENDOR_ID_HIFN 0x13A3 51#define PCI_DEVICE_ID_HIFN_7955 0x0020 52#define PCI_DEVICE_ID_HIFN_7956 0x001d 53 54/* I/O region sizes */ 55 56#define HIFN_BAR0_SIZE 0x1000 57#define HIFN_BAR1_SIZE 0x2000 58#define HIFN_BAR2_SIZE 0x8000 59 60/* DMA registres */ 61 62#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ 63#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ 64#define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */ 65#define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */ 66#define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */ 67#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ 68#define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */ 69#define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */ 70#define HIFN_CHIP_ID 0x98 /* Chip ID */ 71 72/* 73 * Processing Unit Registers (offset from BASEREG0) 74 */ 75#define HIFN_0_PUDATA 0x00 /* Processing Unit Data */ 76#define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */ 77#define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */ 78#define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */ 79#define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */ 80#define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */ 81#define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */ 82#define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */ 83#define HIFN_0_SPACESIZE 0x20 /* Register space size */ 84 85/* Processing Unit Control Register (HIFN_0_PUCTRL) */ 86#define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */ 87#define HIFN_PUCTRL_STOP 0x0008 /* stop pu */ 88#define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */ 89#define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */ 90#define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */ 91 92/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */ 93#define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */ 94#define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */ 95#define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 96#define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 97#define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */ 98#define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */ 99#define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */ 100#define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */ 101#define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */ 102#define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */ 103 104/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */ 105#define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */ 106#define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */ 107#define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */ 108#define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */ 109#define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */ 110#define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */ 111#define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */ 112#define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */ 113#define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */ 114#define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */ 115#define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */ 116#define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */ 117#define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */ 118#define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */ 119#define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */ 120#define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */ 121#define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */ 122#define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */ 123#define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */ 124#define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */ 125#define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */ 126#define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */ 127#define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */ 128 129/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */ 130#define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */ 131#define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */ 132#define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 133#define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 134#define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */ 135#define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */ 136#define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */ 137#define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */ 138#define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */ 139#define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */ 140 141/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */ 142#define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */ 143#define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */ 144#define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */ 145#define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */ 146#define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */ 147#define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */ 148#define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */ 149#define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */ 150#define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */ 151#define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */ 152#define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */ 153#define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */ 154#define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */ 155#define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */ 156#define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */ 157#define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */ 158#define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */ 159 160/* FIFO Status Register (HIFN_0_FIFOSTAT) */ 161#define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */ 162#define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */ 163 164/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */ 165#define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */ 166 167/* 168 * DMA Interface Registers (offset from BASEREG1) 169 */ 170#define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */ 171#define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */ 172#define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */ 173#define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */ 174#define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */ 175#define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */ 176#define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */ 177#define HIFN_1_PLL 0x4c /* 795x: PLL config */ 178#define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */ 179#define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */ 180#define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */ 181#define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */ 182#define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */ 183#define HIFN_1_REVID 0x98 /* Revision ID */ 184#define HIFN_1_UNLOCK_SECRET1 0xf4 185#define HIFN_1_UNLOCK_SECRET2 0xfc 186#define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */ 187#define HIFN_1_PUB_BASE 0x300 /* Public Base Address */ 188#define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */ 189#define HIFN_1_PUB_OP 0x308 /* Public Operand */ 190#define HIFN_1_PUB_STATUS 0x30c /* Public Status */ 191#define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */ 192#define HIFN_1_RNG_CONFIG 0x314 /* RNG config */ 193#define HIFN_1_RNG_DATA 0x318 /* RNG data */ 194#define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */ 195#define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */ 196 197/* DMA Status and Control Register (HIFN_1_DMA_CSR) */ 198#define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */ 199#define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */ 200#define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */ 201#define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */ 202#define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */ 203#define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */ 204#define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */ 205#define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */ 206#define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */ 207#define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */ 208#define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */ 209#define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */ 210#define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */ 211#define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 212#define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */ 213#define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */ 214#define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */ 215#define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */ 216#define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */ 217#define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */ 218#define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */ 219#define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */ 220#define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 221#define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */ 222#define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */ 223#define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */ 224#define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */ 225#define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */ 226#define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */ 227#define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */ 228#define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */ 229#define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */ 230#define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 231#define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */ 232#define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */ 233#define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */ 234#define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */ 235#define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */ 236 237/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */ 238#define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */ 239#define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */ 240#define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */ 241#define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */ 242#define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */ 243#define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */ 244#define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */ 245#define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */ 246#define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */ 247#define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */ 248#define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */ 249#define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */ 250#define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */ 251#define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */ 252#define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */ 253#define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */ 254#define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */ 255#define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */ 256#define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */ 257#define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */ 258#define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */ 259#define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */ 260 261/* DMA Configuration Register (HIFN_1_DMA_CNFG) */ 262#define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */ 263#define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */ 264#define HIFN_DMACNFG_UNLOCK 0x00000800 265#define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */ 266#define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */ 267#define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */ 268#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */ 269#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */ 270 271/* PLL configuration register */ 272#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */ 273#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */ 274#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */ 275#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */ 276#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */ 277#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */ 278#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */ 279#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */ 280#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */ 281#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */ 282#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */ 283#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */ 284#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */ 285#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */ 286#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */ 287#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */ 288#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */ 289 290#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */ 291 292/* Public key reset register (HIFN_1_PUB_RESET) */ 293#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */ 294 295/* Public base address register (HIFN_1_PUB_BASE) */ 296#define HIFN_PUBBASE_ADDR 0x00003fff /* base address */ 297 298/* Public operand length register (HIFN_1_PUB_OPLEN) */ 299#define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */ 300#define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */ 301#define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */ 302#define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */ 303#define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */ 304#define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */ 305 306/* Public operation register (HIFN_1_PUB_OP) */ 307#define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */ 308#define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */ 309#define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */ 310#define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */ 311#define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */ 312#define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */ 313#define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */ 314#define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */ 315#define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */ 316#define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */ 317#define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */ 318#define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */ 319#define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */ 320#define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */ 321#define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */ 322#define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */ 323#define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */ 324#define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */ 325#define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */ 326#define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */ 327 328/* Public status register (HIFN_1_PUB_STATUS) */ 329#define HIFN_PUBSTS_DONE 0x00000001 /* operation done */ 330#define HIFN_PUBSTS_CARRY 0x00000002 /* carry */ 331 332/* Public interrupt enable register (HIFN_1_PUB_IEN) */ 333#define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */ 334 335/* Random number generator config register (HIFN_1_RNG_CONFIG) */ 336#define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */ 337 338#define HIFN_NAMESIZE 32 339#define HIFN_MAX_RESULT_ORDER 5 340 341#define HIFN_D_CMD_RSIZE (24 * 1) 342#define HIFN_D_SRC_RSIZE (80 * 1) 343#define HIFN_D_DST_RSIZE (80 * 1) 344#define HIFN_D_RES_RSIZE (24 * 1) 345 346#define HIFN_D_DST_DALIGN 4 347 348#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) 349 350#define AES_MIN_KEY_SIZE 16 351#define AES_MAX_KEY_SIZE 32 352 353#define HIFN_DES_KEY_LENGTH 8 354#define HIFN_3DES_KEY_LENGTH 24 355#define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE 356#define HIFN_IV_LENGTH 8 357#define HIFN_AES_IV_LENGTH 16 358#define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH 359 360#define HIFN_MAC_KEY_LENGTH 64 361#define HIFN_MD5_LENGTH 16 362#define HIFN_SHA1_LENGTH 20 363#define HIFN_MAC_TRUNC_LENGTH 12 364 365#define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260) 366#define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4) 367#define HIFN_USED_RESULT 12 368 369struct hifn_desc { 370 volatile __le32 l; 371 volatile __le32 p; 372}; 373 374struct hifn_dma { 375 struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1]; 376 struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1]; 377 struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1]; 378 struct hifn_desc resr[HIFN_D_RES_RSIZE + 1]; 379 380 u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; 381 u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; 382 383 /* 384 * Our current positions for insertion and removal from the descriptor 385 * rings. 386 */ 387 volatile int cmdi, srci, dsti, resi; 388 volatile int cmdu, srcu, dstu, resu; 389 int cmdk, srck, dstk, resk; 390}; 391 392#define HIFN_FLAG_CMD_BUSY (1 << 0) 393#define HIFN_FLAG_SRC_BUSY (1 << 1) 394#define HIFN_FLAG_DST_BUSY (1 << 2) 395#define HIFN_FLAG_RES_BUSY (1 << 3) 396#define HIFN_FLAG_OLD_KEY (1 << 4) 397 398#define HIFN_DEFAULT_ACTIVE_NUM 5 399 400struct hifn_device { 401 char name[HIFN_NAMESIZE]; 402 403 int irq; 404 405 struct pci_dev *pdev; 406 void __iomem *bar[3]; 407 408 void *desc_virt; 409 dma_addr_t desc_dma; 410 411 u32 dmareg; 412 413 void *sa[HIFN_D_RES_RSIZE]; 414 415 spinlock_t lock; 416 417 u32 flags; 418 int active, started; 419 struct delayed_work work; 420 unsigned long reset; 421 unsigned long success; 422 unsigned long prev_success; 423 424 u8 snum; 425 426 struct tasklet_struct tasklet; 427 428 struct crypto_queue queue; 429 struct list_head alg_list; 430 431 unsigned int pk_clk_freq; 432 433#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 434 unsigned int rng_wait_time; 435 ktime_t rngtime; 436 struct hwrng rng; 437#endif 438}; 439 440#define HIFN_D_LENGTH 0x0000ffff 441#define HIFN_D_NOINVALID 0x01000000 442#define HIFN_D_MASKDONEIRQ 0x02000000 443#define HIFN_D_DESTOVER 0x04000000 444#define HIFN_D_OVER 0x08000000 445#define HIFN_D_LAST 0x20000000 446#define HIFN_D_JUMP 0x40000000 447#define HIFN_D_VALID 0x80000000 448 449struct hifn_base_command { 450 volatile __le16 masks; 451 volatile __le16 session_num; 452 volatile __le16 total_source_count; 453 volatile __le16 total_dest_count; 454}; 455 456#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */ 457#define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */ 458#define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */ 459#define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */ 460#define HIFN_BASE_CMD_DECODE 0x2000 461#define HIFN_BASE_CMD_SRCLEN_M 0xc000 462#define HIFN_BASE_CMD_SRCLEN_S 14 463#define HIFN_BASE_CMD_DSTLEN_M 0x3000 464#define HIFN_BASE_CMD_DSTLEN_S 12 465#define HIFN_BASE_CMD_LENMASK_HI 0x30000 466#define HIFN_BASE_CMD_LENMASK_LO 0x0ffff 467 468/* 469 * Structure to help build up the command data structure. 470 */ 471struct hifn_crypt_command { 472 volatile __le16 masks; 473 volatile __le16 header_skip; 474 volatile __le16 source_count; 475 volatile __le16 reserved; 476}; 477 478#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ 479#define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */ 480#define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */ 481#define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */ 482#define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */ 483#define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */ 484#define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */ 485#define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */ 486#define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */ 487#define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */ 488#define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */ 489#define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */ 490#define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */ 491#define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */ 492#define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */ 493#define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */ 494#define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */ 495#define HIFN_CRYPT_CMD_SRCLEN_M 0xc000 496#define HIFN_CRYPT_CMD_SRCLEN_S 14 497 498/* 499 * Structure to help build up the command data structure. 500 */ 501struct hifn_mac_command { 502 volatile __le16 masks; 503 volatile __le16 header_skip; 504 volatile __le16 source_count; 505 volatile __le16 reserved; 506}; 507 508#define HIFN_MAC_CMD_ALG_MASK 0x0001 509#define HIFN_MAC_CMD_ALG_SHA1 0x0000 510#define HIFN_MAC_CMD_ALG_MD5 0x0001 511#define HIFN_MAC_CMD_MODE_MASK 0x000c 512#define HIFN_MAC_CMD_MODE_HMAC 0x0000 513#define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004 514#define HIFN_MAC_CMD_MODE_HASH 0x0008 515#define HIFN_MAC_CMD_MODE_FULL 0x0004 516#define HIFN_MAC_CMD_TRUNC 0x0010 517#define HIFN_MAC_CMD_RESULT 0x0020 518#define HIFN_MAC_CMD_APPEND 0x0040 519#define HIFN_MAC_CMD_SRCLEN_M 0xc000 520#define HIFN_MAC_CMD_SRCLEN_S 14 521 522/* 523 * MAC POS IPsec initiates authentication after encryption on encodes 524 * and before decryption on decodes. 525 */ 526#define HIFN_MAC_CMD_POS_IPSEC 0x0200 527#define HIFN_MAC_CMD_NEW_KEY 0x0800 528 529struct hifn_comp_command { 530 volatile __le16 masks; 531 volatile __le16 header_skip; 532 volatile __le16 source_count; 533 volatile __le16 reserved; 534}; 535 536#define HIFN_COMP_CMD_SRCLEN_M 0xc000 537#define HIFN_COMP_CMD_SRCLEN_S 14 538#define HIFN_COMP_CMD_ONE 0x0100 /* must be one */ 539#define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */ 540#define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */ 541#define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */ 542#define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */ 543#define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */ 544#define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */ 545#define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */ 546 547struct hifn_base_result { 548 volatile __le16 flags; 549 volatile __le16 session; 550 volatile __le16 src_cnt; /* 15:0 of source count */ 551 volatile __le16 dst_cnt; /* 15:0 of dest count */ 552}; 553 554#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ 555#define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */ 556#define HIFN_BASE_RES_SRCLEN_S 14 557#define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */ 558#define HIFN_BASE_RES_DSTLEN_S 12 559 560struct hifn_comp_result { 561 volatile __le16 flags; 562 volatile __le16 crc; 563}; 564 565#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ 566#define HIFN_COMP_RES_LCB_S 8 567#define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */ 568#define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */ 569#define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */ 570 571struct hifn_mac_result { 572 volatile __le16 flags; 573 volatile __le16 reserved; 574 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ 575}; 576 577#define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */ 578#define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */ 579 580struct hifn_crypt_result { 581 volatile __le16 flags; 582 volatile __le16 reserved; 583}; 584 585#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ 586 587#ifndef HIFN_POLL_FREQUENCY 588#define HIFN_POLL_FREQUENCY 0x1 589#endif 590 591#ifndef HIFN_POLL_SCALAR 592#define HIFN_POLL_SCALAR 0x0 593#endif 594 595#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ 596#define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */ 597 598struct hifn_crypto_alg { 599 struct list_head entry; 600 struct skcipher_alg alg; 601 struct hifn_device *dev; 602}; 603 604#define ASYNC_SCATTERLIST_CACHE 16 605 606#define ASYNC_FLAGS_MISALIGNED (1 << 0) 607 608struct hifn_cipher_walk { 609 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; 610 u32 flags; 611 int num; 612}; 613 614struct hifn_context { 615 u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; 616 struct hifn_device *dev; 617 unsigned int keysize; 618}; 619 620struct hifn_request_context { 621 u8 *iv; 622 unsigned int ivsize; 623 u8 op, type, mode, unused; 624 struct hifn_cipher_walk walk; 625}; 626 627#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) 628 629static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg) 630{ 631 return readl(dev->bar[0] + reg); 632} 633 634static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) 635{ 636 return readl(dev->bar[1] + reg); 637} 638 639static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) 640{ 641 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); 642} 643 644static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) 645{ 646 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); 647} 648 649static void hifn_wait_puc(struct hifn_device *dev) 650{ 651 int i; 652 u32 ret; 653 654 for (i = 10000; i > 0; --i) { 655 ret = hifn_read_0(dev, HIFN_0_PUCTRL); 656 if (!(ret & HIFN_PUCTRL_RESET)) 657 break; 658 659 udelay(1); 660 } 661 662 if (!i) 663 dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); 664} 665 666static void hifn_reset_puc(struct hifn_device *dev) 667{ 668 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 669 hifn_wait_puc(dev); 670} 671 672static void hifn_stop_device(struct hifn_device *dev) 673{ 674 hifn_write_1(dev, HIFN_1_DMA_CSR, 675 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 676 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS); 677 hifn_write_0(dev, HIFN_0_PUIER, 0); 678 hifn_write_1(dev, HIFN_1_DMA_IER, 0); 679} 680 681static void hifn_reset_dma(struct hifn_device *dev, int full) 682{ 683 hifn_stop_device(dev); 684 685 /* 686 * Setting poll frequency and others to 0. 687 */ 688 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 689 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 690 mdelay(1); 691 692 /* 693 * Reset DMA. 694 */ 695 if (full) { 696 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE); 697 mdelay(1); 698 } else { 699 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE | 700 HIFN_DMACNFG_MSTRESET); 701 hifn_reset_puc(dev); 702 } 703 704 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 705 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 706 707 hifn_reset_puc(dev); 708} 709 710static u32 hifn_next_signature(u32 a, u_int cnt) 711{ 712 int i; 713 u32 v; 714 715 for (i = 0; i < cnt; i++) { 716 /* get the parity */ 717 v = a & 0x80080125; 718 v ^= v >> 16; 719 v ^= v >> 8; 720 v ^= v >> 4; 721 v ^= v >> 2; 722 v ^= v >> 1; 723 724 a = (v & 1) ^ (a << 1); 725 } 726 727 return a; 728} 729 730static struct pci2id { 731 u_short pci_vendor; 732 u_short pci_prod; 733 char card_id[13]; 734} pci2id[] = { 735 { 736 PCI_VENDOR_ID_HIFN, 737 PCI_DEVICE_ID_HIFN_7955, 738 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 739 0x00, 0x00, 0x00, 0x00, 0x00 } 740 }, 741 { 742 PCI_VENDOR_ID_HIFN, 743 PCI_DEVICE_ID_HIFN_7956, 744 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 745 0x00, 0x00, 0x00, 0x00, 0x00 } 746 } 747}; 748 749#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 750static int hifn_rng_data_present(struct hwrng *rng, int wait) 751{ 752 struct hifn_device *dev = (struct hifn_device *)rng->priv; 753 s64 nsec; 754 755 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); 756 nsec -= dev->rng_wait_time; 757 if (nsec <= 0) 758 return 1; 759 if (!wait) 760 return 0; 761 ndelay(nsec); 762 return 1; 763} 764 765static int hifn_rng_data_read(struct hwrng *rng, u32 *data) 766{ 767 struct hifn_device *dev = (struct hifn_device *)rng->priv; 768 769 *data = hifn_read_1(dev, HIFN_1_RNG_DATA); 770 dev->rngtime = ktime_get(); 771 return 4; 772} 773 774static int hifn_register_rng(struct hifn_device *dev) 775{ 776 /* 777 * We must wait at least 256 Pk_clk cycles between two reads of the rng. 778 */ 779 dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, 780 dev->pk_clk_freq) * 256; 781 782 dev->rng.name = dev->name; 783 dev->rng.data_present = hifn_rng_data_present; 784 dev->rng.data_read = hifn_rng_data_read; 785 dev->rng.priv = (unsigned long)dev; 786 787 return hwrng_register(&dev->rng); 788} 789 790static void hifn_unregister_rng(struct hifn_device *dev) 791{ 792 hwrng_unregister(&dev->rng); 793} 794#else 795#define hifn_register_rng(dev) 0 796#define hifn_unregister_rng(dev) 797#endif 798 799static int hifn_init_pubrng(struct hifn_device *dev) 800{ 801 int i; 802 803 hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) | 804 HIFN_PUBRST_RESET); 805 806 for (i = 100; i > 0; --i) { 807 mdelay(1); 808 809 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0) 810 break; 811 } 812 813 if (!i) { 814 dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); 815 } else { 816 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); 817 dev->dmareg |= HIFN_DMAIER_PUBDONE; 818 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 819 820 dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); 821 } 822 823 /* Enable RNG engine. */ 824 825 hifn_write_1(dev, HIFN_1_RNG_CONFIG, 826 hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA); 827 dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); 828 829#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG 830 /* First value must be discarded */ 831 hifn_read_1(dev, HIFN_1_RNG_DATA); 832 dev->rngtime = ktime_get(); 833#endif 834 return 0; 835} 836 837static int hifn_enable_crypto(struct hifn_device *dev) 838{ 839 u32 dmacfg, addr; 840 char *offtbl = NULL; 841 int i; 842 843 for (i = 0; i < ARRAY_SIZE(pci2id); i++) { 844 if (pci2id[i].pci_vendor == dev->pdev->vendor && 845 pci2id[i].pci_prod == dev->pdev->device) { 846 offtbl = pci2id[i].card_id; 847 break; 848 } 849 } 850 851 if (!offtbl) { 852 dev_err(&dev->pdev->dev, "Unknown card!\n"); 853 return -ENODEV; 854 } 855 856 dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG); 857 858 hifn_write_1(dev, HIFN_1_DMA_CNFG, 859 HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET | 860 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE); 861 mdelay(1); 862 addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1); 863 mdelay(1); 864 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0); 865 mdelay(1); 866 867 for (i = 0; i < 12; ++i) { 868 addr = hifn_next_signature(addr, offtbl[i] + 0x101); 869 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr); 870 871 mdelay(1); 872 } 873 hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg); 874 875 dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); 876 877 return 0; 878} 879 880static void hifn_init_dma(struct hifn_device *dev) 881{ 882 struct hifn_dma *dma = dev->desc_virt; 883 u32 dptr = dev->desc_dma; 884 int i; 885 886 for (i = 0; i < HIFN_D_CMD_RSIZE; ++i) 887 dma->cmdr[i].p = __cpu_to_le32(dptr + 888 offsetof(struct hifn_dma, command_bufs[i][0])); 889 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 890 dma->resr[i].p = __cpu_to_le32(dptr + 891 offsetof(struct hifn_dma, result_bufs[i][0])); 892 893 /* Setup LAST descriptors. */ 894 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + 895 offsetof(struct hifn_dma, cmdr[0])); 896 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + 897 offsetof(struct hifn_dma, srcr[0])); 898 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr + 899 offsetof(struct hifn_dma, dstr[0])); 900 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr + 901 offsetof(struct hifn_dma, resr[0])); 902 903 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0; 904 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0; 905 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; 906} 907 908/* 909 * Initialize the PLL. We need to know the frequency of the reference clock 910 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that 911 * allows us to operate without the risk of overclocking the chip. If it 912 * actually uses 33MHz, the chip will operate at half the speed, this can be 913 * overridden by specifying the frequency as module parameter (pci33). 914 * 915 * Unfortunately the PCI clock is not very suitable since the HIFN needs a 916 * stable clock and the PCI clock frequency may vary, so the default is the 917 * external clock. There is no way to find out its frequency, we default to 918 * 66MHz since according to Mike Ham of HiFn, almost every board in existence 919 * has an external crystal populated at 66MHz. 920 */ 921static void hifn_init_pll(struct hifn_device *dev) 922{ 923 unsigned int freq, m; 924 u32 pllcfg; 925 926 pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1; 927 928 if (strncmp(hifn_pll_ref, "ext", 3) == 0) 929 pllcfg |= HIFN_PLL_REF_CLK_PLL; 930 else 931 pllcfg |= HIFN_PLL_REF_CLK_HBI; 932 933 if (hifn_pll_ref[3] != '\0') 934 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 935 else { 936 freq = 66; 937 dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", 938 freq, hifn_pll_ref); 939 } 940 941 m = HIFN_PLL_FCK_MAX / freq; 942 943 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; 944 if (m <= 8) 945 pllcfg |= HIFN_PLL_IS_1_8; 946 else 947 pllcfg |= HIFN_PLL_IS_9_12; 948 949 /* Select clock source and enable clock bypass */ 950 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 951 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP); 952 953 /* Let the chip lock to the input clock */ 954 mdelay(10); 955 956 /* Disable clock bypass */ 957 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 958 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI); 959 960 /* Switch the engines to the PLL */ 961 hifn_write_1(dev, HIFN_1_PLL, pllcfg | 962 HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL); 963 964 /* 965 * The Fpk_clk runs at half the total speed. Its frequency is needed to 966 * calculate the minimum time between two reads of the rng. Since 33MHz 967 * is actually 33.333... we overestimate the frequency here, resulting 968 * in slightly larger intervals. 969 */ 970 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; 971} 972 973static void hifn_init_registers(struct hifn_device *dev) 974{ 975 u32 dptr = dev->desc_dma; 976 977 /* Initialization magic... */ 978 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA); 979 hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD); 980 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 981 982 /* write all 4 ring address registers */ 983 hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + 984 offsetof(struct hifn_dma, cmdr[0])); 985 hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + 986 offsetof(struct hifn_dma, srcr[0])); 987 hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + 988 offsetof(struct hifn_dma, dstr[0])); 989 hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + 990 offsetof(struct hifn_dma, resr[0])); 991 992 mdelay(2); 993#if 0 994 hifn_write_1(dev, HIFN_1_DMA_CSR, 995 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS | 996 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS | 997 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 998 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 999 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 1000 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 1001 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 1002 HIFN_DMACSR_S_WAIT | 1003 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1004 HIFN_DMACSR_C_WAIT | 1005 HIFN_DMACSR_ENGINE | 1006 HIFN_DMACSR_PUBDONE); 1007#else 1008 hifn_write_1(dev, HIFN_1_DMA_CSR, 1009 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA | 1010 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA | 1011 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST | 1012 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER | 1013 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST | 1014 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER | 1015 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST | 1016 HIFN_DMACSR_S_WAIT | 1017 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST | 1018 HIFN_DMACSR_C_WAIT | 1019 HIFN_DMACSR_ENGINE | 1020 HIFN_DMACSR_PUBDONE); 1021#endif 1022 hifn_read_1(dev, HIFN_1_DMA_CSR); 1023 1024 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT | 1025 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER | 1026 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT | 1027 HIFN_DMAIER_ENGINE; 1028 dev->dmareg &= ~HIFN_DMAIER_C_WAIT; 1029 1030 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1031 hifn_read_1(dev, HIFN_1_DMA_IER); 1032#if 0 1033 hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG | 1034 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES | 1035 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 | 1036 HIFN_PUCNFG_DRAM); 1037#else 1038 hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342); 1039#endif 1040 hifn_init_pll(dev); 1041 1042 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1043 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | 1044 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST | 1045 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) | 1046 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL)); 1047} 1048 1049static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf, 1050 unsigned dlen, unsigned slen, u16 mask, u8 snum) 1051{ 1052 struct hifn_base_command *base_cmd; 1053 u8 *buf_pos = buf; 1054 1055 base_cmd = (struct hifn_base_command *)buf_pos; 1056 base_cmd->masks = __cpu_to_le16(mask); 1057 base_cmd->total_source_count = 1058 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO); 1059 base_cmd->total_dest_count = 1060 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO); 1061 1062 dlen >>= 16; 1063 slen >>= 16; 1064 base_cmd->session_num = __cpu_to_le16(snum | 1065 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) | 1066 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M)); 1067 1068 return sizeof(struct hifn_base_command); 1069} 1070 1071static int hifn_setup_crypto_command(struct hifn_device *dev, 1072 u8 *buf, unsigned dlen, unsigned slen, 1073 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode) 1074{ 1075 struct hifn_dma *dma = dev->desc_virt; 1076 struct hifn_crypt_command *cry_cmd; 1077 u8 *buf_pos = buf; 1078 u16 cmd_len; 1079 1080 cry_cmd = (struct hifn_crypt_command *)buf_pos; 1081 1082 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff); 1083 dlen >>= 16; 1084 cry_cmd->masks = __cpu_to_le16(mode | 1085 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & 1086 HIFN_CRYPT_CMD_SRCLEN_M)); 1087 cry_cmd->header_skip = 0; 1088 cry_cmd->reserved = 0; 1089 1090 buf_pos += sizeof(struct hifn_crypt_command); 1091 1092 dma->cmdu++; 1093 if (dma->cmdu > 1) { 1094 dev->dmareg |= HIFN_DMAIER_C_WAIT; 1095 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1096 } 1097 1098 if (keylen) { 1099 memcpy(buf_pos, key, keylen); 1100 buf_pos += keylen; 1101 } 1102 if (ivsize) { 1103 memcpy(buf_pos, iv, ivsize); 1104 buf_pos += ivsize; 1105 } 1106 1107 cmd_len = buf_pos - buf; 1108 1109 return cmd_len; 1110} 1111 1112static int hifn_setup_cmd_desc(struct hifn_device *dev, 1113 struct hifn_context *ctx, struct hifn_request_context *rctx, 1114 void *priv, unsigned int nbytes) 1115{ 1116 struct hifn_dma *dma = dev->desc_virt; 1117 int cmd_len, sa_idx; 1118 u8 *buf, *buf_pos; 1119 u16 mask; 1120 1121 sa_idx = dma->cmdi; 1122 buf_pos = buf = dma->command_bufs[dma->cmdi]; 1123 1124 mask = 0; 1125 switch (rctx->op) { 1126 case ACRYPTO_OP_DECRYPT: 1127 mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; 1128 break; 1129 case ACRYPTO_OP_ENCRYPT: 1130 mask = HIFN_BASE_CMD_CRYPT; 1131 break; 1132 case ACRYPTO_OP_HMAC: 1133 mask = HIFN_BASE_CMD_MAC; 1134 break; 1135 default: 1136 goto err_out; 1137 } 1138 1139 buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, 1140 nbytes, mask, dev->snum); 1141 1142 if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { 1143 u16 md = 0; 1144 1145 if (ctx->keysize) 1146 md |= HIFN_CRYPT_CMD_NEW_KEY; 1147 if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) 1148 md |= HIFN_CRYPT_CMD_NEW_IV; 1149 1150 switch (rctx->mode) { 1151 case ACRYPTO_MODE_ECB: 1152 md |= HIFN_CRYPT_CMD_MODE_ECB; 1153 break; 1154 case ACRYPTO_MODE_CBC: 1155 md |= HIFN_CRYPT_CMD_MODE_CBC; 1156 break; 1157 case ACRYPTO_MODE_CFB: 1158 md |= HIFN_CRYPT_CMD_MODE_CFB; 1159 break; 1160 case ACRYPTO_MODE_OFB: 1161 md |= HIFN_CRYPT_CMD_MODE_OFB; 1162 break; 1163 default: 1164 goto err_out; 1165 } 1166 1167 switch (rctx->type) { 1168 case ACRYPTO_TYPE_AES_128: 1169 if (ctx->keysize != 16) 1170 goto err_out; 1171 md |= HIFN_CRYPT_CMD_KSZ_128 | 1172 HIFN_CRYPT_CMD_ALG_AES; 1173 break; 1174 case ACRYPTO_TYPE_AES_192: 1175 if (ctx->keysize != 24) 1176 goto err_out; 1177 md |= HIFN_CRYPT_CMD_KSZ_192 | 1178 HIFN_CRYPT_CMD_ALG_AES; 1179 break; 1180 case ACRYPTO_TYPE_AES_256: 1181 if (ctx->keysize != 32) 1182 goto err_out; 1183 md |= HIFN_CRYPT_CMD_KSZ_256 | 1184 HIFN_CRYPT_CMD_ALG_AES; 1185 break; 1186 case ACRYPTO_TYPE_3DES: 1187 if (ctx->keysize != 24) 1188 goto err_out; 1189 md |= HIFN_CRYPT_CMD_ALG_3DES; 1190 break; 1191 case ACRYPTO_TYPE_DES: 1192 if (ctx->keysize != 8) 1193 goto err_out; 1194 md |= HIFN_CRYPT_CMD_ALG_DES; 1195 break; 1196 default: 1197 goto err_out; 1198 } 1199 1200 buf_pos += hifn_setup_crypto_command(dev, buf_pos, 1201 nbytes, nbytes, ctx->key, ctx->keysize, 1202 rctx->iv, rctx->ivsize, md); 1203 } 1204 1205 dev->sa[sa_idx] = priv; 1206 dev->started++; 1207 1208 cmd_len = buf_pos - buf; 1209 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | 1210 HIFN_D_LAST | HIFN_D_MASKDONEIRQ); 1211 1212 if (++dma->cmdi == HIFN_D_CMD_RSIZE) { 1213 dma->cmdr[dma->cmdi].l = __cpu_to_le32( 1214 HIFN_D_VALID | HIFN_D_LAST | 1215 HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); 1216 dma->cmdi = 0; 1217 } else { 1218 dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); 1219 } 1220 1221 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { 1222 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1223 dev->flags |= HIFN_FLAG_CMD_BUSY; 1224 } 1225 return 0; 1226 1227err_out: 1228 return -EINVAL; 1229} 1230 1231static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, 1232 unsigned int offset, unsigned int size, int last) 1233{ 1234 struct hifn_dma *dma = dev->desc_virt; 1235 int idx; 1236 dma_addr_t addr; 1237 1238 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1239 DMA_TO_DEVICE); 1240 1241 idx = dma->srci; 1242 1243 dma->srcr[idx].p = __cpu_to_le32(addr); 1244 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1245 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1246 1247 if (++idx == HIFN_D_SRC_RSIZE) { 1248 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1249 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1250 (last ? HIFN_D_LAST : 0)); 1251 idx = 0; 1252 } 1253 1254 dma->srci = idx; 1255 dma->srcu++; 1256 1257 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { 1258 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); 1259 dev->flags |= HIFN_FLAG_SRC_BUSY; 1260 } 1261 1262 return size; 1263} 1264 1265static void hifn_setup_res_desc(struct hifn_device *dev) 1266{ 1267 struct hifn_dma *dma = dev->desc_virt; 1268 1269 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | 1270 HIFN_D_VALID | HIFN_D_LAST); 1271 /* 1272 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | 1273 * HIFN_D_LAST); 1274 */ 1275 1276 if (++dma->resi == HIFN_D_RES_RSIZE) { 1277 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | 1278 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); 1279 dma->resi = 0; 1280 } 1281 1282 dma->resu++; 1283 1284 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { 1285 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); 1286 dev->flags |= HIFN_FLAG_RES_BUSY; 1287 } 1288} 1289 1290static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, 1291 unsigned offset, unsigned size, int last) 1292{ 1293 struct hifn_dma *dma = dev->desc_virt; 1294 int idx; 1295 dma_addr_t addr; 1296 1297 addr = dma_map_page(&dev->pdev->dev, page, offset, size, 1298 DMA_FROM_DEVICE); 1299 1300 idx = dma->dsti; 1301 dma->dstr[idx].p = __cpu_to_le32(addr); 1302 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | 1303 HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); 1304 1305 if (++idx == HIFN_D_DST_RSIZE) { 1306 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | 1307 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | 1308 (last ? HIFN_D_LAST : 0)); 1309 idx = 0; 1310 } 1311 dma->dsti = idx; 1312 dma->dstu++; 1313 1314 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { 1315 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); 1316 dev->flags |= HIFN_FLAG_DST_BUSY; 1317 } 1318} 1319 1320static int hifn_setup_dma(struct hifn_device *dev, 1321 struct hifn_context *ctx, struct hifn_request_context *rctx, 1322 struct scatterlist *src, struct scatterlist *dst, 1323 unsigned int nbytes, void *priv) 1324{ 1325 struct scatterlist *t; 1326 struct page *spage, *dpage; 1327 unsigned int soff, doff; 1328 unsigned int n, len; 1329 1330 n = nbytes; 1331 while (n) { 1332 spage = sg_page(src); 1333 soff = src->offset; 1334 len = min(src->length, n); 1335 1336 hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); 1337 1338 src++; 1339 n -= len; 1340 } 1341 1342 t = &rctx->walk.cache[0]; 1343 n = nbytes; 1344 while (n) { 1345 if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1346 BUG_ON(!sg_page(t)); 1347 dpage = sg_page(t); 1348 doff = 0; 1349 len = t->length; 1350 } else { 1351 BUG_ON(!sg_page(dst)); 1352 dpage = sg_page(dst); 1353 doff = dst->offset; 1354 len = dst->length; 1355 } 1356 len = min(len, n); 1357 1358 hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); 1359 1360 dst++; 1361 t++; 1362 n -= len; 1363 } 1364 1365 hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); 1366 hifn_setup_res_desc(dev); 1367 return 0; 1368} 1369 1370static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, 1371 int num, gfp_t gfp_flags) 1372{ 1373 int i; 1374 1375 num = min(ASYNC_SCATTERLIST_CACHE, num); 1376 sg_init_table(w->cache, num); 1377 1378 w->num = 0; 1379 for (i = 0; i < num; ++i) { 1380 struct page *page = alloc_page(gfp_flags); 1381 struct scatterlist *s; 1382 1383 if (!page) 1384 break; 1385 1386 s = &w->cache[i]; 1387 1388 sg_set_page(s, page, PAGE_SIZE, 0); 1389 w->num++; 1390 } 1391 1392 return i; 1393} 1394 1395static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) 1396{ 1397 int i; 1398 1399 for (i = 0; i < w->num; ++i) { 1400 struct scatterlist *s = &w->cache[i]; 1401 1402 __free_page(sg_page(s)); 1403 1404 s->length = 0; 1405 } 1406 1407 w->num = 0; 1408} 1409 1410static int skcipher_add(unsigned int *drestp, struct scatterlist *dst, 1411 unsigned int size, unsigned int *nbytesp) 1412{ 1413 unsigned int copy, drest = *drestp, nbytes = *nbytesp; 1414 int idx = 0; 1415 1416 if (drest < size || size > nbytes) 1417 return -EINVAL; 1418 1419 while (size) { 1420 copy = min3(drest, size, dst->length); 1421 1422 size -= copy; 1423 drest -= copy; 1424 nbytes -= copy; 1425 1426 pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", 1427 __func__, copy, size, drest, nbytes); 1428 1429 dst++; 1430 idx++; 1431 } 1432 1433 *nbytesp = nbytes; 1434 *drestp = drest; 1435 1436 return idx; 1437} 1438 1439static int hifn_cipher_walk(struct skcipher_request *req, 1440 struct hifn_cipher_walk *w) 1441{ 1442 struct scatterlist *dst, *t; 1443 unsigned int nbytes = req->cryptlen, offset, copy, diff; 1444 int idx, tidx, err; 1445 1446 tidx = idx = 0; 1447 offset = 0; 1448 while (nbytes) { 1449 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) 1450 return -EINVAL; 1451 1452 dst = &req->dst[idx]; 1453 1454 pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", 1455 __func__, dst->length, dst->offset, offset, nbytes); 1456 1457 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1458 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || 1459 offset) { 1460 unsigned slen = min(dst->length - offset, nbytes); 1461 unsigned dlen = PAGE_SIZE; 1462 1463 t = &w->cache[idx]; 1464 1465 err = skcipher_add(&dlen, dst, slen, &nbytes); 1466 if (err < 0) 1467 return err; 1468 1469 idx += err; 1470 1471 copy = slen & ~(HIFN_D_DST_DALIGN - 1); 1472 diff = slen & (HIFN_D_DST_DALIGN - 1); 1473 1474 if (dlen < nbytes) { 1475 /* 1476 * Destination page does not have enough space 1477 * to put there additional blocksized chunk, 1478 * so we mark that page as containing only 1479 * blocksize aligned chunks: 1480 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); 1481 * and increase number of bytes to be processed 1482 * in next chunk: 1483 * nbytes += diff; 1484 */ 1485 nbytes += diff; 1486 1487 /* 1488 * Temporary of course... 1489 * Kick author if you will catch this one. 1490 */ 1491 pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n", 1492 __func__, dlen, nbytes, slen, offset); 1493 pr_err("%s: please contact author to fix this " 1494 "issue, generally you should not catch " 1495 "this path under any condition but who " 1496 "knows how did you use crypto code.\n" 1497 "Thank you.\n", __func__); 1498 BUG(); 1499 } else { 1500 copy += diff + nbytes; 1501 1502 dst = &req->dst[idx]; 1503 1504 err = skcipher_add(&dlen, dst, nbytes, &nbytes); 1505 if (err < 0) 1506 return err; 1507 1508 idx += err; 1509 } 1510 1511 t->length = copy; 1512 t->offset = offset; 1513 } else { 1514 nbytes -= min(dst->length, nbytes); 1515 idx++; 1516 } 1517 1518 tidx++; 1519 } 1520 1521 return tidx; 1522} 1523 1524static int hifn_setup_session(struct skcipher_request *req) 1525{ 1526 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1527 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1528 struct hifn_device *dev = ctx->dev; 1529 unsigned long dlen, flags; 1530 unsigned int nbytes = req->cryptlen, idx = 0; 1531 int err = -EINVAL, sg_num; 1532 struct scatterlist *dst; 1533 1534 if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) 1535 goto err_out_exit; 1536 1537 rctx->walk.flags = 0; 1538 1539 while (nbytes) { 1540 dst = &req->dst[idx]; 1541 dlen = min(dst->length, nbytes); 1542 1543 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || 1544 !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) 1545 rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; 1546 1547 nbytes -= dlen; 1548 idx++; 1549 } 1550 1551 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1552 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); 1553 if (err < 0) 1554 return err; 1555 } 1556 1557 sg_num = hifn_cipher_walk(req, &rctx->walk); 1558 if (sg_num < 0) { 1559 err = sg_num; 1560 goto err_out_exit; 1561 } 1562 1563 spin_lock_irqsave(&dev->lock, flags); 1564 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { 1565 err = -EAGAIN; 1566 goto err_out; 1567 } 1568 1569 err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req); 1570 if (err) 1571 goto err_out; 1572 1573 dev->snum++; 1574 1575 dev->active = HIFN_DEFAULT_ACTIVE_NUM; 1576 spin_unlock_irqrestore(&dev->lock, flags); 1577 1578 return 0; 1579 1580err_out: 1581 spin_unlock_irqrestore(&dev->lock, flags); 1582err_out_exit: 1583 if (err) { 1584 dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " 1585 "type: %u, err: %d.\n", 1586 rctx->iv, rctx->ivsize, 1587 ctx->key, ctx->keysize, 1588 rctx->mode, rctx->op, rctx->type, err); 1589 } 1590 1591 return err; 1592} 1593 1594static int hifn_start_device(struct hifn_device *dev) 1595{ 1596 int err; 1597 1598 dev->started = dev->active = 0; 1599 hifn_reset_dma(dev, 1); 1600 1601 err = hifn_enable_crypto(dev); 1602 if (err) 1603 return err; 1604 1605 hifn_reset_puc(dev); 1606 1607 hifn_init_dma(dev); 1608 1609 hifn_init_registers(dev); 1610 1611 hifn_init_pubrng(dev); 1612 1613 return 0; 1614} 1615 1616static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset, 1617 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) 1618{ 1619 unsigned int srest = *srestp, nbytes = *nbytesp, copy; 1620 void *daddr; 1621 int idx = 0; 1622 1623 if (srest < size || size > nbytes) 1624 return -EINVAL; 1625 1626 while (size) { 1627 copy = min3(srest, dst->length, size); 1628 1629 daddr = kmap_atomic(sg_page(dst)); 1630 memcpy(daddr + dst->offset + offset, saddr, copy); 1631 kunmap_atomic(daddr); 1632 1633 nbytes -= copy; 1634 size -= copy; 1635 srest -= copy; 1636 saddr += copy; 1637 offset = 0; 1638 1639 pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", 1640 __func__, copy, size, srest, nbytes); 1641 1642 dst++; 1643 idx++; 1644 } 1645 1646 *nbytesp = nbytes; 1647 *srestp = srest; 1648 1649 return idx; 1650} 1651 1652static inline void hifn_complete_sa(struct hifn_device *dev, int i) 1653{ 1654 unsigned long flags; 1655 1656 spin_lock_irqsave(&dev->lock, flags); 1657 dev->sa[i] = NULL; 1658 dev->started--; 1659 if (dev->started < 0) 1660 dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, 1661 dev->started); 1662 spin_unlock_irqrestore(&dev->lock, flags); 1663 BUG_ON(dev->started < 0); 1664} 1665 1666static void hifn_process_ready(struct skcipher_request *req, int error) 1667{ 1668 struct hifn_request_context *rctx = skcipher_request_ctx(req); 1669 1670 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1671 unsigned int nbytes = req->cryptlen; 1672 int idx = 0, err; 1673 struct scatterlist *dst, *t; 1674 void *saddr; 1675 1676 while (nbytes) { 1677 t = &rctx->walk.cache[idx]; 1678 dst = &req->dst[idx]; 1679 1680 pr_debug("\n%s: sg_page(t): %p, t->length: %u, " 1681 "sg_page(dst): %p, dst->length: %u, " 1682 "nbytes: %u.\n", 1683 __func__, sg_page(t), t->length, 1684 sg_page(dst), dst->length, nbytes); 1685 1686 if (!t->length) { 1687 nbytes -= min(dst->length, nbytes); 1688 idx++; 1689 continue; 1690 } 1691 1692 saddr = kmap_atomic(sg_page(t)); 1693 1694 err = skcipher_get(saddr, &t->length, t->offset, 1695 dst, nbytes, &nbytes); 1696 if (err < 0) { 1697 kunmap_atomic(saddr); 1698 break; 1699 } 1700 1701 idx += err; 1702 kunmap_atomic(saddr); 1703 } 1704 1705 hifn_cipher_walk_exit(&rctx->walk); 1706 } 1707 1708 skcipher_request_complete(req, error); 1709} 1710 1711static void hifn_clear_rings(struct hifn_device *dev, int error) 1712{ 1713 struct hifn_dma *dma = dev->desc_virt; 1714 int i, u; 1715 1716 dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1717 "k: %d.%d.%d.%d.\n", 1718 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1719 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1720 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1721 1722 i = dma->resk; u = dma->resu; 1723 while (u != 0) { 1724 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1725 break; 1726 1727 if (dev->sa[i]) { 1728 dev->success++; 1729 dev->reset = 0; 1730 hifn_process_ready(dev->sa[i], error); 1731 hifn_complete_sa(dev, i); 1732 } 1733 1734 if (++i == HIFN_D_RES_RSIZE) 1735 i = 0; 1736 u--; 1737 } 1738 dma->resk = i; dma->resu = u; 1739 1740 i = dma->srck; u = dma->srcu; 1741 while (u != 0) { 1742 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1743 break; 1744 if (++i == HIFN_D_SRC_RSIZE) 1745 i = 0; 1746 u--; 1747 } 1748 dma->srck = i; dma->srcu = u; 1749 1750 i = dma->cmdk; u = dma->cmdu; 1751 while (u != 0) { 1752 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1753 break; 1754 if (++i == HIFN_D_CMD_RSIZE) 1755 i = 0; 1756 u--; 1757 } 1758 dma->cmdk = i; dma->cmdu = u; 1759 1760 i = dma->dstk; u = dma->dstu; 1761 while (u != 0) { 1762 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) 1763 break; 1764 if (++i == HIFN_D_DST_RSIZE) 1765 i = 0; 1766 u--; 1767 } 1768 dma->dstk = i; dma->dstu = u; 1769 1770 dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " 1771 "k: %d.%d.%d.%d.\n", 1772 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1773 dma->cmdu, dma->srcu, dma->dstu, dma->resu, 1774 dma->cmdk, dma->srck, dma->dstk, dma->resk); 1775} 1776 1777static void hifn_work(struct work_struct *work) 1778{ 1779 struct delayed_work *dw = to_delayed_work(work); 1780 struct hifn_device *dev = container_of(dw, struct hifn_device, work); 1781 unsigned long flags; 1782 int reset = 0; 1783 u32 r = 0; 1784 1785 spin_lock_irqsave(&dev->lock, flags); 1786 if (dev->active == 0) { 1787 struct hifn_dma *dma = dev->desc_virt; 1788 1789 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) { 1790 dev->flags &= ~HIFN_FLAG_CMD_BUSY; 1791 r |= HIFN_DMACSR_C_CTRL_DIS; 1792 } 1793 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) { 1794 dev->flags &= ~HIFN_FLAG_SRC_BUSY; 1795 r |= HIFN_DMACSR_S_CTRL_DIS; 1796 } 1797 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) { 1798 dev->flags &= ~HIFN_FLAG_DST_BUSY; 1799 r |= HIFN_DMACSR_D_CTRL_DIS; 1800 } 1801 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) { 1802 dev->flags &= ~HIFN_FLAG_RES_BUSY; 1803 r |= HIFN_DMACSR_R_CTRL_DIS; 1804 } 1805 if (r) 1806 hifn_write_1(dev, HIFN_1_DMA_CSR, r); 1807 } else 1808 dev->active--; 1809 1810 if ((dev->prev_success == dev->success) && dev->started) 1811 reset = 1; 1812 dev->prev_success = dev->success; 1813 spin_unlock_irqrestore(&dev->lock, flags); 1814 1815 if (reset) { 1816 if (++dev->reset >= 5) { 1817 int i; 1818 struct hifn_dma *dma = dev->desc_virt; 1819 1820 dev_info(&dev->pdev->dev, 1821 "r: %08x, active: %d, started: %d, " 1822 "success: %lu: qlen: %u/%u, reset: %d.\n", 1823 r, dev->active, dev->started, 1824 dev->success, dev->queue.qlen, dev->queue.max_qlen, 1825 reset); 1826 1827 dev_info(&dev->pdev->dev, "%s: res: ", __func__); 1828 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1829 pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); 1830 if (dev->sa[i]) { 1831 hifn_process_ready(dev->sa[i], -ENODEV); 1832 hifn_complete_sa(dev, i); 1833 } 1834 } 1835 pr_info("\n"); 1836 1837 hifn_reset_dma(dev, 1); 1838 hifn_stop_device(dev); 1839 hifn_start_device(dev); 1840 dev->reset = 0; 1841 } 1842 1843 tasklet_schedule(&dev->tasklet); 1844 } 1845 1846 schedule_delayed_work(&dev->work, HZ); 1847} 1848 1849static irqreturn_t hifn_interrupt(int irq, void *data) 1850{ 1851 struct hifn_device *dev = data; 1852 struct hifn_dma *dma = dev->desc_virt; 1853 u32 dmacsr, restart; 1854 1855 dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR); 1856 1857 dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " 1858 "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", 1859 dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, 1860 dma->cmdi, dma->srci, dma->dsti, dma->resi, 1861 dma->cmdu, dma->srcu, dma->dstu, dma->resu); 1862 1863 if ((dmacsr & dev->dmareg) == 0) 1864 return IRQ_NONE; 1865 1866 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg); 1867 1868 if (dmacsr & HIFN_DMACSR_ENGINE) 1869 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR)); 1870 if (dmacsr & HIFN_DMACSR_PUBDONE) 1871 hifn_write_1(dev, HIFN_1_PUB_STATUS, 1872 hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE); 1873 1874 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER); 1875 if (restart) { 1876 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); 1877 1878 dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", 1879 !!(dmacsr & HIFN_DMACSR_R_OVER), 1880 !!(dmacsr & HIFN_DMACSR_D_OVER), 1881 puisr, !!(puisr & HIFN_PUISR_DSTOVER)); 1882 if (!!(puisr & HIFN_PUISR_DSTOVER)) 1883 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); 1884 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | 1885 HIFN_DMACSR_D_OVER)); 1886 } 1887 1888 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | 1889 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); 1890 if (restart) { 1891 dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", 1892 !!(dmacsr & HIFN_DMACSR_C_ABORT), 1893 !!(dmacsr & HIFN_DMACSR_S_ABORT), 1894 !!(dmacsr & HIFN_DMACSR_D_ABORT), 1895 !!(dmacsr & HIFN_DMACSR_R_ABORT)); 1896 hifn_reset_dma(dev, 1); 1897 hifn_init_dma(dev); 1898 hifn_init_registers(dev); 1899 } 1900 1901 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { 1902 dev_dbg(&dev->pdev->dev, "wait on command.\n"); 1903 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); 1904 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); 1905 } 1906 1907 tasklet_schedule(&dev->tasklet); 1908 1909 return IRQ_HANDLED; 1910} 1911 1912static void hifn_flush(struct hifn_device *dev) 1913{ 1914 unsigned long flags; 1915 struct crypto_async_request *async_req; 1916 struct skcipher_request *req; 1917 struct hifn_dma *dma = dev->desc_virt; 1918 int i; 1919 1920 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { 1921 struct hifn_desc *d = &dma->resr[i]; 1922 1923 if (dev->sa[i]) { 1924 hifn_process_ready(dev->sa[i], 1925 (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); 1926 hifn_complete_sa(dev, i); 1927 } 1928 } 1929 1930 spin_lock_irqsave(&dev->lock, flags); 1931 while ((async_req = crypto_dequeue_request(&dev->queue))) { 1932 req = skcipher_request_cast(async_req); 1933 spin_unlock_irqrestore(&dev->lock, flags); 1934 1935 hifn_process_ready(req, -ENODEV); 1936 1937 spin_lock_irqsave(&dev->lock, flags); 1938 } 1939 spin_unlock_irqrestore(&dev->lock, flags); 1940} 1941 1942static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key, 1943 unsigned int len) 1944{ 1945 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1946 struct hifn_device *dev = ctx->dev; 1947 int err; 1948 1949 err = verify_skcipher_des_key(cipher, key); 1950 if (err) 1951 return err; 1952 1953 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1954 1955 memcpy(ctx->key, key, len); 1956 ctx->keysize = len; 1957 1958 return 0; 1959} 1960 1961static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, 1962 unsigned int len) 1963{ 1964 struct hifn_context *ctx = crypto_skcipher_ctx(cipher); 1965 struct hifn_device *dev = ctx->dev; 1966 int err; 1967 1968 err = verify_skcipher_des3_key(cipher, key); 1969 if (err) 1970 return err; 1971 1972 dev->flags &= ~HIFN_FLAG_OLD_KEY; 1973 1974 memcpy(ctx->key, key, len); 1975 ctx->keysize = len; 1976 1977 return 0; 1978} 1979 1980static int hifn_handle_req(struct skcipher_request *req) 1981{ 1982 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1983 struct hifn_device *dev = ctx->dev; 1984 int err = -EAGAIN; 1985 1986 if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH) 1987 err = hifn_setup_session(req); 1988 1989 if (err == -EAGAIN) { 1990 unsigned long flags; 1991 1992 spin_lock_irqsave(&dev->lock, flags); 1993 err = crypto_enqueue_request(&dev->queue, &req->base); 1994 spin_unlock_irqrestore(&dev->lock, flags); 1995 } 1996 1997 return err; 1998} 1999 2000static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op, 2001 u8 type, u8 mode) 2002{ 2003 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 2004 struct hifn_request_context *rctx = skcipher_request_ctx(req); 2005 unsigned ivsize; 2006 2007 ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); 2008 2009 if (req->iv && mode != ACRYPTO_MODE_ECB) { 2010 if (type == ACRYPTO_TYPE_AES_128) 2011 ivsize = HIFN_AES_IV_LENGTH; 2012 else if (type == ACRYPTO_TYPE_DES) 2013 ivsize = HIFN_DES_KEY_LENGTH; 2014 else if (type == ACRYPTO_TYPE_3DES) 2015 ivsize = HIFN_3DES_KEY_LENGTH; 2016 } 2017 2018 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) { 2019 if (ctx->keysize == 24) 2020 type = ACRYPTO_TYPE_AES_192; 2021 else if (ctx->keysize == 32) 2022 type = ACRYPTO_TYPE_AES_256; 2023 } 2024 2025 rctx->op = op; 2026 rctx->mode = mode; 2027 rctx->type = type; 2028 rctx->iv = req->iv; 2029 rctx->ivsize = ivsize; 2030 2031 /* 2032 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2033 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2034 * HEAVY TODO: needs to kick Herbert XU to write documentation. 2035 */ 2036 2037 return hifn_handle_req(req); 2038} 2039 2040static int hifn_process_queue(struct hifn_device *dev) 2041{ 2042 struct crypto_async_request *async_req, *backlog; 2043 struct skcipher_request *req; 2044 unsigned long flags; 2045 int err = 0; 2046 2047 while (dev->started < HIFN_QUEUE_LENGTH) { 2048 spin_lock_irqsave(&dev->lock, flags); 2049 backlog = crypto_get_backlog(&dev->queue); 2050 async_req = crypto_dequeue_request(&dev->queue); 2051 spin_unlock_irqrestore(&dev->lock, flags); 2052 2053 if (!async_req) 2054 break; 2055 2056 if (backlog) 2057 crypto_request_complete(backlog, -EINPROGRESS); 2058 2059 req = skcipher_request_cast(async_req); 2060 2061 err = hifn_handle_req(req); 2062 if (err) 2063 break; 2064 } 2065 2066 return err; 2067} 2068 2069static int hifn_setup_crypto(struct skcipher_request *req, u8 op, 2070 u8 type, u8 mode) 2071{ 2072 int err; 2073 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 2074 struct hifn_device *dev = ctx->dev; 2075 2076 err = hifn_setup_crypto_req(req, op, type, mode); 2077 if (err) 2078 return err; 2079 2080 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2081 hifn_process_queue(dev); 2082 2083 return -EINPROGRESS; 2084} 2085 2086/* 2087 * AES ecryption functions. 2088 */ 2089static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req) 2090{ 2091 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2092 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2093} 2094static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) 2095{ 2096 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2097 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2098} 2099static inline int hifn_encrypt_aes_cfb(struct skcipher_request *req) 2100{ 2101 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2102 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB); 2103} 2104static inline int hifn_encrypt_aes_ofb(struct skcipher_request *req) 2105{ 2106 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2107 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB); 2108} 2109 2110/* 2111 * AES decryption functions. 2112 */ 2113static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req) 2114{ 2115 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2116 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB); 2117} 2118static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) 2119{ 2120 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2121 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); 2122} 2123static inline int hifn_decrypt_aes_cfb(struct skcipher_request *req) 2124{ 2125 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2126 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB); 2127} 2128static inline int hifn_decrypt_aes_ofb(struct skcipher_request *req) 2129{ 2130 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2131 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB); 2132} 2133 2134/* 2135 * DES ecryption functions. 2136 */ 2137static inline int hifn_encrypt_des_ecb(struct skcipher_request *req) 2138{ 2139 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2140 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2141} 2142static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) 2143{ 2144 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2145 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2146} 2147static inline int hifn_encrypt_des_cfb(struct skcipher_request *req) 2148{ 2149 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2150 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB); 2151} 2152static inline int hifn_encrypt_des_ofb(struct skcipher_request *req) 2153{ 2154 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2155 ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB); 2156} 2157 2158/* 2159 * DES decryption functions. 2160 */ 2161static inline int hifn_decrypt_des_ecb(struct skcipher_request *req) 2162{ 2163 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2164 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB); 2165} 2166static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) 2167{ 2168 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2169 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); 2170} 2171static inline int hifn_decrypt_des_cfb(struct skcipher_request *req) 2172{ 2173 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2174 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB); 2175} 2176static inline int hifn_decrypt_des_ofb(struct skcipher_request *req) 2177{ 2178 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2179 ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB); 2180} 2181 2182/* 2183 * 3DES ecryption functions. 2184 */ 2185static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req) 2186{ 2187 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2188 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2189} 2190static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) 2191{ 2192 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2193 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2194} 2195static inline int hifn_encrypt_3des_cfb(struct skcipher_request *req) 2196{ 2197 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2198 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB); 2199} 2200static inline int hifn_encrypt_3des_ofb(struct skcipher_request *req) 2201{ 2202 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, 2203 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB); 2204} 2205 2206/* 3DES decryption functions. */ 2207static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) 2208{ 2209 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2210 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB); 2211} 2212static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) 2213{ 2214 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2215 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); 2216} 2217static inline int hifn_decrypt_3des_cfb(struct skcipher_request *req) 2218{ 2219 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2220 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB); 2221} 2222static inline int hifn_decrypt_3des_ofb(struct skcipher_request *req) 2223{ 2224 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, 2225 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB); 2226} 2227 2228struct hifn_alg_template { 2229 char name[CRYPTO_MAX_ALG_NAME]; 2230 char drv_name[CRYPTO_MAX_ALG_NAME]; 2231 unsigned int bsize; 2232 struct skcipher_alg skcipher; 2233}; 2234 2235static const struct hifn_alg_template hifn_alg_templates[] = { 2236 /* 2237 * 3DES ECB, CBC, CFB and OFB modes. 2238 */ 2239 { 2240 .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8, 2241 .skcipher = { 2242 .min_keysize = HIFN_3DES_KEY_LENGTH, 2243 .max_keysize = HIFN_3DES_KEY_LENGTH, 2244 .setkey = hifn_des3_setkey, 2245 .encrypt = hifn_encrypt_3des_cfb, 2246 .decrypt = hifn_decrypt_3des_cfb, 2247 }, 2248 }, 2249 { 2250 .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8, 2251 .skcipher = { 2252 .min_keysize = HIFN_3DES_KEY_LENGTH, 2253 .max_keysize = HIFN_3DES_KEY_LENGTH, 2254 .setkey = hifn_des3_setkey, 2255 .encrypt = hifn_encrypt_3des_ofb, 2256 .decrypt = hifn_decrypt_3des_ofb, 2257 }, 2258 }, 2259 { 2260 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, 2261 .skcipher = { 2262 .ivsize = HIFN_IV_LENGTH, 2263 .min_keysize = HIFN_3DES_KEY_LENGTH, 2264 .max_keysize = HIFN_3DES_KEY_LENGTH, 2265 .setkey = hifn_des3_setkey, 2266 .encrypt = hifn_encrypt_3des_cbc, 2267 .decrypt = hifn_decrypt_3des_cbc, 2268 }, 2269 }, 2270 { 2271 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, 2272 .skcipher = { 2273 .min_keysize = HIFN_3DES_KEY_LENGTH, 2274 .max_keysize = HIFN_3DES_KEY_LENGTH, 2275 .setkey = hifn_des3_setkey, 2276 .encrypt = hifn_encrypt_3des_ecb, 2277 .decrypt = hifn_decrypt_3des_ecb, 2278 }, 2279 }, 2280 2281 /* 2282 * DES ECB, CBC, CFB and OFB modes. 2283 */ 2284 { 2285 .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8, 2286 .skcipher = { 2287 .min_keysize = HIFN_DES_KEY_LENGTH, 2288 .max_keysize = HIFN_DES_KEY_LENGTH, 2289 .setkey = hifn_setkey, 2290 .encrypt = hifn_encrypt_des_cfb, 2291 .decrypt = hifn_decrypt_des_cfb, 2292 }, 2293 }, 2294 { 2295 .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8, 2296 .skcipher = { 2297 .min_keysize = HIFN_DES_KEY_LENGTH, 2298 .max_keysize = HIFN_DES_KEY_LENGTH, 2299 .setkey = hifn_setkey, 2300 .encrypt = hifn_encrypt_des_ofb, 2301 .decrypt = hifn_decrypt_des_ofb, 2302 }, 2303 }, 2304 { 2305 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, 2306 .skcipher = { 2307 .ivsize = HIFN_IV_LENGTH, 2308 .min_keysize = HIFN_DES_KEY_LENGTH, 2309 .max_keysize = HIFN_DES_KEY_LENGTH, 2310 .setkey = hifn_setkey, 2311 .encrypt = hifn_encrypt_des_cbc, 2312 .decrypt = hifn_decrypt_des_cbc, 2313 }, 2314 }, 2315 { 2316 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, 2317 .skcipher = { 2318 .min_keysize = HIFN_DES_KEY_LENGTH, 2319 .max_keysize = HIFN_DES_KEY_LENGTH, 2320 .setkey = hifn_setkey, 2321 .encrypt = hifn_encrypt_des_ecb, 2322 .decrypt = hifn_decrypt_des_ecb, 2323 }, 2324 }, 2325 2326 /* 2327 * AES ECB, CBC, CFB and OFB modes. 2328 */ 2329 { 2330 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, 2331 .skcipher = { 2332 .min_keysize = AES_MIN_KEY_SIZE, 2333 .max_keysize = AES_MAX_KEY_SIZE, 2334 .setkey = hifn_setkey, 2335 .encrypt = hifn_encrypt_aes_ecb, 2336 .decrypt = hifn_decrypt_aes_ecb, 2337 }, 2338 }, 2339 { 2340 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, 2341 .skcipher = { 2342 .ivsize = HIFN_AES_IV_LENGTH, 2343 .min_keysize = AES_MIN_KEY_SIZE, 2344 .max_keysize = AES_MAX_KEY_SIZE, 2345 .setkey = hifn_setkey, 2346 .encrypt = hifn_encrypt_aes_cbc, 2347 .decrypt = hifn_decrypt_aes_cbc, 2348 }, 2349 }, 2350 { 2351 .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16, 2352 .skcipher = { 2353 .min_keysize = AES_MIN_KEY_SIZE, 2354 .max_keysize = AES_MAX_KEY_SIZE, 2355 .setkey = hifn_setkey, 2356 .encrypt = hifn_encrypt_aes_cfb, 2357 .decrypt = hifn_decrypt_aes_cfb, 2358 }, 2359 }, 2360 { 2361 .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16, 2362 .skcipher = { 2363 .min_keysize = AES_MIN_KEY_SIZE, 2364 .max_keysize = AES_MAX_KEY_SIZE, 2365 .setkey = hifn_setkey, 2366 .encrypt = hifn_encrypt_aes_ofb, 2367 .decrypt = hifn_decrypt_aes_ofb, 2368 }, 2369 }, 2370}; 2371 2372static int hifn_init_tfm(struct crypto_skcipher *tfm) 2373{ 2374 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 2375 struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg); 2376 struct hifn_context *ctx = crypto_skcipher_ctx(tfm); 2377 2378 ctx->dev = ha->dev; 2379 crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context)); 2380 2381 return 0; 2382} 2383 2384static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t) 2385{ 2386 struct hifn_crypto_alg *alg; 2387 int err; 2388 2389 alg = kzalloc(sizeof(*alg), GFP_KERNEL); 2390 if (!alg) 2391 return -ENOMEM; 2392 2393 alg->alg = t->skcipher; 2394 alg->alg.init = hifn_init_tfm; 2395 2396 snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name); 2397 snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s", 2398 t->drv_name, dev->name); 2399 2400 alg->alg.base.cra_priority = 300; 2401 alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; 2402 alg->alg.base.cra_blocksize = t->bsize; 2403 alg->alg.base.cra_ctxsize = sizeof(struct hifn_context); 2404 alg->alg.base.cra_alignmask = 0; 2405 alg->alg.base.cra_module = THIS_MODULE; 2406 2407 alg->dev = dev; 2408 2409 list_add_tail(&alg->entry, &dev->alg_list); 2410 2411 err = crypto_register_skcipher(&alg->alg); 2412 if (err) { 2413 list_del(&alg->entry); 2414 kfree(alg); 2415 } 2416 2417 return err; 2418} 2419 2420static void hifn_unregister_alg(struct hifn_device *dev) 2421{ 2422 struct hifn_crypto_alg *a, *n; 2423 2424 list_for_each_entry_safe(a, n, &dev->alg_list, entry) { 2425 list_del(&a->entry); 2426 crypto_unregister_skcipher(&a->alg); 2427 kfree(a); 2428 } 2429} 2430 2431static int hifn_register_alg(struct hifn_device *dev) 2432{ 2433 int i, err; 2434 2435 for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) { 2436 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]); 2437 if (err) 2438 goto err_out_exit; 2439 } 2440 2441 return 0; 2442 2443err_out_exit: 2444 hifn_unregister_alg(dev); 2445 return err; 2446} 2447 2448static void hifn_tasklet_callback(unsigned long data) 2449{ 2450 struct hifn_device *dev = (struct hifn_device *)data; 2451 2452 /* 2453 * This is ok to call this without lock being held, 2454 * althogh it modifies some parameters used in parallel, 2455 * (like dev->success), but they are used in process 2456 * context or update is atomic (like setting dev->sa[i] to NULL). 2457 */ 2458 hifn_clear_rings(dev, 0); 2459 2460 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2461 hifn_process_queue(dev); 2462} 2463 2464static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2465{ 2466 int err, i; 2467 struct hifn_device *dev; 2468 char name[8]; 2469 2470 err = pci_enable_device(pdev); 2471 if (err) 2472 return err; 2473 pci_set_master(pdev); 2474 2475 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 2476 if (err) 2477 goto err_out_disable_pci_device; 2478 2479 snprintf(name, sizeof(name), "hifn%d", 2480 atomic_inc_return(&hifn_dev_number) - 1); 2481 2482 err = pci_request_regions(pdev, name); 2483 if (err) 2484 goto err_out_disable_pci_device; 2485 2486 if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE || 2487 pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE || 2488 pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) { 2489 dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); 2490 err = -ENODEV; 2491 goto err_out_free_regions; 2492 } 2493 2494 dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg), 2495 GFP_KERNEL); 2496 if (!dev) { 2497 err = -ENOMEM; 2498 goto err_out_free_regions; 2499 } 2500 2501 INIT_LIST_HEAD(&dev->alg_list); 2502 2503 snprintf(dev->name, sizeof(dev->name), "%s", name); 2504 spin_lock_init(&dev->lock); 2505 2506 for (i = 0; i < 3; ++i) { 2507 unsigned long addr, size; 2508 2509 addr = pci_resource_start(pdev, i); 2510 size = pci_resource_len(pdev, i); 2511 2512 dev->bar[i] = ioremap(addr, size); 2513 if (!dev->bar[i]) { 2514 err = -ENOMEM; 2515 goto err_out_unmap_bars; 2516 } 2517 } 2518 2519 dev->desc_virt = dma_alloc_coherent(&pdev->dev, 2520 sizeof(struct hifn_dma), 2521 &dev->desc_dma, GFP_KERNEL); 2522 if (!dev->desc_virt) { 2523 dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); 2524 err = -ENOMEM; 2525 goto err_out_unmap_bars; 2526 } 2527 2528 dev->pdev = pdev; 2529 dev->irq = pdev->irq; 2530 2531 for (i = 0; i < HIFN_D_RES_RSIZE; ++i) 2532 dev->sa[i] = NULL; 2533 2534 pci_set_drvdata(pdev, dev); 2535 2536 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev); 2537 2538 crypto_init_queue(&dev->queue, 1); 2539 2540 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); 2541 if (err) { 2542 dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", 2543 dev->irq, err); 2544 dev->irq = 0; 2545 goto err_out_free_desc; 2546 } 2547 2548 err = hifn_start_device(dev); 2549 if (err) 2550 goto err_out_free_irq; 2551 2552 err = hifn_register_rng(dev); 2553 if (err) 2554 goto err_out_stop_device; 2555 2556 err = hifn_register_alg(dev); 2557 if (err) 2558 goto err_out_unregister_rng; 2559 2560 INIT_DELAYED_WORK(&dev->work, hifn_work); 2561 schedule_delayed_work(&dev->work, HZ); 2562 2563 dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " 2564 "successfully registered as %s.\n", 2565 pci_name(pdev), dev->name); 2566 2567 return 0; 2568 2569err_out_unregister_rng: 2570 hifn_unregister_rng(dev); 2571err_out_stop_device: 2572 hifn_reset_dma(dev, 1); 2573 hifn_stop_device(dev); 2574err_out_free_irq: 2575 free_irq(dev->irq, dev); 2576 tasklet_kill(&dev->tasklet); 2577err_out_free_desc: 2578 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt, 2579 dev->desc_dma); 2580 2581err_out_unmap_bars: 2582 for (i = 0; i < 3; ++i) 2583 if (dev->bar[i]) 2584 iounmap(dev->bar[i]); 2585 kfree(dev); 2586 2587err_out_free_regions: 2588 pci_release_regions(pdev); 2589 2590err_out_disable_pci_device: 2591 pci_disable_device(pdev); 2592 2593 return err; 2594} 2595 2596static void hifn_remove(struct pci_dev *pdev) 2597{ 2598 int i; 2599 struct hifn_device *dev; 2600 2601 dev = pci_get_drvdata(pdev); 2602 2603 if (dev) { 2604 cancel_delayed_work_sync(&dev->work); 2605 2606 hifn_unregister_rng(dev); 2607 hifn_unregister_alg(dev); 2608 hifn_reset_dma(dev, 1); 2609 hifn_stop_device(dev); 2610 2611 free_irq(dev->irq, dev); 2612 tasklet_kill(&dev->tasklet); 2613 2614 hifn_flush(dev); 2615 2616 dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), 2617 dev->desc_virt, dev->desc_dma); 2618 for (i = 0; i < 3; ++i) 2619 if (dev->bar[i]) 2620 iounmap(dev->bar[i]); 2621 2622 kfree(dev); 2623 } 2624 2625 pci_release_regions(pdev); 2626 pci_disable_device(pdev); 2627} 2628 2629static struct pci_device_id hifn_pci_tbl[] = { 2630 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) }, 2631 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) }, 2632 { 0 } 2633}; 2634MODULE_DEVICE_TABLE(pci, hifn_pci_tbl); 2635 2636static struct pci_driver hifn_pci_driver = { 2637 .name = "hifn795x", 2638 .id_table = hifn_pci_tbl, 2639 .probe = hifn_probe, 2640 .remove = hifn_remove, 2641}; 2642 2643static int __init hifn_init(void) 2644{ 2645 unsigned int freq; 2646 int err; 2647 2648 if (strncmp(hifn_pll_ref, "ext", 3) && 2649 strncmp(hifn_pll_ref, "pci", 3)) { 2650 pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); 2651 return -EINVAL; 2652 } 2653 2654 /* 2655 * For the 7955/7956 the reference clock frequency must be in the 2656 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, 2657 * but this chip is currently not supported. 2658 */ 2659 if (hifn_pll_ref[3] != '\0') { 2660 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); 2661 if (freq < 20 || freq > 100) { 2662 pr_err("hifn795x: invalid hifn_pll_ref frequency, must" 2663 "be in the range of 20-100"); 2664 return -EINVAL; 2665 } 2666 } 2667 2668 err = pci_register_driver(&hifn_pci_driver); 2669 if (err < 0) { 2670 pr_err("Failed to register PCI driver for %s device.\n", 2671 hifn_pci_driver.name); 2672 return -ENODEV; 2673 } 2674 2675 pr_info("Driver for HIFN 795x crypto accelerator chip " 2676 "has been successfully registered.\n"); 2677 2678 return 0; 2679} 2680 2681static void __exit hifn_fini(void) 2682{ 2683 pci_unregister_driver(&hifn_pci_driver); 2684 2685 pr_info("Driver for HIFN 795x crypto accelerator chip " 2686 "has been successfully unregistered.\n"); 2687} 2688 2689module_init(hifn_init); 2690module_exit(hifn_fini); 2691 2692MODULE_LICENSE("GPL"); 2693MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>"); 2694MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip."); 2695