1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Accelerated CRC32(C) using AArch64 CRC instructions 4 * 5 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8 #include <linux/linkage.h> 9 #include <asm/alternative.h> 10 #include <asm/assembler.h> 11 12 .arch armv8-a+crc 13 14 .macro __crc32, c 15 cmp x2, #16 16 b.lt 8f // less than 16 bytes 17 18 and x7, x2, #0x1f 19 and x2, x2, #~0x1f 20 cbz x7, 32f // multiple of 32 bytes 21 22 and x8, x7, #0xf 23 ldp x3, x4, [x1] 24 add x8, x8, x1 25 add x1, x1, x7 26 ldp x5, x6, [x8] 27 CPU_BE( rev x3, x3 ) 28 CPU_BE( rev x4, x4 ) 29 CPU_BE( rev x5, x5 ) 30 CPU_BE( rev x6, x6 ) 31 32 tst x7, #8 33 crc32\c\()x w8, w0, x3 34 csel x3, x3, x4, eq 35 csel w0, w0, w8, eq 36 tst x7, #4 37 lsr x4, x3, #32 38 crc32\c\()w w8, w0, w3 39 csel x3, x3, x4, eq 40 csel w0, w0, w8, eq 41 tst x7, #2 42 lsr w4, w3, #16 43 crc32\c\()h w8, w0, w3 44 csel w3, w3, w4, eq 45 csel w0, w0, w8, eq 46 tst x7, #1 47 crc32\c\()b w8, w0, w3 48 csel w0, w0, w8, eq 49 tst x7, #16 50 crc32\c\()x w8, w0, x5 51 crc32\c\()x w8, w8, x6 52 csel w0, w0, w8, eq 53 cbz x2, 0f 54 55 32: ldp x3, x4, [x1], #32 56 sub x2, x2, #32 57 ldp x5, x6, [x1, #-16] 58 CPU_BE( rev x3, x3 ) 59 CPU_BE( rev x4, x4 ) 60 CPU_BE( rev x5, x5 ) 61 CPU_BE( rev x6, x6 ) 62 crc32\c\()x w0, w0, x3 63 crc32\c\()x w0, w0, x4 64 crc32\c\()x w0, w0, x5 65 crc32\c\()x w0, w0, x6 66 cbnz x2, 32b 67 0: ret 68 69 8: tbz x2, #3, 4f 70 ldr x3, [x1], #8 71 CPU_BE( rev x3, x3 ) 72 crc32\c\()x w0, w0, x3 73 4: tbz x2, #2, 2f 74 ldr w3, [x1], #4 75 CPU_BE( rev w3, w3 ) 76 crc32\c\()w w0, w0, w3 77 2: tbz x2, #1, 1f 78 ldrh w3, [x1], #2 79 CPU_BE( rev16 w3, w3 ) 80 crc32\c\()h w0, w0, w3 81 1: tbz x2, #0, 0f 82 ldrb w3, [x1] 83 crc32\c\()b w0, w0, w3 84 0: ret 85 .endm 86 87 .align 5 88 SYM_FUNC_START(crc32_le) 89 alternative_if_not ARM64_HAS_CRC32 90 b crc32_le_base 91 alternative_else_nop_endif 92 __crc32 93 SYM_FUNC_END(crc32_le) 94 95 .align 5 96 SYM_FUNC_START(__crc32c_le) 97 alternative_if_not ARM64_HAS_CRC32 98 b __crc32c_le_base 99 alternative_else_nop_endif 100 __crc32 c 101 SYM_FUNC_END(__crc32c_le) 102