162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-only */ 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * Accelerated CRC32(C) using AArch64 CRC instructions 462306a36Sopenharmony_ci * 562306a36Sopenharmony_ci * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 662306a36Sopenharmony_ci */ 762306a36Sopenharmony_ci 862306a36Sopenharmony_ci#include <linux/linkage.h> 962306a36Sopenharmony_ci#include <asm/alternative.h> 1062306a36Sopenharmony_ci#include <asm/assembler.h> 1162306a36Sopenharmony_ci 1262306a36Sopenharmony_ci .arch armv8-a+crc 1362306a36Sopenharmony_ci 1462306a36Sopenharmony_ci .macro byteorder, reg, be 1562306a36Sopenharmony_ci .if \be 1662306a36Sopenharmony_ciCPU_LE( rev \reg, \reg ) 1762306a36Sopenharmony_ci .else 1862306a36Sopenharmony_ciCPU_BE( rev \reg, \reg ) 1962306a36Sopenharmony_ci .endif 2062306a36Sopenharmony_ci .endm 2162306a36Sopenharmony_ci 2262306a36Sopenharmony_ci .macro byteorder16, reg, be 2362306a36Sopenharmony_ci .if \be 2462306a36Sopenharmony_ciCPU_LE( rev16 \reg, \reg ) 2562306a36Sopenharmony_ci .else 2662306a36Sopenharmony_ciCPU_BE( rev16 \reg, \reg ) 2762306a36Sopenharmony_ci .endif 2862306a36Sopenharmony_ci .endm 2962306a36Sopenharmony_ci 3062306a36Sopenharmony_ci .macro bitorder, reg, be 3162306a36Sopenharmony_ci .if \be 3262306a36Sopenharmony_ci rbit \reg, \reg 3362306a36Sopenharmony_ci .endif 3462306a36Sopenharmony_ci .endm 3562306a36Sopenharmony_ci 3662306a36Sopenharmony_ci .macro bitorder16, reg, be 3762306a36Sopenharmony_ci .if \be 3862306a36Sopenharmony_ci rbit \reg, \reg 3962306a36Sopenharmony_ci lsr \reg, \reg, #16 4062306a36Sopenharmony_ci .endif 4162306a36Sopenharmony_ci .endm 4262306a36Sopenharmony_ci 4362306a36Sopenharmony_ci .macro bitorder8, reg, be 4462306a36Sopenharmony_ci .if \be 4562306a36Sopenharmony_ci rbit \reg, \reg 4662306a36Sopenharmony_ci lsr \reg, \reg, #24 4762306a36Sopenharmony_ci .endif 4862306a36Sopenharmony_ci .endm 4962306a36Sopenharmony_ci 5062306a36Sopenharmony_ci .macro __crc32, c, be=0 5162306a36Sopenharmony_ci bitorder w0, \be 5262306a36Sopenharmony_ci cmp x2, #16 5362306a36Sopenharmony_ci b.lt 8f // less than 16 bytes 5462306a36Sopenharmony_ci 5562306a36Sopenharmony_ci and x7, x2, #0x1f 5662306a36Sopenharmony_ci and x2, x2, #~0x1f 5762306a36Sopenharmony_ci cbz x7, 32f // multiple of 32 bytes 5862306a36Sopenharmony_ci 5962306a36Sopenharmony_ci and x8, x7, #0xf 6062306a36Sopenharmony_ci ldp x3, x4, [x1] 6162306a36Sopenharmony_ci add x8, x8, x1 6262306a36Sopenharmony_ci add x1, x1, x7 6362306a36Sopenharmony_ci ldp x5, x6, [x8] 6462306a36Sopenharmony_ci byteorder x3, \be 6562306a36Sopenharmony_ci byteorder x4, \be 6662306a36Sopenharmony_ci byteorder x5, \be 6762306a36Sopenharmony_ci byteorder x6, \be 6862306a36Sopenharmony_ci bitorder x3, \be 6962306a36Sopenharmony_ci bitorder x4, \be 7062306a36Sopenharmony_ci bitorder x5, \be 7162306a36Sopenharmony_ci bitorder x6, \be 7262306a36Sopenharmony_ci 7362306a36Sopenharmony_ci tst x7, #8 7462306a36Sopenharmony_ci crc32\c\()x w8, w0, x3 7562306a36Sopenharmony_ci csel x3, x3, x4, eq 7662306a36Sopenharmony_ci csel w0, w0, w8, eq 7762306a36Sopenharmony_ci tst x7, #4 7862306a36Sopenharmony_ci lsr x4, x3, #32 7962306a36Sopenharmony_ci crc32\c\()w w8, w0, w3 8062306a36Sopenharmony_ci csel x3, x3, x4, eq 8162306a36Sopenharmony_ci csel w0, w0, w8, eq 8262306a36Sopenharmony_ci tst x7, #2 8362306a36Sopenharmony_ci lsr w4, w3, #16 8462306a36Sopenharmony_ci crc32\c\()h w8, w0, w3 8562306a36Sopenharmony_ci csel w3, w3, w4, eq 8662306a36Sopenharmony_ci csel w0, w0, w8, eq 8762306a36Sopenharmony_ci tst x7, #1 8862306a36Sopenharmony_ci crc32\c\()b w8, w0, w3 8962306a36Sopenharmony_ci csel w0, w0, w8, eq 9062306a36Sopenharmony_ci tst x7, #16 9162306a36Sopenharmony_ci crc32\c\()x w8, w0, x5 9262306a36Sopenharmony_ci crc32\c\()x w8, w8, x6 9362306a36Sopenharmony_ci csel w0, w0, w8, eq 9462306a36Sopenharmony_ci cbz x2, 0f 9562306a36Sopenharmony_ci 9662306a36Sopenharmony_ci32: ldp x3, x4, [x1], #32 9762306a36Sopenharmony_ci sub x2, x2, #32 9862306a36Sopenharmony_ci ldp x5, x6, [x1, #-16] 9962306a36Sopenharmony_ci byteorder x3, \be 10062306a36Sopenharmony_ci byteorder x4, \be 10162306a36Sopenharmony_ci byteorder x5, \be 10262306a36Sopenharmony_ci byteorder x6, \be 10362306a36Sopenharmony_ci bitorder x3, \be 10462306a36Sopenharmony_ci bitorder x4, \be 10562306a36Sopenharmony_ci bitorder x5, \be 10662306a36Sopenharmony_ci bitorder x6, \be 10762306a36Sopenharmony_ci crc32\c\()x w0, w0, x3 10862306a36Sopenharmony_ci crc32\c\()x w0, w0, x4 10962306a36Sopenharmony_ci crc32\c\()x w0, w0, x5 11062306a36Sopenharmony_ci crc32\c\()x w0, w0, x6 11162306a36Sopenharmony_ci cbnz x2, 32b 11262306a36Sopenharmony_ci0: bitorder w0, \be 11362306a36Sopenharmony_ci ret 11462306a36Sopenharmony_ci 11562306a36Sopenharmony_ci8: tbz x2, #3, 4f 11662306a36Sopenharmony_ci ldr x3, [x1], #8 11762306a36Sopenharmony_ci byteorder x3, \be 11862306a36Sopenharmony_ci bitorder x3, \be 11962306a36Sopenharmony_ci crc32\c\()x w0, w0, x3 12062306a36Sopenharmony_ci4: tbz x2, #2, 2f 12162306a36Sopenharmony_ci ldr w3, [x1], #4 12262306a36Sopenharmony_ci byteorder w3, \be 12362306a36Sopenharmony_ci bitorder w3, \be 12462306a36Sopenharmony_ci crc32\c\()w w0, w0, w3 12562306a36Sopenharmony_ci2: tbz x2, #1, 1f 12662306a36Sopenharmony_ci ldrh w3, [x1], #2 12762306a36Sopenharmony_ci byteorder16 w3, \be 12862306a36Sopenharmony_ci bitorder16 w3, \be 12962306a36Sopenharmony_ci crc32\c\()h w0, w0, w3 13062306a36Sopenharmony_ci1: tbz x2, #0, 0f 13162306a36Sopenharmony_ci ldrb w3, [x1] 13262306a36Sopenharmony_ci bitorder8 w3, \be 13362306a36Sopenharmony_ci crc32\c\()b w0, w0, w3 13462306a36Sopenharmony_ci0: bitorder w0, \be 13562306a36Sopenharmony_ci ret 13662306a36Sopenharmony_ci .endm 13762306a36Sopenharmony_ci 13862306a36Sopenharmony_ci .align 5 13962306a36Sopenharmony_ciSYM_FUNC_START(crc32_le) 14062306a36Sopenharmony_cialternative_if_not ARM64_HAS_CRC32 14162306a36Sopenharmony_ci b crc32_le_base 14262306a36Sopenharmony_cialternative_else_nop_endif 14362306a36Sopenharmony_ci __crc32 14462306a36Sopenharmony_ciSYM_FUNC_END(crc32_le) 14562306a36Sopenharmony_ci 14662306a36Sopenharmony_ci .align 5 14762306a36Sopenharmony_ciSYM_FUNC_START(__crc32c_le) 14862306a36Sopenharmony_cialternative_if_not ARM64_HAS_CRC32 14962306a36Sopenharmony_ci b __crc32c_le_base 15062306a36Sopenharmony_cialternative_else_nop_endif 15162306a36Sopenharmony_ci __crc32 c 15262306a36Sopenharmony_ciSYM_FUNC_END(__crc32c_le) 15362306a36Sopenharmony_ci 15462306a36Sopenharmony_ci .align 5 15562306a36Sopenharmony_ciSYM_FUNC_START(crc32_be) 15662306a36Sopenharmony_cialternative_if_not ARM64_HAS_CRC32 15762306a36Sopenharmony_ci b crc32_be_base 15862306a36Sopenharmony_cialternative_else_nop_endif 15962306a36Sopenharmony_ci __crc32 be=1 16062306a36Sopenharmony_ciSYM_FUNC_END(crc32_be) 161