18c2ecf20Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0-only */ 28c2ecf20Sopenharmony_ci/* 38c2ecf20Sopenharmony_ci * Accelerated CRC32(C) using AArch64 CRC instructions 48c2ecf20Sopenharmony_ci * 58c2ecf20Sopenharmony_ci * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 68c2ecf20Sopenharmony_ci */ 78c2ecf20Sopenharmony_ci 88c2ecf20Sopenharmony_ci#include <linux/linkage.h> 98c2ecf20Sopenharmony_ci#include <asm/alternative.h> 108c2ecf20Sopenharmony_ci#include <asm/assembler.h> 118c2ecf20Sopenharmony_ci 128c2ecf20Sopenharmony_ci .arch armv8-a+crc 138c2ecf20Sopenharmony_ci 148c2ecf20Sopenharmony_ci .macro __crc32, c 158c2ecf20Sopenharmony_ci cmp x2, #16 168c2ecf20Sopenharmony_ci b.lt 8f // less than 16 bytes 178c2ecf20Sopenharmony_ci 188c2ecf20Sopenharmony_ci and x7, x2, #0x1f 198c2ecf20Sopenharmony_ci and x2, x2, #~0x1f 208c2ecf20Sopenharmony_ci cbz x7, 32f // multiple of 32 bytes 218c2ecf20Sopenharmony_ci 228c2ecf20Sopenharmony_ci and x8, x7, #0xf 238c2ecf20Sopenharmony_ci ldp x3, x4, [x1] 248c2ecf20Sopenharmony_ci add x8, x8, x1 258c2ecf20Sopenharmony_ci add x1, x1, x7 268c2ecf20Sopenharmony_ci ldp x5, x6, [x8] 278c2ecf20Sopenharmony_ciCPU_BE( rev x3, x3 ) 288c2ecf20Sopenharmony_ciCPU_BE( rev x4, x4 ) 298c2ecf20Sopenharmony_ciCPU_BE( rev x5, x5 ) 308c2ecf20Sopenharmony_ciCPU_BE( rev x6, x6 ) 318c2ecf20Sopenharmony_ci 328c2ecf20Sopenharmony_ci tst x7, #8 338c2ecf20Sopenharmony_ci crc32\c\()x w8, w0, x3 348c2ecf20Sopenharmony_ci csel x3, x3, x4, eq 358c2ecf20Sopenharmony_ci csel w0, w0, w8, eq 368c2ecf20Sopenharmony_ci tst x7, #4 378c2ecf20Sopenharmony_ci lsr x4, x3, #32 388c2ecf20Sopenharmony_ci crc32\c\()w w8, w0, w3 398c2ecf20Sopenharmony_ci csel x3, x3, x4, eq 408c2ecf20Sopenharmony_ci csel w0, w0, w8, eq 418c2ecf20Sopenharmony_ci tst x7, #2 428c2ecf20Sopenharmony_ci lsr w4, w3, #16 438c2ecf20Sopenharmony_ci crc32\c\()h w8, w0, w3 448c2ecf20Sopenharmony_ci csel w3, w3, w4, eq 458c2ecf20Sopenharmony_ci csel w0, w0, w8, eq 468c2ecf20Sopenharmony_ci tst x7, #1 478c2ecf20Sopenharmony_ci crc32\c\()b w8, w0, w3 488c2ecf20Sopenharmony_ci csel w0, w0, w8, eq 498c2ecf20Sopenharmony_ci tst x7, #16 508c2ecf20Sopenharmony_ci crc32\c\()x w8, w0, x5 518c2ecf20Sopenharmony_ci crc32\c\()x w8, w8, x6 528c2ecf20Sopenharmony_ci csel w0, w0, w8, eq 538c2ecf20Sopenharmony_ci cbz x2, 0f 548c2ecf20Sopenharmony_ci 558c2ecf20Sopenharmony_ci32: ldp x3, x4, [x1], #32 568c2ecf20Sopenharmony_ci sub x2, x2, #32 578c2ecf20Sopenharmony_ci ldp x5, x6, [x1, #-16] 588c2ecf20Sopenharmony_ciCPU_BE( rev x3, x3 ) 598c2ecf20Sopenharmony_ciCPU_BE( rev x4, x4 ) 608c2ecf20Sopenharmony_ciCPU_BE( rev x5, x5 ) 618c2ecf20Sopenharmony_ciCPU_BE( rev x6, x6 ) 628c2ecf20Sopenharmony_ci crc32\c\()x w0, w0, x3 638c2ecf20Sopenharmony_ci crc32\c\()x w0, w0, x4 648c2ecf20Sopenharmony_ci crc32\c\()x w0, w0, x5 658c2ecf20Sopenharmony_ci crc32\c\()x w0, w0, x6 668c2ecf20Sopenharmony_ci cbnz x2, 32b 678c2ecf20Sopenharmony_ci0: ret 688c2ecf20Sopenharmony_ci 698c2ecf20Sopenharmony_ci8: tbz x2, #3, 4f 708c2ecf20Sopenharmony_ci ldr x3, [x1], #8 718c2ecf20Sopenharmony_ciCPU_BE( rev x3, x3 ) 728c2ecf20Sopenharmony_ci crc32\c\()x w0, w0, x3 738c2ecf20Sopenharmony_ci4: tbz x2, #2, 2f 748c2ecf20Sopenharmony_ci ldr w3, [x1], #4 758c2ecf20Sopenharmony_ciCPU_BE( rev w3, w3 ) 768c2ecf20Sopenharmony_ci crc32\c\()w w0, w0, w3 778c2ecf20Sopenharmony_ci2: tbz x2, #1, 1f 788c2ecf20Sopenharmony_ci ldrh w3, [x1], #2 798c2ecf20Sopenharmony_ciCPU_BE( rev16 w3, w3 ) 808c2ecf20Sopenharmony_ci crc32\c\()h w0, w0, w3 818c2ecf20Sopenharmony_ci1: tbz x2, #0, 0f 828c2ecf20Sopenharmony_ci ldrb w3, [x1] 838c2ecf20Sopenharmony_ci crc32\c\()b w0, w0, w3 848c2ecf20Sopenharmony_ci0: ret 858c2ecf20Sopenharmony_ci .endm 868c2ecf20Sopenharmony_ci 878c2ecf20Sopenharmony_ci .align 5 888c2ecf20Sopenharmony_ciSYM_FUNC_START(crc32_le) 898c2ecf20Sopenharmony_cialternative_if_not ARM64_HAS_CRC32 908c2ecf20Sopenharmony_ci b crc32_le_base 918c2ecf20Sopenharmony_cialternative_else_nop_endif 928c2ecf20Sopenharmony_ci __crc32 938c2ecf20Sopenharmony_ciSYM_FUNC_END(crc32_le) 948c2ecf20Sopenharmony_ci 958c2ecf20Sopenharmony_ci .align 5 968c2ecf20Sopenharmony_ciSYM_FUNC_START(__crc32c_le) 978c2ecf20Sopenharmony_cialternative_if_not ARM64_HAS_CRC32 988c2ecf20Sopenharmony_ci b __crc32c_le_base 998c2ecf20Sopenharmony_cialternative_else_nop_endif 1008c2ecf20Sopenharmony_ci __crc32 c 1018c2ecf20Sopenharmony_ciSYM_FUNC_END(__crc32c_le) 102