1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * NH - ε-almost-universal hash function, NEON accelerated version 4 * 5 * Copyright 2018 Google LLC 6 * 7 * Author: Eric Biggers <ebiggers@google.com> 8 */ 9 10 #include <linux/linkage.h> 11 12 .text 13 .fpu neon 14 15 KEY .req r0 16 MESSAGE .req r1 17 MESSAGE_LEN .req r2 18 HASH .req r3 19 20 PASS0_SUMS .req q0 21 PASS0_SUM_A .req d0 22 PASS0_SUM_B .req d1 23 PASS1_SUMS .req q1 24 PASS1_SUM_A .req d2 25 PASS1_SUM_B .req d3 26 PASS2_SUMS .req q2 27 PASS2_SUM_A .req d4 28 PASS2_SUM_B .req d5 29 PASS3_SUMS .req q3 30 PASS3_SUM_A .req d6 31 PASS3_SUM_B .req d7 32 K0 .req q4 33 K1 .req q5 34 K2 .req q6 35 K3 .req q7 36 T0 .req q8 37 T0_L .req d16 38 T0_H .req d17 39 T1 .req q9 40 T1_L .req d18 41 T1_H .req d19 42 T2 .req q10 43 T2_L .req d20 44 T2_H .req d21 45 T3 .req q11 46 T3_L .req d22 47 T3_H .req d23 48 49 .macro _nh_stride k0, k1, k2, k3 50 51 // Load next message stride 52 vld1.8 {T3}, [MESSAGE]! 53 54 // Load next key stride 55 vld1.32 {\k3}, [KEY]! 56 57 // Add message words to key words 58 vadd.u32 T0, T3, \k0 59 vadd.u32 T1, T3, \k1 60 vadd.u32 T2, T3, \k2 61 vadd.u32 T3, T3, \k3 62 63 // Multiply 32x32 => 64 and accumulate 64 vmlal.u32 PASS0_SUMS, T0_L, T0_H 65 vmlal.u32 PASS1_SUMS, T1_L, T1_H 66 vmlal.u32 PASS2_SUMS, T2_L, T2_H 67 vmlal.u32 PASS3_SUMS, T3_L, T3_H 68 .endm 69 70 /* 71 * void nh_neon(const u32 *key, const u8 *message, size_t message_len, 72 * __le64 hash[NH_NUM_PASSES]) 73 * 74 * It's guaranteed that message_len % 16 == 0. 75 */ 76 ENTRY(nh_neon) 77 78 vld1.32 {K0,K1}, [KEY]! 79 vmov.u64 PASS0_SUMS, #0 80 vmov.u64 PASS1_SUMS, #0 81 vld1.32 {K2}, [KEY]! 82 vmov.u64 PASS2_SUMS, #0 83 vmov.u64 PASS3_SUMS, #0 84 85 subs MESSAGE_LEN, MESSAGE_LEN, #64 86 blt .Lloop4_done 87 .Lloop4: 88 _nh_stride K0, K1, K2, K3 89 _nh_stride K1, K2, K3, K0 90 _nh_stride K2, K3, K0, K1 91 _nh_stride K3, K0, K1, K2 92 subs MESSAGE_LEN, MESSAGE_LEN, #64 93 bge .Lloop4 94 95 .Lloop4_done: 96 ands MESSAGE_LEN, MESSAGE_LEN, #63 97 beq .Ldone 98 _nh_stride K0, K1, K2, K3 99 100 subs MESSAGE_LEN, MESSAGE_LEN, #16 101 beq .Ldone 102 _nh_stride K1, K2, K3, K0 103 104 subs MESSAGE_LEN, MESSAGE_LEN, #16 105 beq .Ldone 106 _nh_stride K2, K3, K0, K1 107 108 .Ldone: 109 // Sum the accumulators for each pass, then store the sums to 'hash' 110 vadd.u64 T0_L, PASS0_SUM_A, PASS0_SUM_B 111 vadd.u64 T0_H, PASS1_SUM_A, PASS1_SUM_B 112 vadd.u64 T1_L, PASS2_SUM_A, PASS2_SUM_B 113 vadd.u64 T1_H, PASS3_SUM_A, PASS3_SUM_B 114 vst1.8 {T0-T1}, [HASH] 115 bx lr 116 ENDPROC(nh_neon) 117