1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2012 Intel Corporation 4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 */ 6 7#include <linux/raid/pq.h> 8 9#ifdef __KERNEL__ 10#include <asm/neon.h> 11#include "neon.h" 12#else 13#define kernel_neon_begin() 14#define kernel_neon_end() 15#define cpu_has_neon() (1) 16#endif 17 18static int raid6_has_neon(void) 19{ 20 return cpu_has_neon(); 21} 22 23static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, 24 int failb, void **ptrs) 25{ 26 u8 *p, *q, *dp, *dq; 27 const u8 *pbmul; /* P multiplier table for B data */ 28 const u8 *qmul; /* Q multiplier table (for both) */ 29 30 p = (u8 *)ptrs[disks - 2]; 31 q = (u8 *)ptrs[disks - 1]; 32 33 /* 34 * Compute syndrome with zero for the missing data pages 35 * Use the dead data pages as temporary storage for 36 * delta p and delta q 37 */ 38 dp = (u8 *)ptrs[faila]; 39 ptrs[faila] = (void *)raid6_empty_zero_page; 40 ptrs[disks - 2] = dp; 41 dq = (u8 *)ptrs[failb]; 42 ptrs[failb] = (void *)raid6_empty_zero_page; 43 ptrs[disks - 1] = dq; 44 45 raid6_call.gen_syndrome(disks, bytes, ptrs); 46 47 /* Restore pointer table */ 48 ptrs[faila] = dp; 49 ptrs[failb] = dq; 50 ptrs[disks - 2] = p; 51 ptrs[disks - 1] = q; 52 53 /* Now, pick the proper data tables */ 54 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; 55 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 56 raid6_gfexp[failb]]]; 57 58 kernel_neon_begin(); 59 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); 60 kernel_neon_end(); 61} 62 63static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, 64 void **ptrs) 65{ 66 u8 *p, *q, *dq; 67 const u8 *qmul; /* Q multiplier table */ 68 69 p = (u8 *)ptrs[disks - 2]; 70 q = (u8 *)ptrs[disks - 1]; 71 72 /* 73 * Compute syndrome with zero for the missing data page 74 * Use the dead data page as temporary storage for delta q 75 */ 76 dq = (u8 *)ptrs[faila]; 77 ptrs[faila] = (void *)raid6_empty_zero_page; 78 ptrs[disks - 1] = dq; 79 80 raid6_call.gen_syndrome(disks, bytes, ptrs); 81 82 /* Restore pointer table */ 83 ptrs[faila] = dq; 84 ptrs[disks - 1] = q; 85 86 /* Now, pick the proper data tables */ 87 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 88 89 kernel_neon_begin(); 90 __raid6_datap_recov_neon(bytes, p, q, dq, qmul); 91 kernel_neon_end(); 92} 93 94const struct raid6_recov_calls raid6_recov_neon = { 95 .data2 = raid6_2data_recov_neon, 96 .datap = raid6_datap_recov_neon, 97 .valid = raid6_has_neon, 98 .name = "neon", 99 .priority = 10, 100}; 101