1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5#ifndef __ASM_FP_H
6#define __ASM_FP_H
7
8#include <asm/errno.h>
9#include <asm/ptrace.h>
10#include <asm/processor.h>
11#include <asm/sigcontext.h>
12#include <asm/sysreg.h>
13
14#ifndef __ASSEMBLY__
15
16#include <linux/bitmap.h>
17#include <linux/build_bug.h>
18#include <linux/bug.h>
19#include <linux/cache.h>
20#include <linux/init.h>
21#include <linux/stddef.h>
22#include <linux/types.h>
23
24#ifdef CONFIG_COMPAT
25/* Masks for extracting the FPSR and FPCR from the FPSCR */
26#define VFP_FPSCR_STAT_MASK	0xf800009f
27#define VFP_FPSCR_CTRL_MASK	0x07f79f00
28/*
29 * The VFP state has 32x64-bit registers and a single 32-bit
30 * control/status register.
31 */
32#define VFP_STATE_SIZE		((32 * 8) + 4)
33#endif
34
35/*
36 * When we defined the maximum SVE vector length we defined the ABI so
37 * that the maximum vector length included all the reserved for future
38 * expansion bits in ZCR rather than those just currently defined by
39 * the architecture.  Using this length to allocate worst size buffers
40 * results in excessively large allocations, and this effect is even
41 * more pronounced for SME due to ZA.  Define more suitable VLs for
42 * these situations.
43 */
44#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
45#define SME_VQ_MAX	((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
46
47struct task_struct;
48
49extern void fpsimd_save_state(struct user_fpsimd_state *state);
50extern void fpsimd_load_state(struct user_fpsimd_state *state);
51
52extern void fpsimd_thread_switch(struct task_struct *next);
53extern void fpsimd_flush_thread(void);
54
55extern void fpsimd_signal_preserve_current_state(void);
56extern void fpsimd_preserve_current_state(void);
57extern void fpsimd_restore_current_state(void);
58extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
59extern void fpsimd_kvm_prepare(void);
60
61struct cpu_fp_state {
62	struct user_fpsimd_state *st;
63	void *sve_state;
64	void *sme_state;
65	u64 *svcr;
66	unsigned int sve_vl;
67	unsigned int sme_vl;
68	enum fp_type *fp_type;
69	enum fp_type to_save;
70};
71
72extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
73
74extern void fpsimd_flush_task_state(struct task_struct *target);
75extern void fpsimd_save_and_flush_cpu_state(void);
76
77static inline bool thread_sm_enabled(struct thread_struct *thread)
78{
79	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
80}
81
82static inline bool thread_za_enabled(struct thread_struct *thread)
83{
84	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
85}
86
87/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
88#define VL_ARCH_MAX 0x100
89
90/* Offset of FFR in the SVE register dump */
91static inline size_t sve_ffr_offset(int vl)
92{
93	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
94}
95
96static inline void *sve_pffr(struct thread_struct *thread)
97{
98	unsigned int vl;
99
100	if (system_supports_sme() && thread_sm_enabled(thread))
101		vl = thread_get_sme_vl(thread);
102	else
103		vl = thread_get_sve_vl(thread);
104
105	return (char *)thread->sve_state + sve_ffr_offset(vl);
106}
107
108static inline void *thread_zt_state(struct thread_struct *thread)
109{
110	/* The ZT register state is stored immediately after the ZA state */
111	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
112	return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
113}
114
115extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
116extern void sve_load_state(void const *state, u32 const *pfpsr,
117			   int restore_ffr);
118extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
119extern unsigned int sve_get_vl(void);
120extern void sve_set_vq(unsigned long vq_minus_1);
121extern void sme_set_vq(unsigned long vq_minus_1);
122extern void sme_save_state(void *state, int zt);
123extern void sme_load_state(void const *state, int zt);
124
125struct arm64_cpu_capabilities;
126extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
127extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused);
128extern void sme2_kernel_enable(const struct arm64_cpu_capabilities *__unused);
129extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused);
130
131extern u64 read_zcr_features(void);
132extern u64 read_smcr_features(void);
133
134/*
135 * Helpers to translate bit indices in sve_vq_map to VQ values (and
136 * vice versa).  This allows find_next_bit() to be used to find the
137 * _maximum_ VQ not exceeding a certain value.
138 */
139static inline unsigned int __vq_to_bit(unsigned int vq)
140{
141	return SVE_VQ_MAX - vq;
142}
143
144static inline unsigned int __bit_to_vq(unsigned int bit)
145{
146	return SVE_VQ_MAX - bit;
147}
148
149
150struct vl_info {
151	enum vec_type type;
152	const char *name;		/* For display purposes */
153
154	/* Minimum supported vector length across all CPUs */
155	int min_vl;
156
157	/* Maximum supported vector length across all CPUs */
158	int max_vl;
159	int max_virtualisable_vl;
160
161	/*
162	 * Set of available vector lengths,
163	 * where length vq encoded as bit __vq_to_bit(vq):
164	 */
165	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
166
167	/* Set of vector lengths present on at least one cpu: */
168	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
169};
170
171#ifdef CONFIG_ARM64_SVE
172
173extern void sve_alloc(struct task_struct *task, bool flush);
174extern void fpsimd_release_task(struct task_struct *task);
175extern void fpsimd_sync_to_sve(struct task_struct *task);
176extern void fpsimd_force_sync_to_sve(struct task_struct *task);
177extern void sve_sync_to_fpsimd(struct task_struct *task);
178extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
179
180extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
181				 unsigned long vl, unsigned long flags);
182
183extern int sve_set_current_vl(unsigned long arg);
184extern int sve_get_current_vl(void);
185
186static inline void sve_user_disable(void)
187{
188	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
189}
190
191static inline void sve_user_enable(void)
192{
193	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
194}
195
196#define sve_cond_update_zcr_vq(val, reg)		\
197	do {						\
198		u64 __zcr = read_sysreg_s((reg));	\
199		u64 __new = __zcr & ~ZCR_ELx_LEN_MASK;	\
200		__new |= (val) & ZCR_ELx_LEN_MASK;	\
201		if (__zcr != __new)			\
202			write_sysreg_s(__new, (reg));	\
203	} while (0)
204
205/*
206 * Probing and setup functions.
207 * Calls to these functions must be serialised with one another.
208 */
209enum vec_type;
210
211extern void __init vec_init_vq_map(enum vec_type type);
212extern void vec_update_vq_map(enum vec_type type);
213extern int vec_verify_vq_map(enum vec_type type);
214extern void __init sve_setup(void);
215
216extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
217
218static inline void write_vl(enum vec_type type, u64 val)
219{
220	u64 tmp;
221
222	switch (type) {
223#ifdef CONFIG_ARM64_SVE
224	case ARM64_VEC_SVE:
225		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
226		write_sysreg_s(tmp | val, SYS_ZCR_EL1);
227		break;
228#endif
229#ifdef CONFIG_ARM64_SME
230	case ARM64_VEC_SME:
231		tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
232		write_sysreg_s(tmp | val, SYS_SMCR_EL1);
233		break;
234#endif
235	default:
236		WARN_ON_ONCE(1);
237		break;
238	}
239}
240
241static inline int vec_max_vl(enum vec_type type)
242{
243	return vl_info[type].max_vl;
244}
245
246static inline int vec_max_virtualisable_vl(enum vec_type type)
247{
248	return vl_info[type].max_virtualisable_vl;
249}
250
251static inline int sve_max_vl(void)
252{
253	return vec_max_vl(ARM64_VEC_SVE);
254}
255
256static inline int sve_max_virtualisable_vl(void)
257{
258	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
259}
260
261/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
262static inline bool vq_available(enum vec_type type, unsigned int vq)
263{
264	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
265}
266
267static inline bool sve_vq_available(unsigned int vq)
268{
269	return vq_available(ARM64_VEC_SVE, vq);
270}
271
272size_t sve_state_size(struct task_struct const *task);
273
274#else /* ! CONFIG_ARM64_SVE */
275
276static inline void sve_alloc(struct task_struct *task, bool flush) { }
277static inline void fpsimd_release_task(struct task_struct *task) { }
278static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
279static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
280
281static inline int sve_max_virtualisable_vl(void)
282{
283	return 0;
284}
285
286static inline int sve_set_current_vl(unsigned long arg)
287{
288	return -EINVAL;
289}
290
291static inline int sve_get_current_vl(void)
292{
293	return -EINVAL;
294}
295
296static inline int sve_max_vl(void)
297{
298	return -EINVAL;
299}
300
301static inline bool sve_vq_available(unsigned int vq) { return false; }
302
303static inline void sve_user_disable(void) { BUILD_BUG(); }
304static inline void sve_user_enable(void) { BUILD_BUG(); }
305
306#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
307
308static inline void vec_init_vq_map(enum vec_type t) { }
309static inline void vec_update_vq_map(enum vec_type t) { }
310static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
311static inline void sve_setup(void) { }
312
313static inline size_t sve_state_size(struct task_struct const *task)
314{
315	return 0;
316}
317
318#endif /* ! CONFIG_ARM64_SVE */
319
320#ifdef CONFIG_ARM64_SME
321
322static inline void sme_user_disable(void)
323{
324	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
325}
326
327static inline void sme_user_enable(void)
328{
329	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
330}
331
332static inline void sme_smstart_sm(void)
333{
334	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
335}
336
337static inline void sme_smstop_sm(void)
338{
339	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
340}
341
342static inline void sme_smstop(void)
343{
344	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
345}
346
347extern void __init sme_setup(void);
348
349static inline int sme_max_vl(void)
350{
351	return vec_max_vl(ARM64_VEC_SME);
352}
353
354static inline int sme_max_virtualisable_vl(void)
355{
356	return vec_max_virtualisable_vl(ARM64_VEC_SME);
357}
358
359extern void sme_alloc(struct task_struct *task, bool flush);
360extern unsigned int sme_get_vl(void);
361extern int sme_set_current_vl(unsigned long arg);
362extern int sme_get_current_vl(void);
363extern void sme_suspend_exit(void);
364
365/*
366 * Return how many bytes of memory are required to store the full SME
367 * specific state for task, given task's currently configured vector
368 * length.
369 */
370static inline size_t sme_state_size(struct task_struct const *task)
371{
372	unsigned int vl = task_get_sme_vl(task);
373	size_t size;
374
375	size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
376
377	if (system_supports_sme2())
378		size += ZT_SIG_REG_SIZE;
379
380	return size;
381}
382
383#else
384
385static inline void sme_user_disable(void) { BUILD_BUG(); }
386static inline void sme_user_enable(void) { BUILD_BUG(); }
387
388static inline void sme_smstart_sm(void) { }
389static inline void sme_smstop_sm(void) { }
390static inline void sme_smstop(void) { }
391
392static inline void sme_alloc(struct task_struct *task, bool flush) { }
393static inline void sme_setup(void) { }
394static inline unsigned int sme_get_vl(void) { return 0; }
395static inline int sme_max_vl(void) { return 0; }
396static inline int sme_max_virtualisable_vl(void) { return 0; }
397static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
398static inline int sme_get_current_vl(void) { return -EINVAL; }
399static inline void sme_suspend_exit(void) { }
400
401static inline size_t sme_state_size(struct task_struct const *task)
402{
403	return 0;
404}
405
406#endif /* ! CONFIG_ARM64_SME */
407
408/* For use by EFI runtime services calls only */
409extern void __efi_fpsimd_begin(void);
410extern void __efi_fpsimd_end(void);
411
412#endif
413
414#endif
415