1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * PowerPC Memory Protection Keys management
4 *
5 * Copyright 2017, Ram Pai, IBM Corporation.
6 */
7
8#include <asm/mman.h>
9#include <asm/mmu_context.h>
10#include <asm/mmu.h>
11#include <asm/setup.h>
12#include <asm/smp.h>
13#include <asm/firmware.h>
14
15#include <linux/pkeys.h>
16#include <linux/of_fdt.h>
17
18
19int  num_pkey;		/* Max number of pkeys supported */
20/*
21 *  Keys marked in the reservation list cannot be allocated by  userspace
22 */
23u32 reserved_allocation_mask __ro_after_init;
24
25/* Bits set for the initially allocated keys */
26static u32 initial_allocation_mask __ro_after_init;
27
28/*
29 * Even if we allocate keys with sys_pkey_alloc(), we need to make sure
30 * other thread still find the access denied using the same keys.
31 */
32u64 default_amr __ro_after_init  = ~0x0UL;
33u64 default_iamr __ro_after_init = 0x5555555555555555UL;
34u64 default_uamor __ro_after_init;
35EXPORT_SYMBOL(default_amr);
36/*
37 * Key used to implement PROT_EXEC mmap. Denies READ/WRITE
38 * We pick key 2 because 0 is special key and 1 is reserved as per ISA.
39 */
40static int execute_only_key = 2;
41static bool pkey_execute_disable_supported;
42
43
44#define AMR_BITS_PER_PKEY 2
45#define AMR_RD_BIT 0x1UL
46#define AMR_WR_BIT 0x2UL
47#define IAMR_EX_BIT 0x1UL
48#define PKEY_REG_BITS (sizeof(u64) * 8)
49#define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
50
51static int __init dt_scan_storage_keys(unsigned long node,
52				       const char *uname, int depth,
53				       void *data)
54{
55	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
56	const __be32 *prop;
57	int *pkeys_total = (int *) data;
58
59	/* We are scanning "cpu" nodes only */
60	if (type == NULL || strcmp(type, "cpu") != 0)
61		return 0;
62
63	prop = of_get_flat_dt_prop(node, "ibm,processor-storage-keys", NULL);
64	if (!prop)
65		return 0;
66	*pkeys_total = be32_to_cpu(prop[0]);
67	return 1;
68}
69
70static int __init scan_pkey_feature(void)
71{
72	int ret;
73	int pkeys_total = 0;
74
75	/*
76	 * Pkey is not supported with Radix translation.
77	 */
78	if (early_radix_enabled())
79		return 0;
80
81	ret = of_scan_flat_dt(dt_scan_storage_keys, &pkeys_total);
82	if (ret == 0) {
83		/*
84		 * Let's assume 32 pkeys on P8/P9 bare metal, if its not defined by device
85		 * tree. We make this exception since some version of skiboot forgot to
86		 * expose this property on power8/9.
87		 */
88		if (!firmware_has_feature(FW_FEATURE_LPAR)) {
89			unsigned long pvr = mfspr(SPRN_PVR);
90
91			if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E ||
92			    PVR_VER(pvr) == PVR_POWER8NVL || PVR_VER(pvr) == PVR_POWER9)
93				pkeys_total = 32;
94		}
95	}
96
97#ifdef CONFIG_PPC_MEM_KEYS
98	/*
99	 * Adjust the upper limit, based on the number of bits supported by
100	 * arch-neutral code.
101	 */
102	pkeys_total = min_t(int, pkeys_total,
103			    ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + 1));
104#endif
105	return pkeys_total;
106}
107
108void __init pkey_early_init_devtree(void)
109{
110	int pkeys_total, i;
111
112#ifdef CONFIG_PPC_MEM_KEYS
113	/*
114	 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
115	 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
116	 * Ensure that the bits a distinct.
117	 */
118	BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
119		     (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
120
121	/*
122	 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
123	 * in the vmaflag. Make sure that is really the case.
124	 */
125	BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
126		     __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
127				!= (sizeof(u64) * BITS_PER_BYTE));
128#endif
129	/*
130	 * Only P7 and above supports SPRN_AMR update with MSR[PR] = 1
131	 */
132	if (!early_cpu_has_feature(CPU_FTR_ARCH_206))
133		return;
134
135	/* scan the device tree for pkey feature */
136	pkeys_total = scan_pkey_feature();
137	if (!pkeys_total)
138		goto out;
139
140	/* Allow all keys to be modified by default */
141	default_uamor = ~0x0UL;
142
143	cur_cpu_spec->mmu_features |= MMU_FTR_PKEY;
144
145	/*
146	 * The device tree cannot be relied to indicate support for
147	 * execute_disable support. Instead we use a PVR check.
148	 */
149	if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
150		pkey_execute_disable_supported = false;
151	else
152		pkey_execute_disable_supported = true;
153
154#ifdef CONFIG_PPC_4K_PAGES
155	/*
156	 * The OS can manage only 8 pkeys due to its inability to represent them
157	 * in the Linux 4K PTE. Mark all other keys reserved.
158	 */
159	num_pkey = min(8, pkeys_total);
160#else
161	num_pkey = pkeys_total;
162#endif
163
164	if (unlikely(num_pkey <= execute_only_key) || !pkey_execute_disable_supported) {
165		/*
166		 * Insufficient number of keys to support
167		 * execute only key. Mark it unavailable.
168		 */
169		execute_only_key = -1;
170	} else {
171		/*
172		 * Mark the execute_only_pkey as not available for
173		 * user allocation via pkey_alloc.
174		 */
175		reserved_allocation_mask |= (0x1 << execute_only_key);
176
177		/*
178		 * Deny READ/WRITE for execute_only_key.
179		 * Allow execute in IAMR.
180		 */
181		default_amr  |= (0x3ul << pkeyshift(execute_only_key));
182		default_iamr &= ~(0x1ul << pkeyshift(execute_only_key));
183
184		/*
185		 * Clear the uamor bits for this key.
186		 */
187		default_uamor &= ~(0x3ul << pkeyshift(execute_only_key));
188	}
189
190	if (unlikely(num_pkey <= 3)) {
191		/*
192		 * Insufficient number of keys to support
193		 * KUAP/KUEP feature.
194		 */
195		disable_kuep = true;
196		disable_kuap = true;
197		WARN(1, "Disabling kernel user protection due to low (%d) max supported keys\n", num_pkey);
198	} else {
199		/*  handle key which is used by kernel for KAUP */
200		reserved_allocation_mask |= (0x1 << 3);
201		/*
202		 * Mark access for kup_key in default amr so that
203		 * we continue to operate with that AMR in
204		 * copy_to/from_user().
205		 */
206		default_amr   &= ~(0x3ul << pkeyshift(3));
207		default_iamr  &= ~(0x1ul << pkeyshift(3));
208		default_uamor &= ~(0x3ul << pkeyshift(3));
209	}
210
211	/*
212	 * Allow access for only key 0. And prevent any other modification.
213	 */
214	default_amr   &= ~(0x3ul << pkeyshift(0));
215	default_iamr  &= ~(0x1ul << pkeyshift(0));
216	default_uamor &= ~(0x3ul << pkeyshift(0));
217	/*
218	 * key 0 is special in that we want to consider it an allocated
219	 * key which is preallocated. We don't allow changing AMR bits
220	 * w.r.t key 0. But one can pkey_free(key0)
221	 */
222	initial_allocation_mask |= (0x1 << 0);
223
224	/*
225	 * key 1 is recommended not to be used. PowerISA(3.0) page 1015,
226	 * programming note.
227	 */
228	reserved_allocation_mask |= (0x1 << 1);
229	default_uamor &= ~(0x3ul << pkeyshift(1));
230
231	/*
232	 * Prevent the usage of OS reserved keys. Update UAMOR
233	 * for those keys. Also mark the rest of the bits in the
234	 * 32 bit mask as reserved.
235	 */
236	for (i = num_pkey; i < 32 ; i++) {
237		reserved_allocation_mask |= (0x1 << i);
238		default_uamor &= ~(0x3ul << pkeyshift(i));
239	}
240	/*
241	 * Prevent the allocation of reserved keys too.
242	 */
243	initial_allocation_mask |= reserved_allocation_mask;
244
245	pr_info("Enabling pkeys with max key count %d\n", num_pkey);
246out:
247	/*
248	 * Setup uamor on boot cpu
249	 */
250	mtspr(SPRN_UAMOR, default_uamor);
251
252	return;
253}
254
255#ifdef CONFIG_PPC_KUEP
256void setup_kuep(bool disabled)
257{
258	if (disabled)
259		return;
260	/*
261	 * On hash if PKEY feature is not enabled, disable KUAP too.
262	 */
263	if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY))
264		return;
265
266	if (smp_processor_id() == boot_cpuid) {
267		pr_info("Activating Kernel Userspace Execution Prevention\n");
268		cur_cpu_spec->mmu_features |= MMU_FTR_BOOK3S_KUEP;
269	}
270
271	/*
272	 * Radix always uses key0 of the IAMR to determine if an access is
273	 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
274	 * fetch.
275	 */
276	mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
277	isync();
278}
279#endif
280
281#ifdef CONFIG_PPC_KUAP
282void setup_kuap(bool disabled)
283{
284	if (disabled)
285		return;
286	/*
287	 * On hash if PKEY feature is not enabled, disable KUAP too.
288	 */
289	if (!early_radix_enabled() && !early_mmu_has_feature(MMU_FTR_PKEY))
290		return;
291
292	if (smp_processor_id() == boot_cpuid) {
293		pr_info("Activating Kernel Userspace Access Prevention\n");
294		cur_cpu_spec->mmu_features |= MMU_FTR_KUAP;
295	}
296
297	/*
298	 * Set the default kernel AMR values on all cpus.
299	 */
300	mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
301	isync();
302}
303#endif
304
305#ifdef CONFIG_PPC_MEM_KEYS
306void pkey_mm_init(struct mm_struct *mm)
307{
308	if (!mmu_has_feature(MMU_FTR_PKEY))
309		return;
310	mm_pkey_allocation_map(mm) = initial_allocation_mask;
311	mm->context.execute_only_pkey = execute_only_key;
312}
313
314static inline void init_amr(int pkey, u8 init_bits)
315{
316	u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
317	u64 old_amr = current_thread_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
318
319	current->thread.regs->amr = old_amr | new_amr_bits;
320}
321
322static inline void init_iamr(int pkey, u8 init_bits)
323{
324	u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
325	u64 old_iamr = current_thread_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
326
327	if (!likely(pkey_execute_disable_supported))
328		return;
329
330	current->thread.regs->iamr = old_iamr | new_iamr_bits;
331}
332
333/*
334 * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
335 * specified in @init_val.
336 */
337int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
338				unsigned long init_val)
339{
340	u64 new_amr_bits = 0x0ul;
341	u64 new_iamr_bits = 0x0ul;
342	u64 pkey_bits, uamor_pkey_bits;
343
344	/*
345	 * Check whether the key is disabled by UAMOR.
346	 */
347	pkey_bits = 0x3ul << pkeyshift(pkey);
348	uamor_pkey_bits = (default_uamor & pkey_bits);
349
350	/*
351	 * Both the bits in UAMOR corresponding to the key should be set
352	 */
353	if (uamor_pkey_bits != pkey_bits)
354		return -EINVAL;
355
356	if (init_val & PKEY_DISABLE_EXECUTE) {
357		if (!pkey_execute_disable_supported)
358			return -EINVAL;
359		new_iamr_bits |= IAMR_EX_BIT;
360	}
361	init_iamr(pkey, new_iamr_bits);
362
363	/* Set the bits we need in AMR: */
364	if (init_val & PKEY_DISABLE_ACCESS)
365		new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
366	else if (init_val & PKEY_DISABLE_WRITE)
367		new_amr_bits |= AMR_WR_BIT;
368
369	init_amr(pkey, new_amr_bits);
370	return 0;
371}
372
373int execute_only_pkey(struct mm_struct *mm)
374{
375	return mm->context.execute_only_pkey;
376}
377
378static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
379{
380	/* Do this check first since the vm_flags should be hot */
381	if ((vma->vm_flags & VM_ACCESS_FLAGS) != VM_EXEC)
382		return false;
383
384	return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
385}
386
387/*
388 * This should only be called for *plain* mprotect calls.
389 */
390int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
391				  int pkey)
392{
393	/*
394	 * If the currently associated pkey is execute-only, but the requested
395	 * protection is not execute-only, move it back to the default pkey.
396	 */
397	if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC))
398		return 0;
399
400	/*
401	 * The requested protection is execute-only. Hence let's use an
402	 * execute-only pkey.
403	 */
404	if (prot == PROT_EXEC) {
405		pkey = execute_only_pkey(vma->vm_mm);
406		if (pkey > 0)
407			return pkey;
408	}
409
410	/* Nothing to override. */
411	return vma_pkey(vma);
412}
413
414static bool pkey_access_permitted(int pkey, bool write, bool execute)
415{
416	int pkey_shift;
417	u64 amr;
418
419	pkey_shift = pkeyshift(pkey);
420	if (execute)
421		return !(current_thread_iamr() & (IAMR_EX_BIT << pkey_shift));
422
423	amr = current_thread_amr();
424	if (write)
425		return !(amr & (AMR_WR_BIT << pkey_shift));
426
427	return !(amr & (AMR_RD_BIT << pkey_shift));
428}
429
430bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
431{
432	if (!mmu_has_feature(MMU_FTR_PKEY))
433		return true;
434
435	return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
436}
437
438/*
439 * We only want to enforce protection keys on the current thread because we
440 * effectively have no access to AMR/IAMR for other threads or any way to tell
441 * which AMR/IAMR in a threaded process we could use.
442 *
443 * So do not enforce things if the VMA is not from the current mm, or if we are
444 * in a kernel thread.
445 */
446bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
447			       bool execute, bool foreign)
448{
449	if (!mmu_has_feature(MMU_FTR_PKEY))
450		return true;
451	/*
452	 * Do not enforce our key-permissions on a foreign vma.
453	 */
454	if (foreign || vma_is_foreign(vma))
455		return true;
456
457	return pkey_access_permitted(vma_pkey(vma), write, execute);
458}
459
460void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm)
461{
462	if (!mmu_has_feature(MMU_FTR_PKEY))
463		return;
464
465	/* Duplicate the oldmm pkey state in mm: */
466	mm_pkey_allocation_map(mm) = mm_pkey_allocation_map(oldmm);
467	mm->context.execute_only_pkey = oldmm->context.execute_only_pkey;
468}
469
470#endif /* CONFIG_PPC_MEM_KEYS */
471