1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kasan.h>
3#include <linux/sched/task.h>
4#include <linux/memblock.h>
5#include <linux/pgtable.h>
6#include <asm/pgalloc.h>
7#include <asm/kasan.h>
8#include <asm/mem_detect.h>
9#include <asm/processor.h>
10#include <asm/sclp.h>
11#include <asm/facility.h>
12#include <asm/sections.h>
13#include <asm/setup.h>
14#include <asm/uv.h>
15
16unsigned long kasan_vmax;
17static unsigned long segment_pos __initdata;
18static unsigned long segment_low __initdata;
19static unsigned long pgalloc_pos __initdata;
20static unsigned long pgalloc_low __initdata;
21static unsigned long pgalloc_freeable __initdata;
22static bool has_edat __initdata;
23static bool has_nx __initdata;
24
25#define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
26
27static pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
28
29static void __init kasan_early_panic(const char *reason)
30{
31	sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
32	sclp_early_printk(reason);
33	disabled_wait();
34}
35
36static void * __init kasan_early_alloc_segment(void)
37{
38	segment_pos -= _SEGMENT_SIZE;
39
40	if (segment_pos < segment_low)
41		kasan_early_panic("out of memory during initialisation\n");
42
43	return (void *)segment_pos;
44}
45
46static void * __init kasan_early_alloc_pages(unsigned int order)
47{
48	pgalloc_pos -= (PAGE_SIZE << order);
49
50	if (pgalloc_pos < pgalloc_low)
51		kasan_early_panic("out of memory during initialisation\n");
52
53	return (void *)pgalloc_pos;
54}
55
56static void * __init kasan_early_crst_alloc(unsigned long val)
57{
58	unsigned long *table;
59
60	table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
61	if (table)
62		crst_table_init(table, val);
63	return table;
64}
65
66static pte_t * __init kasan_early_pte_alloc(void)
67{
68	static void *pte_leftover;
69	pte_t *pte;
70
71	BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
72
73	if (!pte_leftover) {
74		pte_leftover = kasan_early_alloc_pages(0);
75		pte = pte_leftover + _PAGE_TABLE_SIZE;
76	} else {
77		pte = pte_leftover;
78		pte_leftover = NULL;
79	}
80	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
81	return pte;
82}
83
84enum populate_mode {
85	POPULATE_ONE2ONE,
86	POPULATE_MAP,
87	POPULATE_ZERO_SHADOW,
88	POPULATE_SHALLOW
89};
90static void __init kasan_early_vmemmap_populate(unsigned long address,
91						unsigned long end,
92						enum populate_mode mode)
93{
94	unsigned long pgt_prot_zero, pgt_prot, sgt_prot;
95	pgd_t *pg_dir;
96	p4d_t *p4_dir;
97	pud_t *pu_dir;
98	pmd_t *pm_dir;
99	pte_t *pt_dir;
100
101	pgt_prot_zero = pgprot_val(PAGE_KERNEL_RO);
102	if (!has_nx)
103		pgt_prot_zero &= ~_PAGE_NOEXEC;
104	pgt_prot = pgprot_val(PAGE_KERNEL);
105	sgt_prot = pgprot_val(SEGMENT_KERNEL);
106	if (!has_nx || mode == POPULATE_ONE2ONE) {
107		pgt_prot &= ~_PAGE_NOEXEC;
108		sgt_prot &= ~_SEGMENT_ENTRY_NOEXEC;
109	}
110
111	/*
112	 * The first 1MB of 1:1 mapping is mapped with 4KB pages
113	 */
114	while (address < end) {
115		pg_dir = pgd_offset_k(address);
116		if (pgd_none(*pg_dir)) {
117			if (mode == POPULATE_ZERO_SHADOW &&
118			    IS_ALIGNED(address, PGDIR_SIZE) &&
119			    end - address >= PGDIR_SIZE) {
120				pgd_populate(&init_mm, pg_dir,
121						kasan_early_shadow_p4d);
122				address = (address + PGDIR_SIZE) & PGDIR_MASK;
123				continue;
124			}
125			p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
126			pgd_populate(&init_mm, pg_dir, p4_dir);
127		}
128
129		if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
130		    mode == POPULATE_SHALLOW) {
131			address = (address + P4D_SIZE) & P4D_MASK;
132			continue;
133		}
134
135		p4_dir = p4d_offset(pg_dir, address);
136		if (p4d_none(*p4_dir)) {
137			if (mode == POPULATE_ZERO_SHADOW &&
138			    IS_ALIGNED(address, P4D_SIZE) &&
139			    end - address >= P4D_SIZE) {
140				p4d_populate(&init_mm, p4_dir,
141						kasan_early_shadow_pud);
142				address = (address + P4D_SIZE) & P4D_MASK;
143				continue;
144			}
145			pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
146			p4d_populate(&init_mm, p4_dir, pu_dir);
147		}
148
149		if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) &&
150		    mode == POPULATE_SHALLOW) {
151			address = (address + PUD_SIZE) & PUD_MASK;
152			continue;
153		}
154
155		pu_dir = pud_offset(p4_dir, address);
156		if (pud_none(*pu_dir)) {
157			if (mode == POPULATE_ZERO_SHADOW &&
158			    IS_ALIGNED(address, PUD_SIZE) &&
159			    end - address >= PUD_SIZE) {
160				pud_populate(&init_mm, pu_dir,
161						kasan_early_shadow_pmd);
162				address = (address + PUD_SIZE) & PUD_MASK;
163				continue;
164			}
165			pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
166			pud_populate(&init_mm, pu_dir, pm_dir);
167		}
168
169		pm_dir = pmd_offset(pu_dir, address);
170		if (pmd_none(*pm_dir)) {
171			if (IS_ALIGNED(address, PMD_SIZE) &&
172			    end - address >= PMD_SIZE) {
173				if (mode == POPULATE_ZERO_SHADOW) {
174					pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
175					address = (address + PMD_SIZE) & PMD_MASK;
176					continue;
177				} else if (has_edat && address) {
178					void *page;
179
180					if (mode == POPULATE_ONE2ONE) {
181						page = (void *)address;
182					} else {
183						page = kasan_early_alloc_segment();
184						memset(page, 0, _SEGMENT_SIZE);
185					}
186					pmd_val(*pm_dir) = __pa(page) | sgt_prot;
187					address = (address + PMD_SIZE) & PMD_MASK;
188					continue;
189				}
190			}
191			pt_dir = kasan_early_pte_alloc();
192			pmd_populate(&init_mm, pm_dir, pt_dir);
193		} else if (pmd_large(*pm_dir)) {
194			address = (address + PMD_SIZE) & PMD_MASK;
195			continue;
196		}
197
198		pt_dir = pte_offset_kernel(pm_dir, address);
199		if (pte_none(*pt_dir)) {
200			void *page;
201
202			switch (mode) {
203			case POPULATE_ONE2ONE:
204				page = (void *)address;
205				pte_val(*pt_dir) = __pa(page) | pgt_prot;
206				break;
207			case POPULATE_MAP:
208				page = kasan_early_alloc_pages(0);
209				memset(page, 0, PAGE_SIZE);
210				pte_val(*pt_dir) = __pa(page) | pgt_prot;
211				break;
212			case POPULATE_ZERO_SHADOW:
213				page = kasan_early_shadow_page;
214				pte_val(*pt_dir) = __pa(page) | pgt_prot_zero;
215				break;
216			case POPULATE_SHALLOW:
217				/* should never happen */
218				break;
219			}
220		}
221		address += PAGE_SIZE;
222	}
223}
224
225static void __init kasan_set_pgd(pgd_t *pgd, unsigned long asce_type)
226{
227	unsigned long asce_bits;
228
229	asce_bits = asce_type | _ASCE_TABLE_LENGTH;
230	S390_lowcore.kernel_asce = (__pa(pgd) & PAGE_MASK) | asce_bits;
231	S390_lowcore.user_asce = S390_lowcore.kernel_asce;
232
233	__ctl_load(S390_lowcore.kernel_asce, 1, 1);
234	__ctl_load(S390_lowcore.kernel_asce, 7, 7);
235	__ctl_load(S390_lowcore.kernel_asce, 13, 13);
236}
237
238static void __init kasan_enable_dat(void)
239{
240	psw_t psw;
241
242	psw.mask = __extract_psw();
243	psw_bits(psw).dat = 1;
244	psw_bits(psw).as = PSW_BITS_AS_HOME;
245	__load_psw_mask(psw.mask);
246}
247
248static void __init kasan_early_detect_facilities(void)
249{
250	if (test_facility(8)) {
251		has_edat = true;
252		__ctl_set_bit(0, 23);
253	}
254	if (!noexec_disabled && test_facility(130)) {
255		has_nx = true;
256		__ctl_set_bit(0, 20);
257	}
258}
259
260static bool __init has_uv_sec_stor_limit(void)
261{
262	/*
263	 * keep these conditions in line with setup_uv()
264	 */
265	if (!is_prot_virt_host())
266		return false;
267
268	if (is_prot_virt_guest())
269		return false;
270
271	if (!test_facility(158))
272		return false;
273
274	return !!uv_info.max_sec_stor_addr;
275}
276
277void __init kasan_early_init(void)
278{
279	unsigned long untracked_mem_end;
280	unsigned long shadow_alloc_size;
281	unsigned long vmax_unlimited;
282	unsigned long initrd_end;
283	unsigned long asce_type;
284	unsigned long memsize;
285	unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO);
286	pte_t pte_z;
287	pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
288	pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
289	p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
290
291	kasan_early_detect_facilities();
292	if (!has_nx)
293		pgt_prot &= ~_PAGE_NOEXEC;
294	pte_z = __pte(__pa(kasan_early_shadow_page) | pgt_prot);
295
296	memsize = get_mem_detect_end();
297	if (!memsize)
298		kasan_early_panic("cannot detect physical memory size\n");
299	/* respect mem= cmdline parameter */
300	if (memory_end_set && memsize > memory_end)
301		memsize = memory_end;
302	if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE)
303		memsize = min(memsize, OLDMEM_SIZE);
304	memsize = min(memsize, KASAN_SHADOW_START);
305
306	if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) {
307		/* 4 level paging */
308		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
309		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
310		crst_table_init((unsigned long *)early_pg_dir,
311				_REGION2_ENTRY_EMPTY);
312		untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE;
313		if (has_uv_sec_stor_limit())
314			kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr);
315		asce_type = _ASCE_TYPE_REGION2;
316	} else {
317		/* 3 level paging */
318		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE));
319		BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE));
320		crst_table_init((unsigned long *)early_pg_dir,
321				_REGION3_ENTRY_EMPTY);
322		untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE;
323		asce_type = _ASCE_TYPE_REGION3;
324	}
325
326	/* init kasan zero shadow */
327	crst_table_init((unsigned long *)kasan_early_shadow_p4d,
328				p4d_val(p4d_z));
329	crst_table_init((unsigned long *)kasan_early_shadow_pud,
330				pud_val(pud_z));
331	crst_table_init((unsigned long *)kasan_early_shadow_pmd,
332				pmd_val(pmd_z));
333	memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
334
335	shadow_alloc_size = memsize >> KASAN_SHADOW_SCALE_SHIFT;
336	pgalloc_low = round_up((unsigned long)_end, _SEGMENT_SIZE);
337	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD)) {
338		initrd_end =
339		    round_up(INITRD_START + INITRD_SIZE, _SEGMENT_SIZE);
340		pgalloc_low = max(pgalloc_low, initrd_end);
341	}
342
343	if (pgalloc_low + shadow_alloc_size > memsize)
344		kasan_early_panic("out of memory during initialisation\n");
345
346	if (has_edat) {
347		segment_pos = round_down(memsize, _SEGMENT_SIZE);
348		segment_low = segment_pos - shadow_alloc_size;
349		pgalloc_pos = segment_low;
350	} else {
351		pgalloc_pos = memsize;
352	}
353	init_mm.pgd = early_pg_dir;
354	/*
355	 * Current memory layout:
356	 * +- 0 -------------+	   +- shadow start -+
357	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
358	 * |		     |	 / |		    |
359	 * +- end of ram ----+	/  +----------------+
360	 * | ... gap ...     | /   |		    |
361	 * |		     |/    |	kasan	    |
362	 * +- shadow start --+	   |	zero	    |
363	 * | 1/8 addr space  |	   |	page	    |
364	 * +- shadow end    -+	   |	mapping	    |
365	 * | ... gap ...     |\    |  (untracked)   |
366	 * +- vmalloc area  -+ \   |		    |
367	 * | vmalloc_size    |	\  |		    |
368	 * +- modules vaddr -+	 \ +----------------+
369	 * | 2Gb	     |	  \|	  unmapped  | allocated per module
370	 * +-----------------+	   +- shadow end ---+
371	 *
372	 * Current memory layout (KASAN_VMALLOC):
373	 * +- 0 -------------+	   +- shadow start -+
374	 * | 1:1 ram mapping |	  /| 1/8 ram	    |
375	 * |		     |	 / |		    |
376	 * +- end of ram ----+	/  +----------------+
377	 * | ... gap ...     | /   |	kasan	    |
378	 * |		     |/    |	zero	    |
379	 * +- shadow start --+	   |	page	    |
380	 * | 1/8 addr space  |	   |	mapping     |
381	 * +- shadow end    -+	   |  (untracked)   |
382	 * | ... gap ...     |\    |		    |
383	 * +- vmalloc area  -+ \   +- vmalloc area -+
384	 * | vmalloc_size    |	\  |shallow populate|
385	 * +- modules vaddr -+	 \ +- modules area -+
386	 * | 2Gb	     |	  \|shallow populate|
387	 * +-----------------+	   +- shadow end ---+
388	 */
389	/* populate kasan shadow (for identity mapping and zero page mapping) */
390	kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP);
391	if (IS_ENABLED(CONFIG_MODULES))
392		untracked_mem_end = kasan_vmax - MODULES_LEN;
393	if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
394		untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN;
395		/* shallowly populate kasan shadow for vmalloc and modules */
396		kasan_early_vmemmap_populate(__sha(untracked_mem_end),
397					     __sha(kasan_vmax), POPULATE_SHALLOW);
398	}
399	/* populate kasan shadow for untracked memory */
400	kasan_early_vmemmap_populate(__sha(max_physmem_end),
401				     __sha(untracked_mem_end),
402				     POPULATE_ZERO_SHADOW);
403	kasan_early_vmemmap_populate(__sha(kasan_vmax),
404				     __sha(vmax_unlimited),
405				     POPULATE_ZERO_SHADOW);
406	/* memory allocated for identity mapping structs will be freed later */
407	pgalloc_freeable = pgalloc_pos;
408	/* populate identity mapping */
409	kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE);
410	kasan_set_pgd(early_pg_dir, asce_type);
411	kasan_enable_dat();
412	/* enable kasan */
413	init_task.kasan_depth = 0;
414	memblock_reserve(pgalloc_pos, memsize - pgalloc_pos);
415	sclp_early_printk("KernelAddressSanitizer initialized\n");
416}
417
418void __init kasan_copy_shadow(pgd_t *pg_dir)
419{
420	/*
421	 * At this point we are still running on early pages setup early_pg_dir,
422	 * while swapper_pg_dir has just been initialized with identity mapping.
423	 * Carry over shadow memory region from early_pg_dir to swapper_pg_dir.
424	 */
425
426	pgd_t *pg_dir_src;
427	pgd_t *pg_dir_dst;
428	p4d_t *p4_dir_src;
429	p4d_t *p4_dir_dst;
430	pud_t *pu_dir_src;
431	pud_t *pu_dir_dst;
432
433	pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START);
434	pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START);
435	p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START);
436	p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START);
437	if (!p4d_folded(*p4_dir_src)) {
438		/* 4 level paging */
439		memcpy(p4_dir_dst, p4_dir_src,
440		       (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t));
441		return;
442	}
443	/* 3 level paging */
444	pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START);
445	pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START);
446	memcpy(pu_dir_dst, pu_dir_src,
447	       (KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t));
448}
449
450void __init kasan_free_early_identity(void)
451{
452	memblock_free(pgalloc_pos, pgalloc_freeable - pgalloc_pos);
453}
454