1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Author:  Xiang Gao <gaoxiang@loongson.cn>
4 *          Huacai Chen <chenhuacai@loongson.cn>
5 *
6 * Copyright (C) 2020-2021 Loongson Technology Corporation Limited
7 */
8#include <linux/init.h>
9#include <linux/kernel.h>
10#include <linux/mm.h>
11#include <linux/mmzone.h>
12#include <linux/export.h>
13#include <linux/nodemask.h>
14#include <linux/swap.h>
15#include <linux/memblock.h>
16#include <linux/pfn.h>
17#include <linux/acpi.h>
18#include <linux/efi.h>
19#include <linux/pci.h>
20#include <asm/page.h>
21#include <asm/pgalloc.h>
22#include <asm/sections.h>
23#include <linux/irq.h>
24#include <asm/bootinfo.h>
25#include <asm/time.h>
26#include <loongson.h>
27#include <asm/numa.h>
28
29int numa_off;
30struct pglist_data *node_data[MAX_NUMNODES];
31unsigned char node_distances[MAX_NUMNODES][MAX_NUMNODES];
32
33EXPORT_SYMBOL(node_data);
34EXPORT_SYMBOL(node_distances);
35
36static struct numa_meminfo numa_meminfo;
37cpumask_t cpus_on_node[MAX_NUMNODES];
38cpumask_t phys_cpus_on_node[MAX_NUMNODES];
39EXPORT_SYMBOL(cpus_on_node);
40
41/*
42 * apicid, cpu, node mappings
43 */
44s16 __cpuid_to_node[CONFIG_NR_CPUS] = {
45	[0 ... CONFIG_NR_CPUS - 1] = NUMA_NO_NODE
46};
47EXPORT_SYMBOL(__cpuid_to_node);
48
49nodemask_t numa_nodes_parsed __initdata;
50
51#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
52unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
53EXPORT_SYMBOL(__per_cpu_offset);
54
55static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
56{
57	if (early_cpu_to_node(from) == early_cpu_to_node(to))
58		return LOCAL_DISTANCE;
59	else
60		return REMOTE_DISTANCE;
61}
62
63static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size,
64				       size_t align)
65{
66	return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS),
67				      MEMBLOCK_ALLOC_ACCESSIBLE, early_cpu_to_node(cpu));
68}
69
70static void __init pcpu_fc_free(void *ptr, size_t size)
71{
72	memblock_free_early(__pa(ptr), size);
73}
74
75static void __init pcpu_populate_pte(unsigned long addr)
76{
77	populate_kernel_pte(addr);
78}
79
80void __init setup_per_cpu_areas(void)
81{
82	unsigned long delta;
83	unsigned int cpu;
84	int rc = -EINVAL;
85
86	if (pcpu_chosen_fc == PCPU_FC_AUTO) {
87		if (nr_node_ids >= 8)
88			pcpu_chosen_fc = PCPU_FC_PAGE;
89		else
90			pcpu_chosen_fc = PCPU_FC_EMBED;
91	}
92
93	/*
94	 * Always reserve area for module percpu variables.  That's
95	 * what the legacy allocator did.
96	 */
97	if (pcpu_chosen_fc != PCPU_FC_PAGE) {
98		rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
99					    PERCPU_DYNAMIC_RESERVE, PMD_SIZE,
100					    pcpu_cpu_distance,
101					    pcpu_fc_alloc, pcpu_fc_free);
102		if (rc < 0)
103			pr_warn("%s allocator failed (%d), falling back to page size\n",
104				pcpu_fc_names[pcpu_chosen_fc], rc);
105	}
106	if (rc < 0)
107		rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,
108					   pcpu_fc_alloc, pcpu_fc_free,
109					   pcpu_populate_pte);
110	if (rc < 0)
111		panic("cannot initialize percpu area (err=%d)", rc);
112
113	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
114	for_each_possible_cpu(cpu)
115		__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
116}
117#endif
118
119/*
120 * Get nodeid by logical cpu number.
121 * __cpuid_to_node maps phyical cpu id to node, so we
122 * should use cpu_logical_map(cpu) to index it.
123 *
124 * This routine is only used in early phase during
125 * booting, after setup_per_cpu_areas calling and numa_node
126 * initialization, cpu_to_node will be used instead.
127 * */
128int early_cpu_to_node(int cpu)
129{
130	int physid = cpu_logical_map(cpu);
131
132	if (physid < 0)
133		return NUMA_NO_NODE;
134
135	return __cpuid_to_node[physid];
136}
137
138void __init early_numa_add_cpu(int cpuid, s16 node)
139{
140	int cpu = __cpu_number_map[cpuid];
141
142	if (cpu < 0)
143		return;
144
145	cpumask_set_cpu(cpu, &cpus_on_node[node]);
146	cpumask_set_cpu(cpuid, &phys_cpus_on_node[node]);
147}
148
149void numa_add_cpu(unsigned int cpu)
150{
151	int nid = cpu_to_node(cpu);
152	cpumask_set_cpu(cpu, &cpus_on_node[nid]);
153}
154
155void numa_remove_cpu(unsigned int cpu)
156{
157	int nid = cpu_to_node(cpu);
158	cpumask_clear_cpu(cpu, &cpus_on_node[nid]);
159}
160
161static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
162				     struct numa_meminfo *mi)
163{
164	/* ignore zero length blks */
165	if (start == end)
166		return 0;
167
168	/* whine about and ignore invalid blks */
169	if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
170		pr_warn("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
171			   nid, start, end - 1);
172		return 0;
173	}
174
175	if (mi->nr_blks >= NR_NODE_MEMBLKS) {
176		pr_err("NUMA: too many memblk ranges\n");
177		return -EINVAL;
178	}
179
180	mi->blk[mi->nr_blks].start = PFN_ALIGN(start);
181	mi->blk[mi->nr_blks].end = PFN_ALIGN(end - PAGE_SIZE + 1);
182	mi->blk[mi->nr_blks].nid = nid;
183	mi->nr_blks++;
184	return 0;
185}
186
187/**
188 * numa_add_memblk - Add one numa_memblk to numa_meminfo
189 * @nid: NUMA node ID of the new memblk
190 * @start: Start address of the new memblk
191 * @end: End address of the new memblk
192 *
193 * Add a new memblk to the default numa_meminfo.
194 *
195 * RETURNS:
196 * 0 on success, -errno on failure.
197 */
198int __init numa_add_memblk(int nid, u64 start, u64 end)
199{
200	return numa_add_memblk_to(nid, start, end, &numa_meminfo);
201}
202
203static void __init alloc_node_data(int nid)
204{
205	void *nd;
206	unsigned long nd_pa;
207	size_t nd_sz = roundup(sizeof(pg_data_t), PAGE_SIZE);
208
209	nd_pa = memblock_phys_alloc_try_nid(nd_sz, SMP_CACHE_BYTES, nid);
210	if (!nd_pa) {
211		pr_err("Cannot find %zu Byte for node_data (initial node: %d)\n", nd_sz, nid);
212		return;
213	}
214
215	nd = __va(nd_pa);
216
217	node_data[nid] = nd;
218	memset(nd, 0, sizeof(pg_data_t));
219}
220
221static void __init node_mem_init(unsigned int node)
222{
223	unsigned long start_pfn, end_pfn;
224	unsigned long node_addrspace_offset;
225
226	node_addrspace_offset = nid_to_addrbase(node);
227	pr_info("Node%d's addrspace_offset is 0x%lx\n",
228			node, node_addrspace_offset);
229
230	get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
231	pr_info("Node%d: start_pfn=0x%lx, end_pfn=0x%lx\n",
232		node, start_pfn, end_pfn);
233
234	alloc_node_data(node);
235}
236
237#ifdef CONFIG_ACPI_NUMA
238
239/*
240 * Sanity check to catch more bad NUMA configurations (they are amazingly
241 * common).  Make sure the nodes cover all memory.
242 */
243static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
244{
245	u64 numaram, biosram;
246	int i;
247
248	numaram = 0;
249	for (i = 0; i < mi->nr_blks; i++) {
250		u64 s = mi->blk[i].start >> PAGE_SHIFT;
251		u64 e = mi->blk[i].end >> PAGE_SHIFT;
252		numaram += e - s;
253		numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
254		if ((s64)numaram < 0)
255			numaram = 0;
256	}
257	max_pfn = max_low_pfn;
258	biosram = max_pfn - absent_pages_in_range(0, max_pfn);
259
260	BUG_ON((s64)(biosram - numaram) >= (1 << (20 - PAGE_SHIFT)));
261	return true;
262}
263
264static void __init add_node_intersection(u32 node, u64 start, u64 size, u32 type)
265{
266	static unsigned long num_physpages;
267
268	num_physpages += (size >> PAGE_SHIFT);
269	pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
270		node, type, start, size);
271	pr_info("       start_pfn:0x%llx, end_pfn:0x%llx, num_physpages:0x%lx\n",
272		start >> PAGE_SHIFT, (start + size) >> PAGE_SHIFT, num_physpages);
273	memblock_set_node(start, size, &memblock.memory, node);
274}
275
276/*
277 * add_numamem_region
278 *
279 * Add a uasable memory region described by BIOS. The
280 * routine gets each intersection between BIOS's region
281 * and node's region, and adds them into node's memblock
282 * pool.
283 *
284 */
285static void __init add_numamem_region(u64 start, u64 end, u32 type)
286{
287	u32 i;
288	u64 ofs = start;
289
290	if (start >= end) {
291		pr_debug("Invalid region: %016llx-%016llx\n", start, end);
292		return;
293	}
294
295	for (i = 0; i < numa_meminfo.nr_blks; i++) {
296		struct numa_memblk *mb = &numa_meminfo.blk[i];
297
298		if (ofs > mb->end)
299			continue;
300
301		if (end > mb->end) {
302			add_node_intersection(mb->nid, ofs, mb->end - ofs, type);
303			ofs = mb->end;
304		} else {
305			add_node_intersection(mb->nid, ofs, end - ofs, type);
306			break;
307		}
308	}
309}
310
311static void __init init_node_memblock(void)
312{
313	u32 mem_type;
314	u64 mem_end, mem_start, mem_size;
315	efi_memory_desc_t *md;
316
317	/* Parse memory information and activate */
318	for_each_efi_memory_desc(md) {
319		mem_type = md->type;
320		mem_start = md->phys_addr;
321		mem_size = md->num_pages << EFI_PAGE_SHIFT;
322		mem_end = mem_start + mem_size;
323
324		switch (mem_type) {
325		case EFI_LOADER_CODE:
326		case EFI_LOADER_DATA:
327		case EFI_BOOT_SERVICES_CODE:
328		case EFI_BOOT_SERVICES_DATA:
329		case EFI_PERSISTENT_MEMORY:
330		case EFI_CONVENTIONAL_MEMORY:
331			add_numamem_region(mem_start, mem_end, mem_type);
332			break;
333		case EFI_PAL_CODE:
334		case EFI_UNUSABLE_MEMORY:
335		case EFI_ACPI_RECLAIM_MEMORY:
336			add_numamem_region(mem_start, mem_end, mem_type);
337			fallthrough;
338		case EFI_RESERVED_TYPE:
339		case EFI_RUNTIME_SERVICES_CODE:
340		case EFI_RUNTIME_SERVICES_DATA:
341		case EFI_MEMORY_MAPPED_IO:
342		case EFI_MEMORY_MAPPED_IO_PORT_SPACE:
343			pr_info("Resvd: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx Bytes\n",
344					mem_type, mem_start, mem_size);
345			break;
346		}
347	}
348}
349
350static void __init numa_default_distance(void)
351{
352	int row, col;
353
354	for (row = 0; row < MAX_NUMNODES; row++)
355		for (col = 0; col < MAX_NUMNODES; col++) {
356			if (col == row)
357				node_distances[row][col] = LOCAL_DISTANCE;
358			else
359				/* We assume that one node per package here!
360				 *
361				 * A SLIT should be used for multiple nodes
362				 * per package to override default setting.
363				 */
364				node_distances[row][col] = REMOTE_DISTANCE;
365	}
366}
367
368int __init init_numa_memory(void)
369{
370	int i;
371	int ret;
372	int node;
373
374	for (i = 0; i < NR_CPUS; i++)
375		set_cpuid_to_node(i, NUMA_NO_NODE);
376
377	numa_default_distance();
378	nodes_clear(numa_nodes_parsed);
379	nodes_clear(node_possible_map);
380	nodes_clear(node_online_map);
381	memset(&numa_meminfo, 0, sizeof(numa_meminfo));
382
383	/* Parse SRAT and SLIT if provided by firmware. */
384	ret = acpi_numa_init();
385	if (ret < 0)
386		return ret;
387
388	node_possible_map = numa_nodes_parsed;
389	if (WARN_ON(nodes_empty(node_possible_map)))
390		return -EINVAL;
391
392	init_node_memblock();
393	if (numa_meminfo_cover_memory(&numa_meminfo) == false)
394		return -EINVAL;
395
396	for_each_node_mask(node, node_possible_map) {
397		node_mem_init(node);
398		node_set_online(node);
399	}
400	max_low_pfn = PHYS_PFN(memblock_end_of_DRAM());
401
402	setup_nr_node_ids();
403	loongson_sysconf.nr_nodes = nr_node_ids;
404	loongson_sysconf.cores_per_node = cpumask_weight(&phys_cpus_on_node[0]);
405
406	return 0;
407}
408
409#endif
410
411void __init paging_init(void)
412{
413	unsigned int node;
414	unsigned long zones_size[MAX_NR_ZONES] = {0, };
415
416	for_each_online_node(node) {
417		unsigned long start_pfn, end_pfn;
418
419		get_pfn_range_for_nid(node, &start_pfn, &end_pfn);
420
421		if (end_pfn > max_low_pfn)
422			max_low_pfn = end_pfn;
423	}
424#ifdef CONFIG_ZONE_DMA32
425	zones_size[ZONE_DMA32] = MAX_DMA32_PFN;
426#endif
427	zones_size[ZONE_NORMAL] = max_low_pfn;
428	free_area_init(zones_size);
429}
430
431void __init mem_init(void)
432{
433	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
434	memblock_free_all();
435	mem_init_print_info(NULL);
436}
437
438int pcibus_to_node(struct pci_bus *bus)
439{
440	return dev_to_node(&bus->dev);
441}
442EXPORT_SYMBOL(pcibus_to_node);
443