1// SPDX-License-Identifier: GPL-2.0
2/* Test selecting other page sizes for mmap/shmget.
3
4   Before running this huge pages for each huge page size must have been
5   reserved.
6   For large pages beyond MAX_ORDER (like 1GB on x86) boot options must be used.
7   Also shmmax must be increased.
8   And you need to run as root to work around some weird permissions in shm.
9   And nothing using huge pages should run in parallel.
10   When the program aborts you may need to clean up the shm segments with
11   ipcrm -m by hand, like this
12   sudo ipcs | awk '$1 == "0x00000000" {print $2}' | xargs -n1 sudo ipcrm -m
13   (warning this will remove all if someone else uses them) */
14
15#define _GNU_SOURCE 1
16#include <sys/mman.h>
17#include <stdlib.h>
18#include <stdio.h>
19#include <sys/ipc.h>
20#include <sys/shm.h>
21#include <sys/stat.h>
22#include <glob.h>
23#include <assert.h>
24#include <unistd.h>
25#include <stdarg.h>
26#include <string.h>
27#include "vm_util.h"
28
29#define err(x) perror(x), exit(1)
30
31#define MAP_HUGE_2MB    (21 << MAP_HUGE_SHIFT)
32#define MAP_HUGE_1GB    (30 << MAP_HUGE_SHIFT)
33#define MAP_HUGE_SHIFT  26
34#define MAP_HUGE_MASK   0x3f
35#if !defined(MAP_HUGETLB)
36#define MAP_HUGETLB	0x40000
37#endif
38
39#define SHM_HUGETLB     04000   /* segment will use huge TLB pages */
40#define SHM_HUGE_SHIFT  26
41#define SHM_HUGE_MASK   0x3f
42#define SHM_HUGE_2MB    (21 << SHM_HUGE_SHIFT)
43#define SHM_HUGE_1GB    (30 << SHM_HUGE_SHIFT)
44
45#define NUM_PAGESIZES   5
46
47#define NUM_PAGES 4
48
49#define Dprintf(fmt...) // printf(fmt)
50
51unsigned long page_sizes[NUM_PAGESIZES];
52int num_page_sizes;
53
54int ilog2(unsigned long v)
55{
56	int l = 0;
57	while ((1UL << l) < v)
58		l++;
59	return l;
60}
61
62void find_pagesizes(void)
63{
64	glob_t g;
65	int i;
66	glob("/sys/kernel/mm/hugepages/hugepages-*kB", 0, NULL, &g);
67	assert(g.gl_pathc <= NUM_PAGESIZES);
68	for (i = 0; i < g.gl_pathc; i++) {
69		sscanf(g.gl_pathv[i], "/sys/kernel/mm/hugepages/hugepages-%lukB",
70				&page_sizes[i]);
71		page_sizes[i] <<= 10;
72		printf("Found %luMB\n", page_sizes[i] >> 20);
73	}
74	num_page_sizes = g.gl_pathc;
75	globfree(&g);
76}
77
78void show(unsigned long ps)
79{
80	char buf[100];
81	if (ps == getpagesize())
82		return;
83	printf("%luMB: ", ps >> 20);
84	fflush(stdout);
85	snprintf(buf, sizeof buf,
86		"cat /sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages",
87		ps >> 10);
88	system(buf);
89}
90
91unsigned long read_sysfs(int warn, char *fmt, ...)
92{
93	char *line = NULL;
94	size_t linelen = 0;
95	char buf[100];
96	FILE *f;
97	va_list ap;
98	unsigned long val = 0;
99
100	va_start(ap, fmt);
101	vsnprintf(buf, sizeof buf, fmt, ap);
102	va_end(ap);
103
104	f = fopen(buf, "r");
105	if (!f) {
106		if (warn)
107			printf("missing %s\n", buf);
108		return 0;
109	}
110	if (getline(&line, &linelen, f) > 0) {
111		sscanf(line, "%lu", &val);
112	}
113	fclose(f);
114	free(line);
115	return val;
116}
117
118unsigned long read_free(unsigned long ps)
119{
120	return read_sysfs(ps != getpagesize(),
121			"/sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages",
122			ps >> 10);
123}
124
125void test_mmap(unsigned long size, unsigned flags)
126{
127	char *map;
128	unsigned long before, after;
129	int err;
130
131	before = read_free(size);
132	map = mmap(NULL, size*NUM_PAGES, PROT_READ|PROT_WRITE,
133			MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|flags, -1, 0);
134
135	if (map == (char *)-1) err("mmap");
136	memset(map, 0xff, size*NUM_PAGES);
137	after = read_free(size);
138	Dprintf("before %lu after %lu diff %ld size %lu\n",
139		before, after, before - after, size);
140	assert(size == getpagesize() || (before - after) == NUM_PAGES);
141	show(size);
142	err = munmap(map, size * NUM_PAGES);
143	assert(!err);
144}
145
146void test_shmget(unsigned long size, unsigned flags)
147{
148	int id;
149	unsigned long before, after;
150	int err;
151
152	before = read_free(size);
153	id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags);
154	if (id < 0) err("shmget");
155
156	struct shm_info i;
157	if (shmctl(id, SHM_INFO, (void *)&i) < 0) err("shmctl");
158	Dprintf("alloc %lu res %lu\n", i.shm_tot, i.shm_rss);
159
160
161	Dprintf("id %d\n", id);
162	char *map = shmat(id, NULL, 0600);
163	if (map == (char*)-1) err("shmat");
164
165	shmctl(id, IPC_RMID, NULL);
166
167	memset(map, 0xff, size*NUM_PAGES);
168	after = read_free(size);
169
170	Dprintf("before %lu after %lu diff %ld size %lu\n",
171		before, after, before - after, size);
172	assert(size == getpagesize() || (before - after) == NUM_PAGES);
173	show(size);
174	err = shmdt(map);
175	assert(!err);
176}
177
178void sanity_checks(void)
179{
180	int i;
181	unsigned long largest = getpagesize();
182
183	for (i = 0; i < num_page_sizes; i++) {
184		if (page_sizes[i] > largest)
185			largest = page_sizes[i];
186
187		if (read_free(page_sizes[i]) < NUM_PAGES) {
188			printf("Not enough huge pages for page size %lu MB, need %u\n",
189				page_sizes[i] >> 20,
190				NUM_PAGES);
191			exit(0);
192		}
193	}
194
195	if (read_sysfs(0, "/proc/sys/kernel/shmmax") < NUM_PAGES * largest) {
196		printf("Please do echo %lu > /proc/sys/kernel/shmmax", largest * NUM_PAGES);
197		exit(0);
198	}
199
200#if defined(__x86_64__)
201	if (largest != 1U<<30) {
202		printf("No GB pages available on x86-64\n"
203		       "Please boot with hugepagesz=1G hugepages=%d\n", NUM_PAGES);
204		exit(0);
205	}
206#endif
207}
208
209int main(void)
210{
211	int i;
212	unsigned default_hps = default_huge_page_size();
213
214	find_pagesizes();
215
216	sanity_checks();
217
218	for (i = 0; i < num_page_sizes; i++) {
219		unsigned long ps = page_sizes[i];
220		int arg = ilog2(ps) << MAP_HUGE_SHIFT;
221		printf("Testing %luMB mmap with shift %x\n", ps >> 20, arg);
222		test_mmap(ps, MAP_HUGETLB | arg);
223	}
224	printf("Testing default huge mmap\n");
225	test_mmap(default_hps, MAP_HUGETLB);
226
227	puts("Testing non-huge shmget");
228	test_shmget(getpagesize(), 0);
229
230	for (i = 0; i < num_page_sizes; i++) {
231		unsigned long ps = page_sizes[i];
232		int arg = ilog2(ps) << SHM_HUGE_SHIFT;
233		printf("Testing %luMB shmget with shift %x\n", ps >> 20, arg);
234		test_shmget(ps, SHM_HUGETLB | arg);
235	}
236	puts("default huge shmget");
237	test_shmget(default_hps, SHM_HUGETLB);
238
239	return 0;
240}
241