1// SPDX-License-Identifier: GPL-2.0
2/* Test selecting other page sizes for mmap/shmget.
3
4   Before running this huge pages for each huge page size must have been
5   reserved.
6   For large pages beyond MAX_ORDER (like 1GB on x86) boot options must be used.
7   Also shmmax must be increased.
8   And you need to run as root to work around some weird permissions in shm.
9   And nothing using huge pages should run in parallel.
10   When the program aborts you may need to clean up the shm segments with
11   ipcrm -m by hand, like this
12   sudo ipcs | awk '$1 == "0x00000000" {print $2}' | xargs -n1 sudo ipcrm -m
13   (warning this will remove all if someone else uses them) */
14
15#define _GNU_SOURCE 1
16#include <sys/mman.h>
17#include <stdlib.h>
18#include <stdio.h>
19#include <sys/ipc.h>
20#include <sys/shm.h>
21#include <sys/stat.h>
22#include <glob.h>
23#include <assert.h>
24#include <unistd.h>
25#include <stdarg.h>
26#include <string.h>
27
28#define err(x) perror(x), exit(1)
29
30#define MAP_HUGE_2MB    (21 << MAP_HUGE_SHIFT)
31#define MAP_HUGE_1GB    (30 << MAP_HUGE_SHIFT)
32#define MAP_HUGE_SHIFT  26
33#define MAP_HUGE_MASK   0x3f
34#if !defined(MAP_HUGETLB)
35#define MAP_HUGETLB	0x40000
36#endif
37
38#define SHM_HUGETLB     04000   /* segment will use huge TLB pages */
39#define SHM_HUGE_SHIFT  26
40#define SHM_HUGE_MASK   0x3f
41#define SHM_HUGE_2MB    (21 << SHM_HUGE_SHIFT)
42#define SHM_HUGE_1GB    (30 << SHM_HUGE_SHIFT)
43
44#define NUM_PAGESIZES   5
45
46#define NUM_PAGES 4
47
48#define Dprintf(fmt...) // printf(fmt)
49
50unsigned long page_sizes[NUM_PAGESIZES];
51int num_page_sizes;
52
53int ilog2(unsigned long v)
54{
55	int l = 0;
56	while ((1UL << l) < v)
57		l++;
58	return l;
59}
60
61void find_pagesizes(void)
62{
63	glob_t g;
64	int i;
65	glob("/sys/kernel/mm/hugepages/hugepages-*kB", 0, NULL, &g);
66	assert(g.gl_pathc <= NUM_PAGESIZES);
67	for (i = 0; i < g.gl_pathc; i++) {
68		sscanf(g.gl_pathv[i], "/sys/kernel/mm/hugepages/hugepages-%lukB",
69				&page_sizes[i]);
70		page_sizes[i] <<= 10;
71		printf("Found %luMB\n", page_sizes[i] >> 20);
72	}
73	num_page_sizes = g.gl_pathc;
74	globfree(&g);
75}
76
77unsigned long default_huge_page_size(void)
78{
79	unsigned long hps = 0;
80	char *line = NULL;
81	size_t linelen = 0;
82	FILE *f = fopen("/proc/meminfo", "r");
83	if (!f)
84		return 0;
85	while (getline(&line, &linelen, f) > 0) {
86		if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
87			hps <<= 10;
88			break;
89		}
90	}
91	free(line);
92	return hps;
93}
94
95void show(unsigned long ps)
96{
97	char buf[100];
98	if (ps == getpagesize())
99		return;
100	printf("%luMB: ", ps >> 20);
101	fflush(stdout);
102	snprintf(buf, sizeof buf,
103		"cat /sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages",
104		ps >> 10);
105	system(buf);
106}
107
108unsigned long read_sysfs(int warn, char *fmt, ...)
109{
110	char *line = NULL;
111	size_t linelen = 0;
112	char buf[100];
113	FILE *f;
114	va_list ap;
115	unsigned long val = 0;
116
117	va_start(ap, fmt);
118	vsnprintf(buf, sizeof buf, fmt, ap);
119	va_end(ap);
120
121	f = fopen(buf, "r");
122	if (!f) {
123		if (warn)
124			printf("missing %s\n", buf);
125		return 0;
126	}
127	if (getline(&line, &linelen, f) > 0) {
128		sscanf(line, "%lu", &val);
129	}
130	fclose(f);
131	free(line);
132	return val;
133}
134
135unsigned long read_free(unsigned long ps)
136{
137	return read_sysfs(ps != getpagesize(),
138			"/sys/kernel/mm/hugepages/hugepages-%lukB/free_hugepages",
139			ps >> 10);
140}
141
142void test_mmap(unsigned long size, unsigned flags)
143{
144	char *map;
145	unsigned long before, after;
146	int err;
147
148	before = read_free(size);
149	map = mmap(NULL, size*NUM_PAGES, PROT_READ|PROT_WRITE,
150			MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB|flags, -1, 0);
151
152	if (map == (char *)-1) err("mmap");
153	memset(map, 0xff, size*NUM_PAGES);
154	after = read_free(size);
155	Dprintf("before %lu after %lu diff %ld size %lu\n",
156		before, after, before - after, size);
157	assert(size == getpagesize() || (before - after) == NUM_PAGES);
158	show(size);
159	err = munmap(map, size);
160	assert(!err);
161}
162
163void test_shmget(unsigned long size, unsigned flags)
164{
165	int id;
166	unsigned long before, after;
167	int err;
168
169	before = read_free(size);
170	id = shmget(IPC_PRIVATE, size * NUM_PAGES, IPC_CREAT|0600|flags);
171	if (id < 0) err("shmget");
172
173	struct shm_info i;
174	if (shmctl(id, SHM_INFO, (void *)&i) < 0) err("shmctl");
175	Dprintf("alloc %lu res %lu\n", i.shm_tot, i.shm_rss);
176
177
178	Dprintf("id %d\n", id);
179	char *map = shmat(id, NULL, 0600);
180	if (map == (char*)-1) err("shmat");
181
182	shmctl(id, IPC_RMID, NULL);
183
184	memset(map, 0xff, size*NUM_PAGES);
185	after = read_free(size);
186
187	Dprintf("before %lu after %lu diff %ld size %lu\n",
188		before, after, before - after, size);
189	assert(size == getpagesize() || (before - after) == NUM_PAGES);
190	show(size);
191	err = shmdt(map);
192	assert(!err);
193}
194
195void sanity_checks(void)
196{
197	int i;
198	unsigned long largest = getpagesize();
199
200	for (i = 0; i < num_page_sizes; i++) {
201		if (page_sizes[i] > largest)
202			largest = page_sizes[i];
203
204		if (read_free(page_sizes[i]) < NUM_PAGES) {
205			printf("Not enough huge pages for page size %lu MB, need %u\n",
206				page_sizes[i] >> 20,
207				NUM_PAGES);
208			exit(0);
209		}
210	}
211
212	if (read_sysfs(0, "/proc/sys/kernel/shmmax") < NUM_PAGES * largest) {
213		printf("Please do echo %lu > /proc/sys/kernel/shmmax", largest * NUM_PAGES);
214		exit(0);
215	}
216
217#if defined(__x86_64__)
218	if (largest != 1U<<30) {
219		printf("No GB pages available on x86-64\n"
220		       "Please boot with hugepagesz=1G hugepages=%d\n", NUM_PAGES);
221		exit(0);
222	}
223#endif
224}
225
226int main(void)
227{
228	int i;
229	unsigned default_hps = default_huge_page_size();
230
231	find_pagesizes();
232
233	sanity_checks();
234
235	for (i = 0; i < num_page_sizes; i++) {
236		unsigned long ps = page_sizes[i];
237		int arg = ilog2(ps) << MAP_HUGE_SHIFT;
238		printf("Testing %luMB mmap with shift %x\n", ps >> 20, arg);
239		test_mmap(ps, MAP_HUGETLB | arg);
240	}
241	printf("Testing default huge mmap\n");
242	test_mmap(default_hps, SHM_HUGETLB);
243
244	puts("Testing non-huge shmget");
245	test_shmget(getpagesize(), 0);
246
247	for (i = 0; i < num_page_sizes; i++) {
248		unsigned long ps = page_sizes[i];
249		int arg = ilog2(ps) << SHM_HUGE_SHIFT;
250		printf("Testing %luMB shmget with shift %x\n", ps >> 20, arg);
251		test_shmget(ps, SHM_HUGETLB | arg);
252	}
253	puts("default huge shmget");
254	test_shmget(default_hps, SHM_HUGETLB);
255
256	return 0;
257}
258