1// SPDX-License-Identifier: GPL-2.0
2/*
3 * hugepage-madvise:
4 *
5 * Basic functional testing of madvise MADV_DONTNEED and MADV_REMOVE
6 * on hugetlb mappings.
7 *
8 * Before running this test, make sure the administrator has pre-allocated
9 * at least MIN_FREE_PAGES hugetlb pages and they are free.  In addition,
10 * the test takes an argument that is the path to a file in a hugetlbfs
11 * filesystem.  Therefore, a hugetlbfs filesystem must be mounted on some
12 * directory.
13 */
14
15#define _GNU_SOURCE
16#include <stdlib.h>
17#include <stdio.h>
18#include <unistd.h>
19#include <sys/mman.h>
20#include <fcntl.h>
21#include "vm_util.h"
22
23#define MIN_FREE_PAGES	20
24#define NR_HUGE_PAGES	10	/* common number of pages to map/allocate */
25
26#define validate_free_pages(exp_free)					\
27	do {								\
28		int fhp = get_free_hugepages();				\
29		if (fhp != (exp_free)) {				\
30			printf("Unexpected number of free huge "	\
31				"pages line %d\n", __LINE__);		\
32			exit(1);					\
33		}							\
34	} while (0)
35
36unsigned long huge_page_size;
37unsigned long base_page_size;
38
39unsigned long get_free_hugepages(void)
40{
41	unsigned long fhp = 0;
42	char *line = NULL;
43	size_t linelen = 0;
44	FILE *f = fopen("/proc/meminfo", "r");
45
46	if (!f)
47		return fhp;
48	while (getline(&line, &linelen, f) > 0) {
49		if (sscanf(line, "HugePages_Free:      %lu", &fhp) == 1)
50			break;
51	}
52
53	free(line);
54	fclose(f);
55	return fhp;
56}
57
58void write_fault_pages(void *addr, unsigned long nr_pages)
59{
60	unsigned long i;
61
62	for (i = 0; i < nr_pages; i++)
63		*((unsigned long *)(addr + (i * huge_page_size))) = i;
64}
65
66void read_fault_pages(void *addr, unsigned long nr_pages)
67{
68	volatile unsigned long dummy = 0;
69	unsigned long i;
70
71	for (i = 0; i < nr_pages; i++) {
72		dummy += *((unsigned long *)(addr + (i * huge_page_size)));
73
74		/* Prevent the compiler from optimizing out the entire loop: */
75		asm volatile("" : "+r" (dummy));
76	}
77}
78
79int main(int argc, char **argv)
80{
81	unsigned long free_hugepages;
82	void *addr, *addr2;
83	int fd;
84	int ret;
85
86	huge_page_size = default_huge_page_size();
87	if (!huge_page_size) {
88		printf("Unable to determine huge page size, exiting!\n");
89		exit(1);
90	}
91	base_page_size = sysconf(_SC_PAGE_SIZE);
92	if (!huge_page_size) {
93		printf("Unable to determine base page size, exiting!\n");
94		exit(1);
95	}
96
97	free_hugepages = get_free_hugepages();
98	if (free_hugepages < MIN_FREE_PAGES) {
99		printf("Not enough free huge pages to test, exiting!\n");
100		exit(1);
101	}
102
103	fd = memfd_create(argv[0], MFD_HUGETLB);
104	if (fd < 0) {
105		perror("memfd_create() failed");
106		exit(1);
107	}
108
109	/*
110	 * Test validity of MADV_DONTNEED addr and length arguments.  mmap
111	 * size is NR_HUGE_PAGES + 2.  One page at the beginning and end of
112	 * the mapping will be unmapped so we KNOW there is nothing mapped
113	 * there.
114	 */
115	addr = mmap(NULL, (NR_HUGE_PAGES + 2) * huge_page_size,
116			PROT_READ | PROT_WRITE,
117			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
118			-1, 0);
119	if (addr == MAP_FAILED) {
120		perror("mmap");
121		exit(1);
122	}
123	if (munmap(addr, huge_page_size) ||
124			munmap(addr + (NR_HUGE_PAGES + 1) * huge_page_size,
125				huge_page_size)) {
126		perror("munmap");
127		exit(1);
128	}
129	addr = addr + huge_page_size;
130
131	write_fault_pages(addr, NR_HUGE_PAGES);
132	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
133
134	/* addr before mapping should fail */
135	ret = madvise(addr - base_page_size, NR_HUGE_PAGES * huge_page_size,
136		MADV_DONTNEED);
137	if (!ret) {
138		printf("Unexpected success of madvise call with invalid addr line %d\n",
139				__LINE__);
140			exit(1);
141	}
142
143	/* addr + length after mapping should fail */
144	ret = madvise(addr, (NR_HUGE_PAGES * huge_page_size) + base_page_size,
145		MADV_DONTNEED);
146	if (!ret) {
147		printf("Unexpected success of madvise call with invalid length line %d\n",
148				__LINE__);
149			exit(1);
150	}
151
152	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
153
154	/*
155	 * Test alignment of MADV_DONTNEED addr and length arguments
156	 */
157	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
158			PROT_READ | PROT_WRITE,
159			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
160			-1, 0);
161	if (addr == MAP_FAILED) {
162		perror("mmap");
163		exit(1);
164	}
165	write_fault_pages(addr, NR_HUGE_PAGES);
166	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
167
168	/* addr is not huge page size aligned and should fail */
169	ret = madvise(addr + base_page_size,
170			NR_HUGE_PAGES * huge_page_size - base_page_size,
171			MADV_DONTNEED);
172	if (!ret) {
173		printf("Unexpected success of madvise call with unaligned start address %d\n",
174				__LINE__);
175			exit(1);
176	}
177
178	/* addr + length should be aligned down to huge page size */
179	if (madvise(addr,
180			((NR_HUGE_PAGES - 1) * huge_page_size) + base_page_size,
181			MADV_DONTNEED)) {
182		perror("madvise");
183		exit(1);
184	}
185
186	/* should free all but last page in mapping */
187	validate_free_pages(free_hugepages - 1);
188
189	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
190	validate_free_pages(free_hugepages);
191
192	/*
193	 * Test MADV_DONTNEED on anonymous private mapping
194	 */
195	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
196			PROT_READ | PROT_WRITE,
197			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
198			-1, 0);
199	if (addr == MAP_FAILED) {
200		perror("mmap");
201		exit(1);
202	}
203	write_fault_pages(addr, NR_HUGE_PAGES);
204	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
205
206	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
207		perror("madvise");
208		exit(1);
209	}
210
211	/* should free all pages in mapping */
212	validate_free_pages(free_hugepages);
213
214	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
215
216	/*
217	 * Test MADV_DONTNEED on private mapping of hugetlb file
218	 */
219	if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) {
220		perror("fallocate");
221		exit(1);
222	}
223	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
224
225	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
226			PROT_READ | PROT_WRITE,
227			MAP_PRIVATE, fd, 0);
228	if (addr == MAP_FAILED) {
229		perror("mmap");
230		exit(1);
231	}
232
233	/* read should not consume any pages */
234	read_fault_pages(addr, NR_HUGE_PAGES);
235	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
236
237	/* madvise should not free any pages */
238	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
239		perror("madvise");
240		exit(1);
241	}
242	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
243
244	/* writes should allocate private pages */
245	write_fault_pages(addr, NR_HUGE_PAGES);
246	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
247
248	/* madvise should free private pages */
249	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
250		perror("madvise");
251		exit(1);
252	}
253	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
254
255	/* writes should allocate private pages */
256	write_fault_pages(addr, NR_HUGE_PAGES);
257	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
258
259	/*
260	 * The fallocate below certainly should free the pages associated
261	 * with the file.  However, pages in the private mapping are also
262	 * freed.  This is not the 'correct' behavior, but is expected
263	 * because this is how it has worked since the initial hugetlb
264	 * implementation.
265	 */
266	if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
267					0, NR_HUGE_PAGES * huge_page_size)) {
268		perror("fallocate");
269		exit(1);
270	}
271	validate_free_pages(free_hugepages);
272
273	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
274
275	/*
276	 * Test MADV_DONTNEED on shared mapping of hugetlb file
277	 */
278	if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) {
279		perror("fallocate");
280		exit(1);
281	}
282	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
283
284	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
285			PROT_READ | PROT_WRITE,
286			MAP_SHARED, fd, 0);
287	if (addr == MAP_FAILED) {
288		perror("mmap");
289		exit(1);
290	}
291
292	/* write should not consume any pages */
293	write_fault_pages(addr, NR_HUGE_PAGES);
294	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
295
296	/* madvise should not free any pages */
297	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
298		perror("madvise");
299		exit(1);
300	}
301	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
302
303	/*
304	 * Test MADV_REMOVE on shared mapping of hugetlb file
305	 *
306	 * madvise is same as hole punch and should free all pages.
307	 */
308	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) {
309		perror("madvise");
310		exit(1);
311	}
312	validate_free_pages(free_hugepages);
313	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
314
315	/*
316	 * Test MADV_REMOVE on shared and private mapping of hugetlb file
317	 */
318	if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) {
319		perror("fallocate");
320		exit(1);
321	}
322	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
323
324	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
325			PROT_READ | PROT_WRITE,
326			MAP_SHARED, fd, 0);
327	if (addr == MAP_FAILED) {
328		perror("mmap");
329		exit(1);
330	}
331
332	/* shared write should not consume any additional pages */
333	write_fault_pages(addr, NR_HUGE_PAGES);
334	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
335
336	addr2 = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
337			PROT_READ | PROT_WRITE,
338			MAP_PRIVATE, fd, 0);
339	if (addr2 == MAP_FAILED) {
340		perror("mmap");
341		exit(1);
342	}
343
344	/* private read should not consume any pages */
345	read_fault_pages(addr2, NR_HUGE_PAGES);
346	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
347
348	/* private write should consume additional pages */
349	write_fault_pages(addr2, NR_HUGE_PAGES);
350	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
351
352	/* madvise of shared mapping should not free any pages */
353	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
354		perror("madvise");
355		exit(1);
356	}
357	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
358
359	/* madvise of private mapping should free private pages */
360	if (madvise(addr2, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
361		perror("madvise");
362		exit(1);
363	}
364	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
365
366	/* private write should consume additional pages again */
367	write_fault_pages(addr2, NR_HUGE_PAGES);
368	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
369
370	/*
371	 * madvise should free both file and private pages although this is
372	 * not correct.  private pages should not be freed, but this is
373	 * expected.  See comment associated with FALLOC_FL_PUNCH_HOLE call.
374	 */
375	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) {
376		perror("madvise");
377		exit(1);
378	}
379	validate_free_pages(free_hugepages);
380
381	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
382	(void)munmap(addr2, NR_HUGE_PAGES * huge_page_size);
383
384	close(fd);
385	return 0;
386}
387