1// SPDX-License-Identifier: GPL-2.0
2#include <string.h>
3#include <fcntl.h>
4#include <dirent.h>
5#include <sys/ioctl.h>
6#include <linux/userfaultfd.h>
7#include <sys/syscall.h>
8#include <unistd.h>
9#include "../kselftest.h"
10#include "vm_util.h"
11
12#define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
13#define SMAP_FILE_PATH "/proc/self/smaps"
14#define MAX_LINE_LENGTH 500
15
16unsigned int __page_size;
17unsigned int __page_shift;
18
19uint64_t pagemap_get_entry(int fd, char *start)
20{
21	const unsigned long pfn = (unsigned long)start / getpagesize();
22	uint64_t entry;
23	int ret;
24
25	ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
26	if (ret != sizeof(entry))
27		ksft_exit_fail_msg("reading pagemap failed\n");
28	return entry;
29}
30
31bool pagemap_is_softdirty(int fd, char *start)
32{
33	return pagemap_get_entry(fd, start) & PM_SOFT_DIRTY;
34}
35
36bool pagemap_is_swapped(int fd, char *start)
37{
38	return pagemap_get_entry(fd, start) & PM_SWAP;
39}
40
41bool pagemap_is_populated(int fd, char *start)
42{
43	return pagemap_get_entry(fd, start) & (PM_PRESENT | PM_SWAP);
44}
45
46unsigned long pagemap_get_pfn(int fd, char *start)
47{
48	uint64_t entry = pagemap_get_entry(fd, start);
49
50	/* If present (63th bit), PFN is at bit 0 -- 54. */
51	if (entry & PM_PRESENT)
52		return entry & 0x007fffffffffffffull;
53	return -1ul;
54}
55
56void clear_softdirty(void)
57{
58	int ret;
59	const char *ctrl = "4";
60	int fd = open("/proc/self/clear_refs", O_WRONLY);
61
62	if (fd < 0)
63		ksft_exit_fail_msg("opening clear_refs failed\n");
64	ret = write(fd, ctrl, strlen(ctrl));
65	close(fd);
66	if (ret != strlen(ctrl))
67		ksft_exit_fail_msg("writing clear_refs failed\n");
68}
69
70bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
71{
72	while (fgets(buf, len, fp)) {
73		if (!strncmp(buf, pattern, strlen(pattern)))
74			return true;
75	}
76	return false;
77}
78
79uint64_t read_pmd_pagesize(void)
80{
81	int fd;
82	char buf[20];
83	ssize_t num_read;
84
85	fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
86	if (fd == -1)
87		return 0;
88
89	num_read = read(fd, buf, 19);
90	if (num_read < 1) {
91		close(fd);
92		return 0;
93	}
94	buf[num_read] = '\0';
95	close(fd);
96
97	return strtoul(buf, NULL, 10);
98}
99
100bool __check_huge(void *addr, char *pattern, int nr_hpages,
101		  uint64_t hpage_size)
102{
103	uint64_t thp = -1;
104	int ret;
105	FILE *fp;
106	char buffer[MAX_LINE_LENGTH];
107	char addr_pattern[MAX_LINE_LENGTH];
108
109	ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
110		       (unsigned long) addr);
111	if (ret >= MAX_LINE_LENGTH)
112		ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
113
114	fp = fopen(SMAP_FILE_PATH, "r");
115	if (!fp)
116		ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
117
118	if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer)))
119		goto err_out;
120
121	/*
122	 * Fetch the pattern in the same block and check the number of
123	 * hugepages.
124	 */
125	if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer)))
126		goto err_out;
127
128	snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern);
129
130	if (sscanf(buffer, addr_pattern, &thp) != 1)
131		ksft_exit_fail_msg("Reading smap error\n");
132
133err_out:
134	fclose(fp);
135	return thp == (nr_hpages * (hpage_size >> 10));
136}
137
138bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size)
139{
140	return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
141}
142
143bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size)
144{
145	return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
146}
147
148bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size)
149{
150	return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
151}
152
153int64_t allocate_transhuge(void *ptr, int pagemap_fd)
154{
155	uint64_t ent[2];
156
157	/* drop pmd */
158	if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
159		 MAP_FIXED | MAP_ANONYMOUS |
160		 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
161		errx(2, "mmap transhuge");
162
163	if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
164		err(2, "MADV_HUGEPAGE");
165
166	/* allocate transparent huge page */
167	*(volatile void **)ptr = ptr;
168
169	if (pread(pagemap_fd, ent, sizeof(ent),
170		  (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
171		err(2, "read pagemap");
172
173	if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
174	    PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
175	    !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
176		return PAGEMAP_PFN(ent[0]);
177
178	return -1;
179}
180
181unsigned long default_huge_page_size(void)
182{
183	unsigned long hps = 0;
184	char *line = NULL;
185	size_t linelen = 0;
186	FILE *f = fopen("/proc/meminfo", "r");
187
188	if (!f)
189		return 0;
190	while (getline(&line, &linelen, f) > 0) {
191		if (sscanf(line, "Hugepagesize:       %lu kB", &hps) == 1) {
192			hps <<= 10;
193			break;
194		}
195	}
196
197	free(line);
198	fclose(f);
199	return hps;
200}
201
202int detect_hugetlb_page_sizes(size_t sizes[], int max)
203{
204	DIR *dir = opendir("/sys/kernel/mm/hugepages/");
205	int count = 0;
206
207	if (!dir)
208		return 0;
209
210	while (count < max) {
211		struct dirent *entry = readdir(dir);
212		size_t kb;
213
214		if (!entry)
215			break;
216		if (entry->d_type != DT_DIR)
217			continue;
218		if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1)
219			continue;
220		sizes[count++] = kb * 1024;
221		ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n",
222			       kb);
223	}
224	closedir(dir);
225	return count;
226}
227
228/* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
229int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
230			      bool miss, bool wp, bool minor, uint64_t *ioctls)
231{
232	struct uffdio_register uffdio_register = { 0 };
233	uint64_t mode = 0;
234	int ret = 0;
235
236	if (miss)
237		mode |= UFFDIO_REGISTER_MODE_MISSING;
238	if (wp)
239		mode |= UFFDIO_REGISTER_MODE_WP;
240	if (minor)
241		mode |= UFFDIO_REGISTER_MODE_MINOR;
242
243	uffdio_register.range.start = (unsigned long)addr;
244	uffdio_register.range.len = len;
245	uffdio_register.mode = mode;
246
247	if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
248		ret = -errno;
249	else if (ioctls)
250		*ioctls = uffdio_register.ioctls;
251
252	return ret;
253}
254
255int uffd_register(int uffd, void *addr, uint64_t len,
256		  bool miss, bool wp, bool minor)
257{
258	return uffd_register_with_ioctls(uffd, addr, len,
259					 miss, wp, minor, NULL);
260}
261
262int uffd_unregister(int uffd, void *addr, uint64_t len)
263{
264	struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
265	int ret = 0;
266
267	if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
268		ret = -errno;
269
270	return ret;
271}
272