1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * GUP long-term page pinning tests.
4 *
5 * Copyright 2023, Red Hat, Inc.
6 *
7 * Author(s): David Hildenbrand <david@redhat.com>
8 */
9#define _GNU_SOURCE
10#include <stdlib.h>
11#include <string.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <unistd.h>
15#include <errno.h>
16#include <fcntl.h>
17#include <assert.h>
18#include <sys/mman.h>
19#include <sys/ioctl.h>
20#include <sys/vfs.h>
21#include <linux/magic.h>
22#include <linux/memfd.h>
23
24#include "local_config.h"
25#ifdef LOCAL_CONFIG_HAVE_LIBURING
26#include <liburing.h>
27#endif /* LOCAL_CONFIG_HAVE_LIBURING */
28
29#include "../../../../mm/gup_test.h"
30#include "../kselftest.h"
31#include "vm_util.h"
32
33static size_t pagesize;
34static int nr_hugetlbsizes;
35static size_t hugetlbsizes[10];
36static int gup_fd;
37
38static __fsword_t get_fs_type(int fd)
39{
40	struct statfs fs;
41	int ret;
42
43	do {
44		ret = fstatfs(fd, &fs);
45	} while (ret && errno == EINTR);
46
47	return ret ? 0 : fs.f_type;
48}
49
50static bool fs_is_unknown(__fsword_t fs_type)
51{
52	/*
53	 * We only support some filesystems in our tests when dealing with
54	 * R/W long-term pinning. For these filesystems, we can be fairly sure
55	 * whether they support it or not.
56	 */
57	switch (fs_type) {
58	case TMPFS_MAGIC:
59	case HUGETLBFS_MAGIC:
60	case BTRFS_SUPER_MAGIC:
61	case EXT4_SUPER_MAGIC:
62	case XFS_SUPER_MAGIC:
63		return false;
64	default:
65		return true;
66	}
67}
68
69static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type)
70{
71	assert(!fs_is_unknown(fs_type));
72	switch (fs_type) {
73	case TMPFS_MAGIC:
74	case HUGETLBFS_MAGIC:
75		return true;
76	default:
77		return false;
78	}
79}
80
81enum test_type {
82	TEST_TYPE_RO,
83	TEST_TYPE_RO_FAST,
84	TEST_TYPE_RW,
85	TEST_TYPE_RW_FAST,
86#ifdef LOCAL_CONFIG_HAVE_LIBURING
87	TEST_TYPE_IOURING,
88#endif /* LOCAL_CONFIG_HAVE_LIBURING */
89};
90
91static void do_test(int fd, size_t size, enum test_type type, bool shared)
92{
93	__fsword_t fs_type = get_fs_type(fd);
94	bool should_work;
95	char *mem;
96	int ret;
97
98	if (ftruncate(fd, size)) {
99		ksft_test_result_fail("ftruncate() failed\n");
100		return;
101	}
102
103	if (fallocate(fd, 0, 0, size)) {
104		if (size == pagesize)
105			ksft_test_result_fail("fallocate() failed\n");
106		else
107			ksft_test_result_skip("need more free huge pages\n");
108		return;
109	}
110
111	mem = mmap(NULL, size, PROT_READ | PROT_WRITE,
112		   shared ? MAP_SHARED : MAP_PRIVATE, fd, 0);
113	if (mem == MAP_FAILED) {
114		if (size == pagesize || shared)
115			ksft_test_result_fail("mmap() failed\n");
116		else
117			ksft_test_result_skip("need more free huge pages\n");
118		return;
119	}
120
121	/*
122	 * Fault in the page writable such that GUP-fast can eventually pin
123	 * it immediately.
124	 */
125	memset(mem, 0, size);
126
127	switch (type) {
128	case TEST_TYPE_RO:
129	case TEST_TYPE_RO_FAST:
130	case TEST_TYPE_RW:
131	case TEST_TYPE_RW_FAST: {
132		struct pin_longterm_test args;
133		const bool fast = type == TEST_TYPE_RO_FAST ||
134				  type == TEST_TYPE_RW_FAST;
135		const bool rw = type == TEST_TYPE_RW ||
136				type == TEST_TYPE_RW_FAST;
137
138		if (gup_fd < 0) {
139			ksft_test_result_skip("gup_test not available\n");
140			break;
141		}
142
143		if (rw && shared && fs_is_unknown(fs_type)) {
144			ksft_test_result_skip("Unknown filesystem\n");
145			return;
146		}
147		/*
148		 * R/O pinning or pinning in a private mapping is always
149		 * expected to work. Otherwise, we expect long-term R/W pinning
150		 * to only succeed for special fielesystems.
151		 */
152		should_work = !shared || !rw ||
153			      fs_supports_writable_longterm_pinning(fs_type);
154
155		args.addr = (__u64)(uintptr_t)mem;
156		args.size = size;
157		args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0;
158		args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0;
159		ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args);
160		if (ret && errno == EINVAL) {
161			ksft_test_result_skip("PIN_LONGTERM_TEST_START failed\n");
162			break;
163		} else if (ret && errno == EFAULT) {
164			ksft_test_result(!should_work, "Should have failed\n");
165			break;
166		} else if (ret) {
167			ksft_test_result_fail("PIN_LONGTERM_TEST_START failed\n");
168			break;
169		}
170
171		if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP))
172			ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed\n");
173
174		/*
175		 * TODO: if the kernel ever supports long-term R/W pinning on
176		 * some previously unsupported filesystems, we might want to
177		 * perform some additional tests for possible data corruptions.
178		 */
179		ksft_test_result(should_work, "Should have worked\n");
180		break;
181	}
182#ifdef LOCAL_CONFIG_HAVE_LIBURING
183	case TEST_TYPE_IOURING: {
184		struct io_uring ring;
185		struct iovec iov;
186
187		/* io_uring always pins pages writable. */
188		if (shared && fs_is_unknown(fs_type)) {
189			ksft_test_result_skip("Unknown filesystem\n");
190			return;
191		}
192		should_work = !shared ||
193			      fs_supports_writable_longterm_pinning(fs_type);
194
195		/* Skip on errors, as we might just lack kernel support. */
196		ret = io_uring_queue_init(1, &ring, 0);
197		if (ret < 0) {
198			ksft_test_result_skip("io_uring_queue_init() failed\n");
199			break;
200		}
201		/*
202		 * Register the range as a fixed buffer. This will FOLL_WRITE |
203		 * FOLL_PIN | FOLL_LONGTERM the range.
204		 */
205		iov.iov_base = mem;
206		iov.iov_len = size;
207		ret = io_uring_register_buffers(&ring, &iov, 1);
208		/* Only new kernels return EFAULT. */
209		if (ret && (errno == ENOSPC || errno == EOPNOTSUPP ||
210			    errno == EFAULT)) {
211			ksft_test_result(!should_work, "Should have failed\n");
212		} else if (ret) {
213			/*
214			 * We might just lack support or have insufficient
215			 * MEMLOCK limits.
216			 */
217			ksft_test_result_skip("io_uring_register_buffers() failed\n");
218		} else {
219			ksft_test_result(should_work, "Should have worked\n");
220			io_uring_unregister_buffers(&ring);
221		}
222
223		io_uring_queue_exit(&ring);
224		break;
225	}
226#endif /* LOCAL_CONFIG_HAVE_LIBURING */
227	default:
228		assert(false);
229	}
230
231	munmap(mem, size);
232}
233
234typedef void (*test_fn)(int fd, size_t size);
235
236static void run_with_memfd(test_fn fn, const char *desc)
237{
238	int fd;
239
240	ksft_print_msg("[RUN] %s ... with memfd\n", desc);
241
242	fd = memfd_create("test", 0);
243	if (fd < 0) {
244		ksft_test_result_fail("memfd_create() failed\n");
245		return;
246	}
247
248	fn(fd, pagesize);
249	close(fd);
250}
251
252static void run_with_tmpfile(test_fn fn, const char *desc)
253{
254	FILE *file;
255	int fd;
256
257	ksft_print_msg("[RUN] %s ... with tmpfile\n", desc);
258
259	file = tmpfile();
260	if (!file) {
261		ksft_test_result_fail("tmpfile() failed\n");
262		return;
263	}
264
265	fd = fileno(file);
266	if (fd < 0) {
267		ksft_test_result_fail("fileno() failed\n");
268		return;
269	}
270
271	fn(fd, pagesize);
272	fclose(file);
273}
274
275static void run_with_local_tmpfile(test_fn fn, const char *desc)
276{
277	char filename[] = __FILE__"_tmpfile_XXXXXX";
278	int fd;
279
280	ksft_print_msg("[RUN] %s ... with local tmpfile\n", desc);
281
282	fd = mkstemp(filename);
283	if (fd < 0) {
284		ksft_test_result_fail("mkstemp() failed\n");
285		return;
286	}
287
288	if (unlink(filename)) {
289		ksft_test_result_fail("unlink() failed\n");
290		goto close;
291	}
292
293	fn(fd, pagesize);
294close:
295	close(fd);
296}
297
298static void run_with_memfd_hugetlb(test_fn fn, const char *desc,
299				   size_t hugetlbsize)
300{
301	int flags = MFD_HUGETLB;
302	int fd;
303
304	ksft_print_msg("[RUN] %s ... with memfd hugetlb (%zu kB)\n", desc,
305		       hugetlbsize / 1024);
306
307	flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT;
308
309	fd = memfd_create("test", flags);
310	if (fd < 0) {
311		ksft_test_result_skip("memfd_create() failed\n");
312		return;
313	}
314
315	fn(fd, hugetlbsize);
316	close(fd);
317}
318
319struct test_case {
320	const char *desc;
321	test_fn fn;
322};
323
324static void test_shared_rw_pin(int fd, size_t size)
325{
326	do_test(fd, size, TEST_TYPE_RW, true);
327}
328
329static void test_shared_rw_fast_pin(int fd, size_t size)
330{
331	do_test(fd, size, TEST_TYPE_RW_FAST, true);
332}
333
334static void test_shared_ro_pin(int fd, size_t size)
335{
336	do_test(fd, size, TEST_TYPE_RO, true);
337}
338
339static void test_shared_ro_fast_pin(int fd, size_t size)
340{
341	do_test(fd, size, TEST_TYPE_RO_FAST, true);
342}
343
344static void test_private_rw_pin(int fd, size_t size)
345{
346	do_test(fd, size, TEST_TYPE_RW, false);
347}
348
349static void test_private_rw_fast_pin(int fd, size_t size)
350{
351	do_test(fd, size, TEST_TYPE_RW_FAST, false);
352}
353
354static void test_private_ro_pin(int fd, size_t size)
355{
356	do_test(fd, size, TEST_TYPE_RO, false);
357}
358
359static void test_private_ro_fast_pin(int fd, size_t size)
360{
361	do_test(fd, size, TEST_TYPE_RO_FAST, false);
362}
363
364#ifdef LOCAL_CONFIG_HAVE_LIBURING
365static void test_shared_iouring(int fd, size_t size)
366{
367	do_test(fd, size, TEST_TYPE_IOURING, true);
368}
369
370static void test_private_iouring(int fd, size_t size)
371{
372	do_test(fd, size, TEST_TYPE_IOURING, false);
373}
374#endif /* LOCAL_CONFIG_HAVE_LIBURING */
375
376static const struct test_case test_cases[] = {
377	{
378		"R/W longterm GUP pin in MAP_SHARED file mapping",
379		test_shared_rw_pin,
380	},
381	{
382		"R/W longterm GUP-fast pin in MAP_SHARED file mapping",
383		test_shared_rw_fast_pin,
384	},
385	{
386		"R/O longterm GUP pin in MAP_SHARED file mapping",
387		test_shared_ro_pin,
388	},
389	{
390		"R/O longterm GUP-fast pin in MAP_SHARED file mapping",
391		test_shared_ro_fast_pin,
392	},
393	{
394		"R/W longterm GUP pin in MAP_PRIVATE file mapping",
395		test_private_rw_pin,
396	},
397	{
398		"R/W longterm GUP-fast pin in MAP_PRIVATE file mapping",
399		test_private_rw_fast_pin,
400	},
401	{
402		"R/O longterm GUP pin in MAP_PRIVATE file mapping",
403		test_private_ro_pin,
404	},
405	{
406		"R/O longterm GUP-fast pin in MAP_PRIVATE file mapping",
407		test_private_ro_fast_pin,
408	},
409#ifdef LOCAL_CONFIG_HAVE_LIBURING
410	{
411		"io_uring fixed buffer with MAP_SHARED file mapping",
412		test_shared_iouring,
413	},
414	{
415		"io_uring fixed buffer with MAP_PRIVATE file mapping",
416		test_private_iouring,
417	},
418#endif /* LOCAL_CONFIG_HAVE_LIBURING */
419};
420
421static void run_test_case(struct test_case const *test_case)
422{
423	int i;
424
425	run_with_memfd(test_case->fn, test_case->desc);
426	run_with_tmpfile(test_case->fn, test_case->desc);
427	run_with_local_tmpfile(test_case->fn, test_case->desc);
428	for (i = 0; i < nr_hugetlbsizes; i++)
429		run_with_memfd_hugetlb(test_case->fn, test_case->desc,
430				       hugetlbsizes[i]);
431}
432
433static int tests_per_test_case(void)
434{
435	return 3 + nr_hugetlbsizes;
436}
437
438int main(int argc, char **argv)
439{
440	int i, err;
441
442	pagesize = getpagesize();
443	nr_hugetlbsizes = detect_hugetlb_page_sizes(hugetlbsizes,
444						    ARRAY_SIZE(hugetlbsizes));
445
446	ksft_print_header();
447	ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case());
448
449	gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
450
451	for (i = 0; i < ARRAY_SIZE(test_cases); i++)
452		run_test_case(&test_cases[i]);
453
454	err = ksft_get_fail_cnt();
455	if (err)
456		ksft_exit_fail_msg("%d out of %d tests failed\n",
457				   err, ksft_test_num());
458	return ksft_exit_pass();
459}
460