1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_FUTEX_H
3#define _ASM_X86_FUTEX_H
4
5#ifdef __KERNEL__
6
7#include <linux/futex.h>
8#include <linux/uaccess.h>
9
10#include <asm/asm.h>
11#include <asm/errno.h>
12#include <asm/processor.h>
13#include <asm/smap.h>
14
15#define unsafe_atomic_op1(insn, oval, uaddr, oparg, label)	\
16do {								\
17	int oldval = 0, ret;					\
18	asm volatile("1:\t" insn "\n"				\
19		     "2:\n"					\
20		     "\t.section .fixup,\"ax\"\n"		\
21		     "3:\tmov\t%3, %1\n"			\
22		     "\tjmp\t2b\n"				\
23		     "\t.previous\n"				\
24		     _ASM_EXTABLE_UA(1b, 3b)			\
25		     : "=r" (oldval), "=r" (ret), "+m" (*uaddr)	\
26		     : "i" (-EFAULT), "0" (oparg), "1" (0));	\
27	if (ret)						\
28		goto label;					\
29	*oval = oldval;						\
30} while(0)
31
32
33#define unsafe_atomic_op2(insn, oval, uaddr, oparg, label)	\
34do {								\
35	int oldval = 0, ret, tem;				\
36	asm volatile("1:\tmovl	%2, %0\n"			\
37		     "2:\tmovl\t%0, %3\n"			\
38		     "\t" insn "\n"				\
39		     "3:\t" LOCK_PREFIX "cmpxchgl %3, %2\n"	\
40		     "\tjnz\t2b\n"				\
41		     "4:\n"					\
42		     "\t.section .fixup,\"ax\"\n"		\
43		     "5:\tmov\t%5, %1\n"			\
44		     "\tjmp\t4b\n"				\
45		     "\t.previous\n"				\
46		     _ASM_EXTABLE_UA(1b, 5b)			\
47		     _ASM_EXTABLE_UA(3b, 5b)			\
48		     : "=&a" (oldval), "=&r" (ret),		\
49		       "+m" (*uaddr), "=&r" (tem)		\
50		     : "r" (oparg), "i" (-EFAULT), "1" (0));	\
51	if (ret)						\
52		goto label;					\
53	*oval = oldval;						\
54} while(0)
55
56static __always_inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval,
57		u32 __user *uaddr)
58{
59	if (!user_access_begin(uaddr, sizeof(u32)))
60		return -EFAULT;
61
62	switch (op) {
63	case FUTEX_OP_SET:
64		unsafe_atomic_op1("xchgl %0, %2", oval, uaddr, oparg, Efault);
65		break;
66	case FUTEX_OP_ADD:
67		unsafe_atomic_op1(LOCK_PREFIX "xaddl %0, %2", oval,
68				   uaddr, oparg, Efault);
69		break;
70	case FUTEX_OP_OR:
71		unsafe_atomic_op2("orl %4, %3", oval, uaddr, oparg, Efault);
72		break;
73	case FUTEX_OP_ANDN:
74		unsafe_atomic_op2("andl %4, %3", oval, uaddr, ~oparg, Efault);
75		break;
76	case FUTEX_OP_XOR:
77		unsafe_atomic_op2("xorl %4, %3", oval, uaddr, oparg, Efault);
78		break;
79	default:
80		user_access_end();
81		return -ENOSYS;
82	}
83	user_access_end();
84	return 0;
85Efault:
86	user_access_end();
87	return -EFAULT;
88}
89
90static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
91						u32 oldval, u32 newval)
92{
93	int ret = 0;
94
95	if (!user_access_begin(uaddr, sizeof(u32)))
96		return -EFAULT;
97	asm volatile("\n"
98		"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"
99		"2:\n"
100		"\t.section .fixup, \"ax\"\n"
101		"3:\tmov     %3, %0\n"
102		"\tjmp     2b\n"
103		"\t.previous\n"
104		_ASM_EXTABLE_UA(1b, 3b)
105		: "+r" (ret), "=a" (oldval), "+m" (*uaddr)
106		: "i" (-EFAULT), "r" (newval), "1" (oldval)
107		: "memory"
108	);
109	user_access_end();
110	*uval = oldval;
111	return ret;
112}
113
114#endif
115#endif /* _ASM_X86_FUTEX_H */
116