1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_IRQ_STACK_H
3#define _ASM_X86_IRQ_STACK_H
4
5#include <linux/ptrace.h>
6
7#include <asm/processor.h>
8
9#ifdef CONFIG_X86_64
10static __always_inline bool irqstack_active(void)
11{
12	return __this_cpu_read(irq_count) != -1;
13}
14
15void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
16void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
17			      struct pt_regs *regs);
18void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
19			   struct irq_desc *desc);
20
21static __always_inline void __run_on_irqstack(void (*func)(void))
22{
23	void *tos = __this_cpu_read(hardirq_stack_ptr);
24
25	__this_cpu_add(irq_count, 1);
26	asm_call_on_stack(tos - 8, func, NULL);
27	__this_cpu_sub(irq_count, 1);
28}
29
30static __always_inline void
31__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
32			 struct pt_regs *regs)
33{
34	void *tos = __this_cpu_read(hardirq_stack_ptr);
35
36	__this_cpu_add(irq_count, 1);
37	asm_call_sysvec_on_stack(tos - 8, func, regs);
38	__this_cpu_sub(irq_count, 1);
39}
40
41static __always_inline void
42__run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
43		      struct irq_desc *desc)
44{
45	void *tos = __this_cpu_read(hardirq_stack_ptr);
46
47	__this_cpu_add(irq_count, 1);
48	asm_call_irq_on_stack(tos - 8, func, desc);
49	__this_cpu_sub(irq_count, 1);
50}
51
52#else /* CONFIG_X86_64 */
53static inline bool irqstack_active(void) { return false; }
54static inline void __run_on_irqstack(void (*func)(void)) { }
55static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
56					    struct pt_regs *regs) { }
57static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
58					 struct irq_desc *desc) { }
59#endif /* !CONFIG_X86_64 */
60
61static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
62{
63	if (IS_ENABLED(CONFIG_X86_32))
64		return false;
65	if (!regs)
66		return !irqstack_active();
67	return !user_mode(regs) && !irqstack_active();
68}
69
70
71static __always_inline void run_on_irqstack_cond(void (*func)(void),
72						 struct pt_regs *regs)
73{
74	lockdep_assert_irqs_disabled();
75
76	if (irq_needs_irq_stack(regs))
77		__run_on_irqstack(func);
78	else
79		func();
80}
81
82static __always_inline void
83run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
84			    struct pt_regs *regs)
85{
86	lockdep_assert_irqs_disabled();
87
88	if (irq_needs_irq_stack(regs))
89		__run_sysvec_on_irqstack(func, regs);
90	else
91		func(regs);
92}
93
94static __always_inline void
95run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
96			 struct pt_regs *regs)
97{
98	lockdep_assert_irqs_disabled();
99
100	if (irq_needs_irq_stack(regs))
101		__run_irq_on_irqstack(func, desc);
102	else
103		func(desc);
104}
105
106#endif
107