summaryrefslogtreecommitdiffstats
path: root/arch/x86/include/asm/irq_stack.h
blob: 775816965c6ae098f249dcf83c61d131da0ced9a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_IRQ_STACK_H
#define _ASM_X86_IRQ_STACK_H

#include <linux/ptrace.h>

#include <asm/processor.h>

#ifdef CONFIG_X86_64
static __always_inline bool irqstack_active(void)
{
	return __this_cpu_read(irq_count) != -1;
}

void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
			      struct pt_regs *regs);
void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
			   struct irq_desc *desc);

static __always_inline void __run_on_irqstack(void (*func)(void))
{
	void *tos = __this_cpu_read(hardirq_stack_ptr);

	__this_cpu_add(irq_count, 1);
	asm_call_on_stack(tos - 8, func, NULL);
	__this_cpu_sub(irq_count, 1);
}

static __always_inline void
__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
			 struct pt_regs *regs)
{
	void *tos = __this_cpu_read(hardirq_stack_ptr);

	__this_cpu_add(irq_count, 1);
	asm_call_sysvec_on_stack(tos - 8, func, regs);
	__this_cpu_sub(irq_count, 1);
}

static __always_inline void
__run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
		      struct irq_desc *desc)
{
	void *tos = __this_cpu_read(hardirq_stack_ptr);

	__this_cpu_add(irq_count, 1);
	asm_call_irq_on_stack(tos - 8, func, desc);
	__this_cpu_sub(irq_count, 1);
}

#else /* CONFIG_X86_64 */
static inline bool irqstack_active(void) { return false; }
static inline void __run_on_irqstack(void (*func)(void)) { }
static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
					    struct pt_regs *regs) { }
static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
					 struct irq_desc *desc) { }
#endif /* !CONFIG_X86_64 */

static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
{
	if (IS_ENABLED(CONFIG_X86_32))
		return false;
	if (!regs)
		return !irqstack_active();
	return !user_mode(regs) && !irqstack_active();
}


static __always_inline void run_on_irqstack_cond(void (*func)(void),
						 struct pt_regs *regs)
{
	lockdep_assert_irqs_disabled();

	if (irq_needs_irq_stack(regs))
		__run_on_irqstack(func);
	else
		func();
}

static __always_inline void
run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
			    struct pt_regs *regs)
{
	lockdep_assert_irqs_disabled();

	if (irq_needs_irq_stack(regs))
		__run_sysvec_on_irqstack(func, regs);
	else
		func(regs);
}

static __always_inline void
run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
			 struct pt_regs *regs)
{
	lockdep_assert_irqs_disabled();

	if (irq_needs_irq_stack(regs))
		__run_irq_on_irqstack(func, desc);
	else
		func(desc);
}

#endif