Line data Source code
1 1 : /*
2 : * Copyright (c) 2010-2014 Wind River Systems, Inc.
3 : *
4 : * SPDX-License-Identifier: Apache-2.0
5 : */
6 :
7 : /**
8 : * @file
9 : * @brief IA-32 specific kernel interface header
10 : *
11 : * This header contains the IA-32 portion of the X86 specific kernel
12 : * interface (see include/zephyr/arch/x86/cpu.h).
13 : */
14 :
15 : #ifndef ZEPHYR_INCLUDE_ARCH_X86_IA32_ARCH_H_
16 : #define ZEPHYR_INCLUDE_ARCH_X86_IA32_ARCH_H_
17 :
18 : #include "sys_io.h"
19 : #include <stdbool.h>
20 : #include <zephyr/kernel_structs.h>
21 : #include <zephyr/arch/common/ffs.h>
22 : #include <zephyr/sys/util.h>
23 : #include <zephyr/arch/exception.h>
24 : #include <zephyr/arch/x86/ia32/gdbstub.h>
25 : #include <zephyr/arch/x86/ia32/thread.h>
26 : #include <zephyr/arch/x86/ia32/syscall.h>
27 :
28 : #ifndef _ASMLANGUAGE
29 : #include <stddef.h> /* for size_t */
30 :
31 : #include <zephyr/arch/common/addr_types.h>
32 : #include <zephyr/arch/x86/ia32/segmentation.h>
33 : #include <zephyr/pm/pm.h>
34 :
35 : #endif /* _ASMLANGUAGE */
36 :
37 : /* GDT layout */
38 0 : #define CODE_SEG 0x08
39 0 : #define DATA_SEG 0x10
40 0 : #define MAIN_TSS 0x18
41 0 : #define DF_TSS 0x20
42 :
43 : /*
44 : * Use for thread local storage.
45 : * Match these to gen_gdt.py.
46 : * The 0x03 is added to limit privilege.
47 : */
48 : #if defined(CONFIG_USERSPACE)
49 0 : #define GS_TLS_SEG (0x38 | 0x03)
50 : #elif defined(CONFIG_X86_STACK_PROTECTION)
51 : #define GS_TLS_SEG (0x28 | 0x03)
52 : #else
53 : #define GS_TLS_SEG (0x18 | 0x03)
54 : #endif
55 :
56 : /**
57 : * Macro used internally by NANO_CPU_INT_REGISTER and NANO_CPU_INT_REGISTER_ASM.
58 : * Not meant to be used explicitly by platform, driver or application code.
59 : */
60 1 : #define MK_ISR_NAME(x) __isr__##x
61 :
62 : #define Z_DYN_STUB_SIZE 8
63 : #define Z_DYN_STUB_OFFSET 0
64 : #define Z_DYN_STUB_LONG_JMP_EXTRA_SIZE 3
65 : #define Z_DYN_STUB_PER_BLOCK 32
66 :
67 :
68 : #ifndef _ASMLANGUAGE
69 :
70 : #ifdef __cplusplus
71 : extern "C" {
72 : #endif
73 :
74 : /* interrupt/exception/error related definitions */
75 :
76 0 : typedef struct s_isrList {
77 : /** Address of ISR/stub */
78 1 : void *fnc;
79 : /** IRQ associated with the ISR/stub, or -1 if this is not
80 : * associated with a real interrupt; in this case vec must
81 : * not be -1
82 : */
83 1 : unsigned int irq;
84 : /** Priority associated with the IRQ. Ignored if vec is not -1 */
85 1 : unsigned int priority;
86 : /** Vector number associated with ISR/stub, or -1 to assign based
87 : * on priority
88 : */
89 1 : unsigned int vec;
90 : /** Privilege level associated with ISR/stub */
91 1 : unsigned int dpl;
92 :
93 : /** If nonzero, specifies a TSS segment selector. Will configure
94 : * a task gate instead of an interrupt gate. fnc parameter will be
95 : * ignored
96 : */
97 1 : unsigned int tss;
98 0 : } ISR_LIST;
99 :
100 :
101 : /**
102 : * @brief Connect a routine to an interrupt vector
103 : *
104 : * This macro "connects" the specified routine, @a r, to the specified interrupt
105 : * vector, @a v using the descriptor privilege level @a d. On the IA-32
106 : * architecture, an interrupt vector is a value from 0 to 255. This macro
107 : * populates the special intList section with the address of the routine, the
108 : * vector number and the descriptor privilege level. The genIdt tool then picks
109 : * up this information and generates an actual IDT entry with this information
110 : * properly encoded.
111 : *
112 : * The @a d argument specifies the privilege level for the interrupt-gate
113 : * descriptor; (hardware) interrupts and exceptions should specify a level of 0,
114 : * whereas handlers for user-mode software generated interrupts should specify 3.
115 : * @param r Routine to be connected
116 : * @param n IRQ number
117 : * @param p IRQ priority
118 : * @param v Interrupt Vector
119 : * @param d Descriptor Privilege Level
120 : */
121 :
122 1 : #define NANO_CPU_INT_REGISTER(r, n, p, v, d) \
123 : static ISR_LIST __attribute__((section(".intList"))) \
124 : __attribute__((used)) MK_ISR_NAME(r) = \
125 : { \
126 : .fnc = &(r), \
127 : .irq = (n), \
128 : .priority = (p), \
129 : .vec = (v), \
130 : .dpl = (d), \
131 : .tss = 0 \
132 : }
133 :
134 : /**
135 : * @brief Connect an IA hardware task to an interrupt vector
136 : *
137 : * This is very similar to NANO_CPU_INT_REGISTER but instead of connecting
138 : * a handler function, the interrupt will induce an IA hardware task
139 : * switch to another hardware task instead.
140 : *
141 : * @param tss_p GDT/LDT segment selector for the TSS representing the task
142 : * @param irq_p IRQ number
143 : * @param priority_p IRQ priority
144 : * @param vec_p Interrupt vector
145 : * @param dpl_p Descriptor privilege level
146 : */
147 : #define _X86_IDT_TSS_REGISTER(tss_p, irq_p, priority_p, vec_p, dpl_p) \
148 : static ISR_LIST __attribute__((section(".intList"))) \
149 : __attribute__((used)) MK_ISR_NAME(vec_p) = \
150 : { \
151 : .fnc = NULL, \
152 : .irq = (irq_p), \
153 : .priority = (priority_p), \
154 : .vec = (vec_p), \
155 : .dpl = (dpl_p), \
156 : .tss = (tss_p) \
157 : }
158 :
159 : /**
160 : * Code snippets for populating the vector ID and priority into the intList
161 : *
162 : * The 'magic' of static interrupts is accomplished by building up an array
163 : * 'intList' at compile time, and the gen_idt tool uses this to create the
164 : * actual IDT data structure.
165 : *
166 : * For controllers like APIC, the vectors in the IDT are not normally assigned
167 : * at build time; instead the sentinel value -1 is saved, and gen_idt figures
168 : * out the right vector to use based on our priority scheme. Groups of 16
169 : * vectors starting at 32 correspond to each priority level.
170 : *
171 : * These macros are only intended to be used by IRQ_CONNECT() macro.
172 : */
173 : #define _VECTOR_ARG(irq_p) (-1)
174 :
175 : #ifdef CONFIG_LINKER_USE_PINNED_SECTION
176 : #define IRQSTUBS_TEXT_SECTION ".pinned_text.irqstubs"
177 : #else
178 0 : #define IRQSTUBS_TEXT_SECTION ".text.irqstubs"
179 : #endif
180 :
181 : /* Internally this function does a few things:
182 : *
183 : * 1. There is a declaration of the interrupt parameters in the .intList
184 : * section, used by gen_idt to create the IDT. This does the same thing
185 : * as the NANO_CPU_INT_REGISTER() macro, but is done in assembly as we
186 : * need to populate the .fnc member with the address of the assembly
187 : * IRQ stub that we generate immediately afterwards.
188 : *
189 : * 2. The IRQ stub itself is declared. The code will go in its own named
190 : * section .text.irqstubs section (which eventually gets linked into 'text')
191 : * and the stub shall be named (isr_name)_irq(irq_line)_stub
192 : *
193 : * 3. The IRQ stub pushes the ISR routine and its argument onto the stack
194 : * and then jumps to the common interrupt handling code in _interrupt_enter().
195 : *
196 : * 4. z_irq_controller_irq_config() is called at runtime to set the mapping
197 : * between the vector and the IRQ line as well as triggering flags
198 : */
199 0 : #define ARCH_IRQ_CONNECT(irq_p, priority_p, isr_p, isr_param_p, flags_p) \
200 : { \
201 : __asm__ __volatile__( \
202 : ".pushsection .intList\n\t" \
203 : ".long %c[isr]_irq%c[irq]_stub\n\t" /* ISR_LIST.fnc */ \
204 : ".long %c[irq]\n\t" /* ISR_LIST.irq */ \
205 : ".long %c[priority]\n\t" /* ISR_LIST.priority */ \
206 : ".long %c[vector]\n\t" /* ISR_LIST.vec */ \
207 : ".long 0\n\t" /* ISR_LIST.dpl */ \
208 : ".long 0\n\t" /* ISR_LIST.tss */ \
209 : ".popsection\n\t" \
210 : ".pushsection " IRQSTUBS_TEXT_SECTION "\n\t" \
211 : ".global %c[isr]_irq%c[irq]_stub\n\t" \
212 : "%c[isr]_irq%c[irq]_stub:\n\t" \
213 : "endbr32\n\t" \
214 : "pushl %[isr_param]\n\t" \
215 : "pushl %[isr]\n\t" \
216 : "jmp _interrupt_enter\n\t" \
217 : ".popsection\n\t" \
218 : : \
219 : : [isr] "i" (isr_p), \
220 : [isr_param] "i" (isr_param_p), \
221 : [priority] "i" (priority_p), \
222 : [vector] "i" _VECTOR_ARG(irq_p), \
223 : [irq] "i" (irq_p)); \
224 : z_irq_controller_irq_config(Z_IRQ_TO_INTERRUPT_VECTOR(irq_p), (irq_p), \
225 : (flags_p)); \
226 : }
227 :
228 : #ifdef CONFIG_PCIE
229 :
230 : #define ARCH_PCIE_IRQ_CONNECT(bdf_p, irq_p, priority_p, \
231 : isr_p, isr_param_p, flags_p) \
232 : ARCH_IRQ_CONNECT(irq_p, priority_p, isr_p, isr_param_p, flags_p)
233 :
234 : #endif /* CONFIG_PCIE */
235 :
236 : /* Direct interrupts won't work as expected with KPTI turned on, because
237 : * all non-user accessible pages in the page table are marked non-present.
238 : * It's likely possible to add logic to ARCH_ISR_DIRECT_HEADER/FOOTER to do
239 : * the necessary trampolining to switch page tables / stacks, but this
240 : * probably loses all the latency benefits that direct interrupts provide
241 : * and one might as well use a regular interrupt anyway.
242 : */
243 : #ifndef CONFIG_X86_KPTI
244 0 : #define ARCH_IRQ_DIRECT_CONNECT(irq_p, priority_p, isr_p, flags_p) \
245 : { \
246 : NANO_CPU_INT_REGISTER(isr_p, irq_p, priority_p, -1, 0); \
247 : z_irq_controller_irq_config(Z_IRQ_TO_INTERRUPT_VECTOR(irq_p), (irq_p), \
248 : (flags_p)); \
249 : }
250 :
251 : #ifdef CONFIG_PM
252 : static inline void arch_irq_direct_pm(void)
253 : {
254 : if (_kernel.idle) {
255 : _kernel.idle = 0;
256 : pm_system_resume();
257 : }
258 : }
259 :
260 : #define ARCH_ISR_DIRECT_PM() arch_irq_direct_pm()
261 : #else
262 0 : #define ARCH_ISR_DIRECT_PM() do { } while (false)
263 : #endif
264 :
265 0 : #define ARCH_ISR_DIRECT_HEADER() arch_isr_direct_header()
266 0 : #define ARCH_ISR_DIRECT_FOOTER(swap) arch_isr_direct_footer(swap)
267 :
268 : /* FIXME:
269 : * tracing/tracing.h cannot be included here due to circular dependency
270 : */
271 : #if defined(CONFIG_TRACING)
272 : void sys_trace_isr_enter(void);
273 : void sys_trace_isr_exit(void);
274 : #endif
275 :
276 0 : static inline void arch_isr_direct_header(void)
277 : {
278 : #if defined(CONFIG_TRACING)
279 : sys_trace_isr_enter();
280 : #endif
281 :
282 : /* We're not going to unlock IRQs, but we still need to increment this
283 : * so that arch_is_in_isr() works
284 : */
285 : ++_kernel.cpus[0].nested;
286 : }
287 :
288 : /*
289 : * FIXME: z_swap_irqlock is an inline function declared in a private header and
290 : * cannot be referenced from a public header, so we move it to an
291 : * external function.
292 : */
293 0 : void arch_isr_direct_footer_swap(unsigned int key);
294 :
295 0 : static inline void arch_isr_direct_footer(int swap)
296 : {
297 : z_irq_controller_eoi();
298 : #if defined(CONFIG_TRACING)
299 : sys_trace_isr_exit();
300 : #endif
301 : --_kernel.cpus[0].nested;
302 :
303 : /* Call swap if all the following is true:
304 : *
305 : * 1) swap argument was enabled to this function
306 : * 2) We are not in a nested interrupt
307 : * 3) Next thread to run in the ready queue is not this thread
308 : */
309 : if (swap != 0 && _kernel.cpus[0].nested == 0 &&
310 : _kernel.ready_q.cache != _current) {
311 : unsigned int flags;
312 :
313 : /* Fetch EFLAGS argument to z_swap() */
314 : __asm__ volatile (
315 : "pushfl\n\t"
316 : "popl %0\n\t"
317 : : "=g" (flags)
318 : :
319 : : "memory"
320 : );
321 :
322 : arch_isr_direct_footer_swap(flags);
323 : }
324 : }
325 :
326 0 : #define ARCH_ISR_DIRECT_DECLARE(name) \
327 : static inline int name##_body(void); \
328 : __attribute__ ((interrupt)) void name(void *stack_frame) \
329 : { \
330 : ARG_UNUSED(stack_frame); \
331 : int check_reschedule; \
332 : ISR_DIRECT_HEADER(); \
333 : check_reschedule = name##_body(); \
334 : ISR_DIRECT_FOOTER(check_reschedule); \
335 : } \
336 : static inline int name##_body(void)
337 : #endif /* !CONFIG_X86_KPTI */
338 :
339 0 : static ALWAYS_INLINE unsigned int arch_irq_lock(void)
340 : {
341 : unsigned int key;
342 :
343 : __asm__ volatile ("pushfl; cli; popl %0" : "=g" (key) :: "memory");
344 :
345 : return key;
346 : }
347 :
348 :
349 : /**
350 : * The NANO_SOFT_IRQ macro must be used as the value for the @a irq parameter
351 : * to NANO_CPU_INT_REGISTER when connecting to an interrupt that does not
352 : * correspond to any IRQ line (such as spurious vector or SW IRQ)
353 : */
354 1 : #define NANO_SOFT_IRQ ((unsigned int) (-1))
355 :
356 : #ifdef CONFIG_X86_ENABLE_TSS
357 : extern struct task_state_segment _main_tss;
358 : #endif
359 :
360 0 : #define ARCH_EXCEPT(reason_p) do { \
361 : __asm__ volatile( \
362 : "push %[reason]\n\t" \
363 : "int %[vector]\n\t" \
364 : : \
365 : : [vector] "i" (Z_X86_OOPS_VECTOR), \
366 : [reason] "i" (reason_p) \
367 : : "memory"); \
368 : CODE_UNREACHABLE; /* LCOV_EXCL_LINE */ \
369 : } while (false)
370 :
371 : /*
372 : * Dynamic thread object memory alignment.
373 : *
374 : * If support for SSEx extensions is enabled a 16 byte boundary is required,
375 : * since the 'fxsave' and 'fxrstor' instructions require this. In all other
376 : * cases a 4 byte boundary is sufficient.
377 : */
378 : #if defined(CONFIG_EAGER_FPU_SHARING) || defined(CONFIG_LAZY_FPU_SHARING)
379 : #ifdef CONFIG_SSE
380 : #define ARCH_DYNAMIC_OBJ_K_THREAD_ALIGNMENT 16
381 : #else
382 : #define ARCH_DYNAMIC_OBJ_K_THREAD_ALIGNMENT (sizeof(void *))
383 : #endif
384 : #else
385 : /* No special alignment requirements, simply align on pointer size. */
386 0 : #define ARCH_DYNAMIC_OBJ_K_THREAD_ALIGNMENT (sizeof(void *))
387 : #endif /* CONFIG_*_FP_SHARING */
388 :
389 :
390 : #ifdef __cplusplus
391 : }
392 : #endif
393 :
394 : #endif /* !_ASMLANGUAGE */
395 :
396 : #endif /* ZEPHYR_INCLUDE_ARCH_X86_IA32_ARCH_H_ */
|
