Line data Source code
1 0 : /*
2 : * Copyright (c) 2020 Intel Corporation
3 : *
4 : * SPDX-License-Identifier: Apache-2.0
5 : */
6 :
7 : #ifndef ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H
8 : #define ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H
9 :
10 : #include <zephyr/kernel/mm.h>
11 :
12 : #include <zephyr/sys/util.h>
13 : #include <zephyr/toolchain.h>
14 :
15 : /**
16 : * @defgroup demand_paging Demand Paging
17 : * @ingroup kernel_memory_management
18 : */
19 :
20 : /**
21 : * @defgroup mem-demand-paging Demand Paging APIs
22 : * @ingroup demand_paging
23 : * @{
24 : */
25 :
26 : #ifndef _ASMLANGUAGE
27 : #include <stdint.h>
28 : #include <stddef.h>
29 : #include <inttypes.h>
30 : #include <zephyr/sys/__assert.h>
31 :
32 : struct k_mem_page_frame;
33 :
34 : /**
35 : * Paging Statistics.
36 : */
37 1 : struct k_mem_paging_stats_t {
38 : #if defined(CONFIG_DEMAND_PAGING_STATS) || defined(__DOXYGEN__)
39 : struct {
40 : /** Number of page faults */
41 1 : unsigned long cnt;
42 :
43 : /** Number of page faults with IRQ locked */
44 1 : unsigned long irq_locked;
45 :
46 : /** Number of page faults with IRQ unlocked */
47 1 : unsigned long irq_unlocked;
48 :
49 : #if !defined(CONFIG_DEMAND_PAGING_ALLOW_IRQ) || defined(__DOXYGEN__)
50 : /** Number of page faults while in ISR */
51 1 : unsigned long in_isr;
52 : #endif /* !CONFIG_DEMAND_PAGING_ALLOW_IRQ */
53 0 : } pagefaults;
54 :
55 : struct {
56 : /** Number of clean pages selected for eviction */
57 1 : unsigned long clean;
58 :
59 : /** Number of dirty pages selected for eviction */
60 1 : unsigned long dirty;
61 0 : } eviction;
62 : #endif /* CONFIG_DEMAND_PAGING_STATS */
63 : };
64 :
65 : /**
66 : * Paging Statistics Histograms.
67 : */
68 1 : struct k_mem_paging_histogram_t {
69 : #if defined(CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM) || defined(__DOXYGEN__)
70 : /* Counts for each bin in timing histogram */
71 0 : unsigned long counts[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS];
72 :
73 : /* Bounds for the bins in timing histogram,
74 : * excluding the first and last (hence, NUM_SLOTS - 1).
75 : */
76 0 : unsigned long bounds[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS];
77 : #endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */
78 : };
79 :
80 : #ifdef __cplusplus
81 : extern "C" {
82 : #endif
83 :
84 : /**
85 : * Evict a page-aligned virtual memory region to the backing store
86 : *
87 : * Useful if it is known that a memory region will not be used for some time.
88 : * All the data pages within the specified region will be evicted to the
89 : * backing store if they weren't already, with their associated page frames
90 : * marked as available for mappings or page-ins.
91 : *
92 : * None of the associated page frames mapped to the provided region should
93 : * be pinned.
94 : *
95 : * Note that there are no guarantees how long these pages will be evicted,
96 : * they could take page faults immediately.
97 : *
98 : * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be
99 : * called by ISRs as the backing store may be in-use.
100 : *
101 : * @param addr Base page-aligned virtual address
102 : * @param size Page-aligned data region size
103 : * @retval 0 Success
104 : * @retval -ENOMEM Insufficient space in backing store to satisfy request.
105 : * The region may be partially paged out.
106 : */
107 1 : int k_mem_page_out(void *addr, size_t size);
108 :
109 : /**
110 : * Load a virtual data region into memory
111 : *
112 : * After the function completes, all the page frames associated with this
113 : * function will be paged in. However, they are not guaranteed to stay there.
114 : * This is useful if the region is known to be used soon.
115 : *
116 : * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be
117 : * called by ISRs as the backing store may be in-use.
118 : *
119 : * @param addr Base page-aligned virtual address
120 : * @param size Page-aligned data region size
121 : */
122 1 : void k_mem_page_in(void *addr, size_t size);
123 :
124 : /**
125 : * Pin an aligned virtual data region, paging in as necessary
126 : *
127 : * After the function completes, all the page frames associated with this
128 : * region will be resident in memory and pinned such that they stay that way.
129 : * This is a stronger version of z_mem_page_in().
130 : *
131 : * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be
132 : * called by ISRs as the backing store may be in-use.
133 : *
134 : * @param addr Base page-aligned virtual address
135 : * @param size Page-aligned data region size
136 : */
137 1 : void k_mem_pin(void *addr, size_t size);
138 :
139 : /**
140 : * Un-pin an aligned virtual data region
141 : *
142 : * After the function completes, all the page frames associated with this
143 : * region will be no longer marked as pinned. This does not evict the region,
144 : * follow this with z_mem_page_out() if you need that.
145 : *
146 : * @param addr Base page-aligned virtual address
147 : * @param size Page-aligned data region size
148 : */
149 1 : void k_mem_unpin(void *addr, size_t size);
150 :
151 : /**
152 : * Get the paging statistics since system startup
153 : *
154 : * This populates the paging statistics struct being passed in
155 : * as argument.
156 : *
157 : * @param[in,out] stats Paging statistics struct to be filled.
158 : */
159 1 : __syscall void k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats);
160 :
161 : struct k_thread;
162 : /**
163 : * Get the paging statistics since system startup for a thread
164 : *
165 : * This populates the paging statistics struct being passed in
166 : * as argument for a particular thread.
167 : *
168 : * @param[in] thread Thread
169 : * @param[in,out] stats Paging statistics struct to be filled.
170 : */
171 : __syscall
172 1 : void k_mem_paging_thread_stats_get(struct k_thread *thread,
173 : struct k_mem_paging_stats_t *stats);
174 :
175 : /**
176 : * Get the eviction timing histogram
177 : *
178 : * This populates the timing histogram struct being passed in
179 : * as argument.
180 : *
181 : * @param[in,out] hist Timing histogram struct to be filled.
182 : */
183 1 : __syscall void k_mem_paging_histogram_eviction_get(
184 : struct k_mem_paging_histogram_t *hist);
185 :
186 : /**
187 : * Get the backing store page-in timing histogram
188 : *
189 : * This populates the timing histogram struct being passed in
190 : * as argument.
191 : *
192 : * @param[in,out] hist Timing histogram struct to be filled.
193 : */
194 1 : __syscall void k_mem_paging_histogram_backing_store_page_in_get(
195 : struct k_mem_paging_histogram_t *hist);
196 :
197 : /**
198 : * Get the backing store page-out timing histogram
199 : *
200 : * This populates the timing histogram struct being passed in
201 : * as argument.
202 : *
203 : * @param[in,out] hist Timing histogram struct to be filled.
204 : */
205 1 : __syscall void k_mem_paging_histogram_backing_store_page_out_get(
206 : struct k_mem_paging_histogram_t *hist);
207 :
208 : #include <zephyr/syscalls/demand_paging.h>
209 :
210 : /** @} */
211 :
212 : /**
213 : * Eviction algorithm APIs
214 : *
215 : * @defgroup mem-demand-paging-eviction Eviction Algorithm APIs
216 : * @ingroup demand_paging
217 : * @{
218 : */
219 :
220 : #if defined(CONFIG_EVICTION_TRACKING) || defined(__DOXYGEN__)
221 :
222 : /**
223 : * Submit a page frame for eviction candidate tracking
224 : *
225 : * The kernel will invoke this to tell the eviction algorithm the provided
226 : * page frame may be considered as a potential eviction candidate.
227 : *
228 : * This function will never be called before the initial
229 : * k_mem_paging_eviction_init().
230 : *
231 : * This function is invoked with interrupts locked.
232 : *
233 : * @param [in] pf The page frame to add
234 : */
235 1 : void k_mem_paging_eviction_add(struct k_mem_page_frame *pf);
236 :
237 : /**
238 : * Remove a page frame from potential eviction candidates
239 : *
240 : * The kernel will invoke this to tell the eviction algorithm the provided
241 : * page frame may no longer be considered as a potential eviction candidate.
242 : *
243 : * This function will only be called with page frames that were submitted
244 : * using k_mem_paging_eviction_add() beforehand.
245 : *
246 : * This function is invoked with interrupts locked.
247 : *
248 : * @param [in] pf The page frame to remove
249 : */
250 1 : void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf);
251 :
252 : /**
253 : * Process a page frame as being newly accessed
254 : *
255 : * The architecture-specific memory fault handler will invoke this to tell the
256 : * eviction algorithm the provided physical address belongs to a page frame
257 : * being accessed and such page frame should become unlikely to be
258 : * considered as the next eviction candidate.
259 : *
260 : * This function is invoked with interrupts locked.
261 : *
262 : * @param [in] phys The physical address being accessed
263 : */
264 1 : void k_mem_paging_eviction_accessed(uintptr_t phys);
265 :
266 : #else /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */
267 :
268 : static inline void k_mem_paging_eviction_add(struct k_mem_page_frame *pf)
269 : {
270 : ARG_UNUSED(pf);
271 : }
272 :
273 : static inline void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf)
274 : {
275 : ARG_UNUSED(pf);
276 : }
277 :
278 : static inline void k_mem_paging_eviction_accessed(uintptr_t phys)
279 : {
280 : ARG_UNUSED(phys);
281 : }
282 :
283 : #endif /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */
284 :
285 : /**
286 : * Select a page frame for eviction
287 : *
288 : * The kernel will invoke this to choose a page frame to evict if there
289 : * are no free page frames. It is not guaranteed that the returned page
290 : * frame will actually be evicted. If it is then the kernel will call
291 : * k_mem_paging_eviction_remove() with it.
292 : *
293 : * This function will never be called before the initial
294 : * k_mem_paging_eviction_init().
295 : *
296 : * This function is invoked with interrupts locked.
297 : *
298 : * @param [out] dirty Whether the page to evict is dirty
299 : * @return The page frame to evict
300 : */
301 1 : struct k_mem_page_frame *k_mem_paging_eviction_select(bool *dirty);
302 :
303 : /**
304 : * Initialization function
305 : *
306 : * Called at POST_KERNEL to perform any necessary initialization tasks for the
307 : * eviction algorithm. k_mem_paging_eviction_select() is guaranteed to never be
308 : * called until this has returned, and this will only be called once.
309 : */
310 1 : void k_mem_paging_eviction_init(void);
311 :
312 : /** @} */
313 :
314 : /**
315 : * Backing store APIs
316 : *
317 : * @defgroup mem-demand-paging-backing-store Backing Store APIs
318 : * @ingroup demand_paging
319 : * @{
320 : */
321 :
322 : /**
323 : * Reserve or fetch a storage location for a data page loaded into a page frame
324 : *
325 : * The returned location token must be unique to the mapped virtual address.
326 : * This location will be used in the backing store to page out data page
327 : * contents for later retrieval. The location value must be page-aligned.
328 : *
329 : * This function may be called multiple times on the same data page. If its
330 : * page frame has its K_MEM_PAGE_FRAME_BACKED bit set, it is expected to return
331 : * the previous backing store location for the data page containing a cached
332 : * clean copy. This clean copy may be updated on page-out, or used to
333 : * discard clean pages without needing to write out their contents.
334 : *
335 : * If the backing store is full, some other backing store location which caches
336 : * a loaded data page may be selected, in which case its associated page frame
337 : * will have the K_MEM_PAGE_FRAME_BACKED bit cleared (as it is no longer cached).
338 : *
339 : * k_mem_page_frame_to_virt(pf) will indicate the virtual address the page is
340 : * currently mapped to. Large, sparse backing stores which can contain the
341 : * entire address space may simply generate location tokens purely as a
342 : * function of that virtual address with no other management necessary.
343 : *
344 : * This function distinguishes whether it was called on behalf of a page
345 : * fault. A free backing store location must always be reserved in order for
346 : * page faults to succeed. If the page_fault parameter is not set, this
347 : * function should return -ENOMEM even if one location is available.
348 : *
349 : * This function is invoked with interrupts locked.
350 : *
351 : * @param pf Virtual address to obtain a storage location
352 : * @param [out] location storage location token
353 : * @param page_fault Whether this request was for a page fault
354 : * @return 0 Success
355 : * @return -ENOMEM Backing store is full
356 : */
357 1 : int k_mem_paging_backing_store_location_get(struct k_mem_page_frame *pf,
358 : uintptr_t *location,
359 : bool page_fault);
360 :
361 : /**
362 : * Free a backing store location
363 : *
364 : * Any stored data may be discarded, and the location token associated with
365 : * this address may be re-used for some other data page.
366 : *
367 : * This function is invoked with interrupts locked.
368 : *
369 : * @param location Location token to free
370 : */
371 1 : void k_mem_paging_backing_store_location_free(uintptr_t location);
372 :
373 : /**
374 : * Obtain persistent location token for on-demand content
375 : *
376 : * Unlike k_mem_paging_backing_store_location_get() this does not allocate
377 : * any backing store space. Instead, it returns a location token corresponding
378 : * to some fixed storage content to be paged in on demand. This is expected
379 : * to be used in conjonction with CONFIG_LINKER_USE_ONDEMAND_SECTION and the
380 : * K_MEM_MAP_UNPAGED flag to create demand mappings at boot time. This may
381 : * also be used e.g. to implement file-based mmap().
382 : *
383 : * @param addr Virtual address to obtain a location token for
384 : * @param [out] location storage location token
385 : * @return 0 for success or negative error code
386 : */
387 1 : int k_mem_paging_backing_store_location_query(void *addr, uintptr_t *location);
388 :
389 : /**
390 : * Copy a data page from K_MEM_SCRATCH_PAGE to the specified location
391 : *
392 : * Immediately before this is called, K_MEM_SCRATCH_PAGE will be mapped read-write
393 : * to the intended source page frame for the calling context.
394 : *
395 : * Calls to this and k_mem_paging_backing_store_page_in() will always be
396 : * serialized, but interrupts may be enabled.
397 : *
398 : * @param location Location token for the data page, for later retrieval
399 : */
400 1 : void k_mem_paging_backing_store_page_out(uintptr_t location);
401 :
402 : /**
403 : * Copy a data page from the provided location to K_MEM_SCRATCH_PAGE.
404 : *
405 : * Immediately before this is called, K_MEM_SCRATCH_PAGE will be mapped read-write
406 : * to the intended destination page frame for the calling context.
407 : *
408 : * Calls to this and k_mem_paging_backing_store_page_out() will always be
409 : * serialized, but interrupts may be enabled.
410 : *
411 : * @param location Location token for the data page
412 : */
413 1 : void k_mem_paging_backing_store_page_in(uintptr_t location);
414 :
415 : /**
416 : * Update internal accounting after a page-in
417 : *
418 : * This is invoked after k_mem_paging_backing_store_page_in() and interrupts
419 : * have been* re-locked, making it safe to access the k_mem_page_frame data.
420 : * The location value will be the same passed to
421 : * k_mem_paging_backing_store_page_in().
422 : *
423 : * The primary use-case for this is to update custom fields for the backing
424 : * store in the page frame, to reflect where the data should be evicted to
425 : * if it is paged out again. This may be a no-op in some implementations.
426 : *
427 : * If the backing store caches paged-in data pages, this is the appropriate
428 : * time to set the K_MEM_PAGE_FRAME_BACKED bit. The kernel only skips paging
429 : * out clean data pages if they are noted as clean in the page tables and the
430 : * K_MEM_PAGE_FRAME_BACKED bit is set in their associated page frame.
431 : *
432 : * @param pf Page frame that was loaded in
433 : * @param location Location of where the loaded data page was retrieved
434 : */
435 1 : void k_mem_paging_backing_store_page_finalize(struct k_mem_page_frame *pf,
436 : uintptr_t location);
437 :
438 : /**
439 : * Backing store initialization function.
440 : *
441 : * The implementation may expect to receive page in/out calls as soon as this
442 : * returns, but not before that. Called at POST_KERNEL.
443 : *
444 : * This function is expected to do two things:
445 : * - Initialize any internal data structures and accounting for the backing
446 : * store.
447 : * - If the backing store already contains all or some loaded kernel data pages
448 : * at boot time, K_MEM_PAGE_FRAME_BACKED should be appropriately set for their
449 : * associated page frames, and any internal accounting set up appropriately.
450 : */
451 1 : void k_mem_paging_backing_store_init(void);
452 :
453 : /** @} */
454 :
455 : #ifdef __cplusplus
456 : }
457 : #endif
458 :
459 : #endif /* !_ASMLANGUAGE */
460 : #endif /* ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H */
|
