Line data Source code
1 0 : /* 2 : * Copyright (c) 2020 Intel Corporation 3 : * 4 : * SPDX-License-Identifier: Apache-2.0 5 : */ 6 : 7 : #ifndef ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H 8 : #define ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H 9 : 10 : #include <zephyr/kernel/mm.h> 11 : 12 : #include <zephyr/sys/util.h> 13 : #include <zephyr/toolchain.h> 14 : 15 : /** 16 : * @defgroup demand_paging Demand Paging 17 : * @ingroup kernel_memory_management 18 : */ 19 : 20 : /** 21 : * @defgroup mem-demand-paging Demand Paging APIs 22 : * @ingroup demand_paging 23 : * @{ 24 : */ 25 : 26 : #ifndef _ASMLANGUAGE 27 : #include <stdint.h> 28 : #include <stddef.h> 29 : #include <inttypes.h> 30 : #include <zephyr/sys/__assert.h> 31 : 32 : struct k_mem_page_frame; 33 : 34 : /** 35 : * Paging Statistics. 36 : */ 37 1 : struct k_mem_paging_stats_t { 38 : #if defined(CONFIG_DEMAND_PAGING_STATS) || defined(__DOXYGEN__) 39 : struct { 40 : /** Number of page faults */ 41 1 : unsigned long cnt; 42 : 43 : /** Number of page faults with IRQ locked */ 44 1 : unsigned long irq_locked; 45 : 46 : /** Number of page faults with IRQ unlocked */ 47 1 : unsigned long irq_unlocked; 48 : 49 : #if !defined(CONFIG_DEMAND_PAGING_ALLOW_IRQ) || defined(__DOXYGEN__) 50 : /** Number of page faults while in ISR */ 51 1 : unsigned long in_isr; 52 : #endif /* !CONFIG_DEMAND_PAGING_ALLOW_IRQ */ 53 0 : } pagefaults; 54 : 55 : struct { 56 : /** Number of clean pages selected for eviction */ 57 1 : unsigned long clean; 58 : 59 : /** Number of dirty pages selected for eviction */ 60 1 : unsigned long dirty; 61 0 : } eviction; 62 : #endif /* CONFIG_DEMAND_PAGING_STATS */ 63 : }; 64 : 65 : /** 66 : * Paging Statistics Histograms. 67 : */ 68 1 : struct k_mem_paging_histogram_t { 69 : #if defined(CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM) || defined(__DOXYGEN__) 70 : /* Counts for each bin in timing histogram */ 71 0 : unsigned long counts[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS]; 72 : 73 : /* Bounds for the bins in timing histogram, 74 : * excluding the first and last (hence, NUM_SLOTS - 1). 75 : */ 76 0 : unsigned long bounds[CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM_NUM_BINS]; 77 : #endif /* CONFIG_DEMAND_PAGING_TIMING_HISTOGRAM */ 78 : }; 79 : 80 : #ifdef __cplusplus 81 : extern "C" { 82 : #endif 83 : 84 : /** 85 : * Evict a page-aligned virtual memory region to the backing store 86 : * 87 : * Useful if it is known that a memory region will not be used for some time. 88 : * All the data pages within the specified region will be evicted to the 89 : * backing store if they weren't already, with their associated page frames 90 : * marked as available for mappings or page-ins. 91 : * 92 : * None of the associated page frames mapped to the provided region should 93 : * be pinned. 94 : * 95 : * Note that there are no guarantees how long these pages will be evicted, 96 : * they could take page faults immediately. 97 : * 98 : * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be 99 : * called by ISRs as the backing store may be in-use. 100 : * 101 : * @param addr Base page-aligned virtual address 102 : * @param size Page-aligned data region size 103 : * @retval 0 Success 104 : * @retval -ENOMEM Insufficient space in backing store to satisfy request. 105 : * The region may be partially paged out. 106 : */ 107 1 : int k_mem_page_out(void *addr, size_t size); 108 : 109 : /** 110 : * Load a virtual data region into memory 111 : * 112 : * After the function completes, all the page frames associated with this 113 : * function will be paged in. However, they are not guaranteed to stay there. 114 : * This is useful if the region is known to be used soon. 115 : * 116 : * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be 117 : * called by ISRs as the backing store may be in-use. 118 : * 119 : * @param addr Base page-aligned virtual address 120 : * @param size Page-aligned data region size 121 : */ 122 1 : void k_mem_page_in(void *addr, size_t size); 123 : 124 : /** 125 : * Pin an aligned virtual data region, paging in as necessary 126 : * 127 : * After the function completes, all the page frames associated with this 128 : * region will be resident in memory and pinned such that they stay that way. 129 : * This is a stronger version of z_mem_page_in(). 130 : * 131 : * If CONFIG_DEMAND_PAGING_ALLOW_IRQ is enabled, this function may not be 132 : * called by ISRs as the backing store may be in-use. 133 : * 134 : * @param addr Base page-aligned virtual address 135 : * @param size Page-aligned data region size 136 : */ 137 1 : void k_mem_pin(void *addr, size_t size); 138 : 139 : /** 140 : * Un-pin an aligned virtual data region 141 : * 142 : * After the function completes, all the page frames associated with this 143 : * region will be no longer marked as pinned. This does not evict the region, 144 : * follow this with z_mem_page_out() if you need that. 145 : * 146 : * @param addr Base page-aligned virtual address 147 : * @param size Page-aligned data region size 148 : */ 149 1 : void k_mem_unpin(void *addr, size_t size); 150 : 151 : /** 152 : * Get the paging statistics since system startup 153 : * 154 : * This populates the paging statistics struct being passed in 155 : * as argument. 156 : * 157 : * @param[in,out] stats Paging statistics struct to be filled. 158 : */ 159 1 : __syscall void k_mem_paging_stats_get(struct k_mem_paging_stats_t *stats); 160 : 161 : struct k_thread; 162 : /** 163 : * Get the paging statistics since system startup for a thread 164 : * 165 : * This populates the paging statistics struct being passed in 166 : * as argument for a particular thread. 167 : * 168 : * @param[in] thread Thread 169 : * @param[in,out] stats Paging statistics struct to be filled. 170 : */ 171 : __syscall 172 1 : void k_mem_paging_thread_stats_get(struct k_thread *thread, 173 : struct k_mem_paging_stats_t *stats); 174 : 175 : /** 176 : * Get the eviction timing histogram 177 : * 178 : * This populates the timing histogram struct being passed in 179 : * as argument. 180 : * 181 : * @param[in,out] hist Timing histogram struct to be filled. 182 : */ 183 1 : __syscall void k_mem_paging_histogram_eviction_get( 184 : struct k_mem_paging_histogram_t *hist); 185 : 186 : /** 187 : * Get the backing store page-in timing histogram 188 : * 189 : * This populates the timing histogram struct being passed in 190 : * as argument. 191 : * 192 : * @param[in,out] hist Timing histogram struct to be filled. 193 : */ 194 1 : __syscall void k_mem_paging_histogram_backing_store_page_in_get( 195 : struct k_mem_paging_histogram_t *hist); 196 : 197 : /** 198 : * Get the backing store page-out timing histogram 199 : * 200 : * This populates the timing histogram struct being passed in 201 : * as argument. 202 : * 203 : * @param[in,out] hist Timing histogram struct to be filled. 204 : */ 205 1 : __syscall void k_mem_paging_histogram_backing_store_page_out_get( 206 : struct k_mem_paging_histogram_t *hist); 207 : 208 : #include <zephyr/syscalls/demand_paging.h> 209 : 210 : /** @} */ 211 : 212 : /** 213 : * Eviction algorithm APIs 214 : * 215 : * @defgroup mem-demand-paging-eviction Eviction Algorithm APIs 216 : * @ingroup demand_paging 217 : * @{ 218 : */ 219 : 220 : #if defined(CONFIG_EVICTION_TRACKING) || defined(__DOXYGEN__) 221 : 222 : /** 223 : * Submit a page frame for eviction candidate tracking 224 : * 225 : * The kernel will invoke this to tell the eviction algorithm the provided 226 : * page frame may be considered as a potential eviction candidate. 227 : * 228 : * This function will never be called before the initial 229 : * k_mem_paging_eviction_init(). 230 : * 231 : * This function is invoked with interrupts locked. 232 : * 233 : * @param [in] pf The page frame to add 234 : */ 235 1 : void k_mem_paging_eviction_add(struct k_mem_page_frame *pf); 236 : 237 : /** 238 : * Remove a page frame from potential eviction candidates 239 : * 240 : * The kernel will invoke this to tell the eviction algorithm the provided 241 : * page frame may no longer be considered as a potential eviction candidate. 242 : * 243 : * This function will only be called with page frames that were submitted 244 : * using k_mem_paging_eviction_add() beforehand. 245 : * 246 : * This function is invoked with interrupts locked. 247 : * 248 : * @param [in] pf The page frame to remove 249 : */ 250 1 : void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf); 251 : 252 : /** 253 : * Process a page frame as being newly accessed 254 : * 255 : * The architecture-specific memory fault handler will invoke this to tell the 256 : * eviction algorithm the provided physical address belongs to a page frame 257 : * being accessed and such page frame should become unlikely to be 258 : * considered as the next eviction candidate. 259 : * 260 : * This function is invoked with interrupts locked. 261 : * 262 : * @param [in] phys The physical address being accessed 263 : */ 264 1 : void k_mem_paging_eviction_accessed(uintptr_t phys); 265 : 266 : #else /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */ 267 : 268 : static inline void k_mem_paging_eviction_add(struct k_mem_page_frame *pf) 269 : { 270 : ARG_UNUSED(pf); 271 : } 272 : 273 : static inline void k_mem_paging_eviction_remove(struct k_mem_page_frame *pf) 274 : { 275 : ARG_UNUSED(pf); 276 : } 277 : 278 : static inline void k_mem_paging_eviction_accessed(uintptr_t phys) 279 : { 280 : ARG_UNUSED(phys); 281 : } 282 : 283 : #endif /* CONFIG_EVICTION_TRACKING || __DOXYGEN__ */ 284 : 285 : /** 286 : * Select a page frame for eviction 287 : * 288 : * The kernel will invoke this to choose a page frame to evict if there 289 : * are no free page frames. It is not guaranteed that the returned page 290 : * frame will actually be evicted. If it is then the kernel will call 291 : * k_mem_paging_eviction_remove() with it. 292 : * 293 : * This function will never be called before the initial 294 : * k_mem_paging_eviction_init(). 295 : * 296 : * This function is invoked with interrupts locked. 297 : * 298 : * @param [out] dirty Whether the page to evict is dirty 299 : * @return The page frame to evict 300 : */ 301 1 : struct k_mem_page_frame *k_mem_paging_eviction_select(bool *dirty); 302 : 303 : /** 304 : * Initialization function 305 : * 306 : * Called at POST_KERNEL to perform any necessary initialization tasks for the 307 : * eviction algorithm. k_mem_paging_eviction_select() is guaranteed to never be 308 : * called until this has returned, and this will only be called once. 309 : */ 310 1 : void k_mem_paging_eviction_init(void); 311 : 312 : /** @} */ 313 : 314 : /** 315 : * Backing store APIs 316 : * 317 : * @defgroup mem-demand-paging-backing-store Backing Store APIs 318 : * @ingroup demand_paging 319 : * @{ 320 : */ 321 : 322 : /** 323 : * Reserve or fetch a storage location for a data page loaded into a page frame 324 : * 325 : * The returned location token must be unique to the mapped virtual address. 326 : * This location will be used in the backing store to page out data page 327 : * contents for later retrieval. The location value must be page-aligned. 328 : * 329 : * This function may be called multiple times on the same data page. If its 330 : * page frame has its K_MEM_PAGE_FRAME_BACKED bit set, it is expected to return 331 : * the previous backing store location for the data page containing a cached 332 : * clean copy. This clean copy may be updated on page-out, or used to 333 : * discard clean pages without needing to write out their contents. 334 : * 335 : * If the backing store is full, some other backing store location which caches 336 : * a loaded data page may be selected, in which case its associated page frame 337 : * will have the K_MEM_PAGE_FRAME_BACKED bit cleared (as it is no longer cached). 338 : * 339 : * k_mem_page_frame_to_virt(pf) will indicate the virtual address the page is 340 : * currently mapped to. Large, sparse backing stores which can contain the 341 : * entire address space may simply generate location tokens purely as a 342 : * function of that virtual address with no other management necessary. 343 : * 344 : * This function distinguishes whether it was called on behalf of a page 345 : * fault. A free backing store location must always be reserved in order for 346 : * page faults to succeed. If the page_fault parameter is not set, this 347 : * function should return -ENOMEM even if one location is available. 348 : * 349 : * This function is invoked with interrupts locked. 350 : * 351 : * @param pf Virtual address to obtain a storage location 352 : * @param [out] location storage location token 353 : * @param page_fault Whether this request was for a page fault 354 : * @return 0 Success 355 : * @return -ENOMEM Backing store is full 356 : */ 357 1 : int k_mem_paging_backing_store_location_get(struct k_mem_page_frame *pf, 358 : uintptr_t *location, 359 : bool page_fault); 360 : 361 : /** 362 : * Free a backing store location 363 : * 364 : * Any stored data may be discarded, and the location token associated with 365 : * this address may be re-used for some other data page. 366 : * 367 : * This function is invoked with interrupts locked. 368 : * 369 : * @param location Location token to free 370 : */ 371 1 : void k_mem_paging_backing_store_location_free(uintptr_t location); 372 : 373 : /** 374 : * Obtain persistent location token for on-demand content 375 : * 376 : * Unlike k_mem_paging_backing_store_location_get() this does not allocate 377 : * any backing store space. Instead, it returns a location token corresponding 378 : * to some fixed storage content to be paged in on demand. This is expected 379 : * to be used in conjonction with CONFIG_LINKER_USE_ONDEMAND_SECTION and the 380 : * K_MEM_MAP_UNPAGED flag to create demand mappings at boot time. This may 381 : * also be used e.g. to implement file-based mmap(). 382 : * 383 : * @param addr Virtual address to obtain a location token for 384 : * @param [out] location storage location token 385 : * @return 0 for success or negative error code 386 : */ 387 1 : int k_mem_paging_backing_store_location_query(void *addr, uintptr_t *location); 388 : 389 : /** 390 : * Copy a data page from K_MEM_SCRATCH_PAGE to the specified location 391 : * 392 : * Immediately before this is called, K_MEM_SCRATCH_PAGE will be mapped read-write 393 : * to the intended source page frame for the calling context. 394 : * 395 : * Calls to this and k_mem_paging_backing_store_page_in() will always be 396 : * serialized, but interrupts may be enabled. 397 : * 398 : * @param location Location token for the data page, for later retrieval 399 : */ 400 1 : void k_mem_paging_backing_store_page_out(uintptr_t location); 401 : 402 : /** 403 : * Copy a data page from the provided location to K_MEM_SCRATCH_PAGE. 404 : * 405 : * Immediately before this is called, K_MEM_SCRATCH_PAGE will be mapped read-write 406 : * to the intended destination page frame for the calling context. 407 : * 408 : * Calls to this and k_mem_paging_backing_store_page_out() will always be 409 : * serialized, but interrupts may be enabled. 410 : * 411 : * @param location Location token for the data page 412 : */ 413 1 : void k_mem_paging_backing_store_page_in(uintptr_t location); 414 : 415 : /** 416 : * Update internal accounting after a page-in 417 : * 418 : * This is invoked after k_mem_paging_backing_store_page_in() and interrupts 419 : * have been* re-locked, making it safe to access the k_mem_page_frame data. 420 : * The location value will be the same passed to 421 : * k_mem_paging_backing_store_page_in(). 422 : * 423 : * The primary use-case for this is to update custom fields for the backing 424 : * store in the page frame, to reflect where the data should be evicted to 425 : * if it is paged out again. This may be a no-op in some implementations. 426 : * 427 : * If the backing store caches paged-in data pages, this is the appropriate 428 : * time to set the K_MEM_PAGE_FRAME_BACKED bit. The kernel only skips paging 429 : * out clean data pages if they are noted as clean in the page tables and the 430 : * K_MEM_PAGE_FRAME_BACKED bit is set in their associated page frame. 431 : * 432 : * @param pf Page frame that was loaded in 433 : * @param location Location of where the loaded data page was retrieved 434 : */ 435 1 : void k_mem_paging_backing_store_page_finalize(struct k_mem_page_frame *pf, 436 : uintptr_t location); 437 : 438 : /** 439 : * Backing store initialization function. 440 : * 441 : * The implementation may expect to receive page in/out calls as soon as this 442 : * returns, but not before that. Called at POST_KERNEL. 443 : * 444 : * This function is expected to do two things: 445 : * - Initialize any internal data structures and accounting for the backing 446 : * store. 447 : * - If the backing store already contains all or some loaded kernel data pages 448 : * at boot time, K_MEM_PAGE_FRAME_BACKED should be appropriately set for their 449 : * associated page frames, and any internal accounting set up appropriately. 450 : */ 451 1 : void k_mem_paging_backing_store_init(void); 452 : 453 : /** @} */ 454 : 455 : #ifdef __cplusplus 456 : } 457 : #endif 458 : 459 : #endif /* !_ASMLANGUAGE */ 460 : #endif /* ZEPHYR_INCLUDE_KERNEL_MM_DEMAND_PAGING_H */
