Line data Source code
1 1 : /* 2 : * Copyright (c) 2015 Wind River Systems, Inc. 3 : * Copyright (c) 2022 Carlo Caione <ccaione@baylibre.com> 4 : * 5 : * SPDX-License-Identifier: Apache-2.0 6 : */ 7 : 8 : #ifndef ZEPHYR_INCLUDE_CACHE_H_ 9 : #define ZEPHYR_INCLUDE_CACHE_H_ 10 : 11 : /** 12 : * @file 13 : * @brief cache API interface 14 : */ 15 : 16 : #include <zephyr/kernel.h> 17 : #include <zephyr/arch/cpu.h> 18 : #include <zephyr/debug/sparse.h> 19 : 20 : #ifdef __cplusplus 21 : extern "C" { 22 : #endif 23 : 24 : #if defined(CONFIG_EXTERNAL_CACHE) 25 : #include <zephyr/drivers/cache.h> 26 : 27 : #elif defined(CONFIG_ARCH_CACHE) 28 : #include <zephyr/arch/cache.h> 29 : 30 : #endif 31 : 32 : /** 33 : * @defgroup cache_interface Cache Interface 34 : * @ingroup os_services 35 : * @{ 36 : */ 37 : 38 : /** 39 : * @cond INTERNAL_HIDDEN 40 : * 41 : */ 42 : 43 : #define _CPU DT_PATH(cpus, cpu_0) 44 : 45 : /** @endcond */ 46 : 47 : /** 48 : * @brief Enable the d-cache 49 : * 50 : * Enable the data cache 51 : * 52 : */ 53 1 : static ALWAYS_INLINE void sys_cache_data_enable(void) 54 : { 55 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 56 : cache_data_enable(); 57 : #endif 58 : } 59 : 60 : /** 61 : * @brief Disable the d-cache 62 : * 63 : * Disable the data cache 64 : * 65 : */ 66 1 : static ALWAYS_INLINE void sys_cache_data_disable(void) 67 : { 68 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 69 : cache_data_disable(); 70 : #endif 71 : } 72 : 73 : /** 74 : * @brief Enable the i-cache 75 : * 76 : * Enable the instruction cache 77 : * 78 : */ 79 1 : static ALWAYS_INLINE void sys_cache_instr_enable(void) 80 : { 81 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 82 : cache_instr_enable(); 83 : #endif 84 : } 85 : 86 : /** 87 : * @brief Disable the i-cache 88 : * 89 : * Disable the instruction cache 90 : * 91 : */ 92 1 : static ALWAYS_INLINE void sys_cache_instr_disable(void) 93 : { 94 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 95 : cache_instr_disable(); 96 : #endif 97 : } 98 : 99 : /** 100 : * @brief Flush the d-cache 101 : * 102 : * Flush the whole data cache. 103 : * 104 : * @retval 0 If succeeded. 105 : * @retval -ENOTSUP If not supported. 106 : * @retval -errno Negative errno for other failures. 107 : */ 108 1 : static ALWAYS_INLINE int sys_cache_data_flush_all(void) 109 : { 110 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 111 : return cache_data_flush_all(); 112 : #endif 113 : return -ENOTSUP; 114 : } 115 : 116 : /** 117 : * @brief Flush the i-cache 118 : * 119 : * Flush the whole instruction cache. 120 : * 121 : * @retval 0 If succeeded. 122 : * @retval -ENOTSUP If not supported. 123 : * @retval -errno Negative errno for other failures. 124 : */ 125 1 : static ALWAYS_INLINE int sys_cache_instr_flush_all(void) 126 : { 127 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 128 : return cache_instr_flush_all(); 129 : #endif 130 : return -ENOTSUP; 131 : } 132 : 133 : /** 134 : * @brief Invalidate the d-cache 135 : * 136 : * Invalidate the whole data cache. 137 : * 138 : * @retval 0 If succeeded. 139 : * @retval -ENOTSUP If not supported. 140 : * @retval -errno Negative errno for other failures. 141 : */ 142 1 : static ALWAYS_INLINE int sys_cache_data_invd_all(void) 143 : { 144 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 145 : return cache_data_invd_all(); 146 : #endif 147 : return -ENOTSUP; 148 : } 149 : 150 : /** 151 : * @brief Invalidate the i-cache 152 : * 153 : * Invalidate the whole instruction cache. 154 : * 155 : * @retval 0 If succeeded. 156 : * @retval -ENOTSUP If not supported. 157 : * @retval -errno Negative errno for other failures. 158 : */ 159 1 : static ALWAYS_INLINE int sys_cache_instr_invd_all(void) 160 : { 161 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 162 : return cache_instr_invd_all(); 163 : #endif 164 : return -ENOTSUP; 165 : } 166 : 167 : /** 168 : * @brief Flush and Invalidate the d-cache 169 : * 170 : * Flush and Invalidate the whole data cache. 171 : * 172 : * @retval 0 If succeeded. 173 : * @retval -ENOTSUP If not supported. 174 : * @retval -errno Negative errno for other failures. 175 : */ 176 1 : static ALWAYS_INLINE int sys_cache_data_flush_and_invd_all(void) 177 : { 178 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 179 : return cache_data_flush_and_invd_all(); 180 : #endif 181 : return -ENOTSUP; 182 : } 183 : 184 : /** 185 : * @brief Flush and Invalidate the i-cache 186 : * 187 : * Flush and Invalidate the whole instruction cache. 188 : * 189 : * @retval 0 If succeeded. 190 : * @retval -ENOTSUP If not supported. 191 : * @retval -errno Negative errno for other failures. 192 : */ 193 1 : static ALWAYS_INLINE int sys_cache_instr_flush_and_invd_all(void) 194 : { 195 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 196 : return cache_instr_flush_and_invd_all(); 197 : #endif 198 : return -ENOTSUP; 199 : } 200 : 201 : /** 202 : * @brief Flush an address range in the d-cache 203 : * 204 : * Flush the specified address range of the data cache. 205 : * 206 : * @note the cache operations act on cache line. When multiple data structures 207 : * share the same cache line being flushed, all the portions of the 208 : * data structures sharing the same line will be flushed. This is usually 209 : * not a problem because writing back is a non-destructive process that 210 : * could be triggered by hardware at any time, so having an aligned 211 : * @p addr or a padded @p size is not strictly necessary. 212 : * 213 : * @param addr Starting address to flush. 214 : * @param size Range size. 215 : * 216 : * @retval 0 If succeeded. 217 : * @retval -ENOTSUP If not supported. 218 : * @retval -errno Negative errno for other failures. 219 : */ 220 1 : __syscall_always_inline int sys_cache_data_flush_range(void *addr, size_t size); 221 : 222 : static ALWAYS_INLINE int z_impl_sys_cache_data_flush_range(void *addr, size_t size) 223 : { 224 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 225 : return cache_data_flush_range(addr, size); 226 : #endif 227 : ARG_UNUSED(addr); 228 : ARG_UNUSED(size); 229 : 230 : return -ENOTSUP; 231 : } 232 : 233 : /** 234 : * @brief Flush an address range in the i-cache 235 : * 236 : * Flush the specified address range of the instruction cache. 237 : * 238 : * @note the cache operations act on cache line. When multiple data structures 239 : * share the same cache line being flushed, all the portions of the 240 : * data structures sharing the same line will be flushed. This is usually 241 : * not a problem because writing back is a non-destructive process that 242 : * could be triggered by hardware at any time, so having an aligned 243 : * @p addr or a padded @p size is not strictly necessary. 244 : * 245 : * @param addr Starting address to flush. 246 : * @param size Range size. 247 : * 248 : * @retval 0 If succeeded. 249 : * @retval -ENOTSUP If not supported. 250 : * @retval -errno Negative errno for other failures. 251 : */ 252 1 : static ALWAYS_INLINE int sys_cache_instr_flush_range(void *addr, size_t size) 253 : { 254 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 255 : return cache_instr_flush_range(addr, size); 256 : #endif 257 : ARG_UNUSED(addr); 258 : ARG_UNUSED(size); 259 : 260 : return -ENOTSUP; 261 : } 262 : 263 : /** 264 : * @brief Invalidate an address range in the d-cache 265 : * 266 : * Invalidate the specified address range of the data cache. 267 : * 268 : * @note the cache operations act on cache line. When multiple data structures 269 : * share the same cache line being invalidated, all the portions of the 270 : * non-read-only data structures sharing the same line will be 271 : * invalidated as well. This is a destructive process that could lead to 272 : * data loss and/or corruption. When @p addr is not aligned to the cache 273 : * line and/or @p size is not a multiple of the cache line size the 274 : * behaviour is undefined. 275 : * 276 : * @param addr Starting address to invalidate. 277 : * @param size Range size. 278 : * 279 : * @retval 0 If succeeded. 280 : * @retval -ENOTSUP If not supported. 281 : * @retval -errno Negative errno for other failures. 282 : */ 283 1 : __syscall_always_inline int sys_cache_data_invd_range(void *addr, size_t size); 284 : 285 : static ALWAYS_INLINE int z_impl_sys_cache_data_invd_range(void *addr, size_t size) 286 : { 287 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 288 : return cache_data_invd_range(addr, size); 289 : #endif 290 : ARG_UNUSED(addr); 291 : ARG_UNUSED(size); 292 : 293 : return -ENOTSUP; 294 : } 295 : 296 : /** 297 : * @brief Invalidate an address range in the i-cache 298 : * 299 : * Invalidate the specified address range of the instruction cache. 300 : * 301 : * @note the cache operations act on cache line. When multiple data structures 302 : * share the same cache line being invalidated, all the portions of the 303 : * non-read-only data structures sharing the same line will be 304 : * invalidated as well. This is a destructive process that could lead to 305 : * data loss and/or corruption. When @p addr is not aligned to the cache 306 : * line and/or @p size is not a multiple of the cache line size the 307 : * behaviour is undefined. 308 : * 309 : * @param addr Starting address to invalidate. 310 : * @param size Range size. 311 : * 312 : * @retval 0 If succeeded. 313 : * @retval -ENOTSUP If not supported. 314 : * @retval -errno Negative errno for other failures. 315 : */ 316 1 : static ALWAYS_INLINE int sys_cache_instr_invd_range(void *addr, size_t size) 317 : { 318 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 319 : return cache_instr_invd_range(addr, size); 320 : #endif 321 : ARG_UNUSED(addr); 322 : ARG_UNUSED(size); 323 : 324 : return -ENOTSUP; 325 : } 326 : 327 : /** 328 : * @brief Flush and Invalidate an address range in the d-cache 329 : * 330 : * Flush and Invalidate the specified address range of the data cache. 331 : * 332 : * @note the cache operations act on cache line. When multiple data structures 333 : * share the same cache line being flushed, all the portions of the 334 : * data structures sharing the same line will be flushed before being 335 : * invalidated. This is usually not a problem because writing back is a 336 : * non-destructive process that could be triggered by hardware at any 337 : * time, so having an aligned @p addr or a padded @p size is not strictly 338 : * necessary. 339 : * 340 : * @param addr Starting address to flush and invalidate. 341 : * @param size Range size. 342 : * 343 : * @retval 0 If succeeded. 344 : * @retval -ENOTSUP If not supported. 345 : * @retval -errno Negative errno for other failures. 346 : */ 347 1 : __syscall_always_inline int sys_cache_data_flush_and_invd_range(void *addr, size_t size); 348 : 349 : static ALWAYS_INLINE int z_impl_sys_cache_data_flush_and_invd_range(void *addr, size_t size) 350 : { 351 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE) 352 : return cache_data_flush_and_invd_range(addr, size); 353 : #endif 354 : ARG_UNUSED(addr); 355 : ARG_UNUSED(size); 356 : 357 : return -ENOTSUP; 358 : } 359 : 360 : /** 361 : * @brief Flush and Invalidate an address range in the i-cache 362 : * 363 : * Flush and Invalidate the specified address range of the instruction cache. 364 : * 365 : * @note the cache operations act on cache line. When multiple data structures 366 : * share the same cache line being flushed, all the portions of the 367 : * data structures sharing the same line will be flushed before being 368 : * invalidated. This is usually not a problem because writing back is a 369 : * non-destructive process that could be triggered by hardware at any 370 : * time, so having an aligned @p addr or a padded @p size is not strictly 371 : * necessary. 372 : * 373 : * @param addr Starting address to flush and invalidate. 374 : * @param size Range size. 375 : * 376 : * @retval 0 If succeeded. 377 : * @retval -ENOTSUP If not supported. 378 : * @retval -errno Negative errno for other failures. 379 : */ 380 1 : static ALWAYS_INLINE int sys_cache_instr_flush_and_invd_range(void *addr, size_t size) 381 : { 382 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE) 383 : return cache_instr_flush_and_invd_range(addr, size); 384 : #endif 385 : ARG_UNUSED(addr); 386 : ARG_UNUSED(size); 387 : 388 : return -ENOTSUP; 389 : } 390 : 391 : /** 392 : * 393 : * @brief Get the d-cache line size. 394 : * 395 : * The API is provided to get the data cache line. 396 : * 397 : * The cache line size is calculated (in order of priority): 398 : * 399 : * - At run-time when @kconfig{CONFIG_DCACHE_LINE_SIZE_DETECT} is set. 400 : * - At compile time using the value set in @kconfig{CONFIG_DCACHE_LINE_SIZE}. 401 : * - At compile time using the `d-cache-line-size` CPU0 property of the DT. 402 : * - 0 otherwise 403 : * 404 : * @retval size Size of the d-cache line. 405 : * @retval 0 If the d-cache is not enabled. 406 : */ 407 1 : static ALWAYS_INLINE size_t sys_cache_data_line_size_get(void) 408 : { 409 : #ifdef CONFIG_DCACHE_LINE_SIZE_DETECT 410 : return cache_data_line_size_get(); 411 : #elif (CONFIG_DCACHE_LINE_SIZE != 0) 412 : return CONFIG_DCACHE_LINE_SIZE; 413 : #else 414 : return DT_PROP_OR(_CPU, d_cache_line_size, 0); 415 : #endif 416 : } 417 : 418 : /** 419 : * 420 : * @brief Get the i-cache line size. 421 : * 422 : * The API is provided to get the instruction cache line. 423 : * 424 : * The cache line size is calculated (in order of priority): 425 : * 426 : * - At run-time when @kconfig{CONFIG_ICACHE_LINE_SIZE_DETECT} is set. 427 : * - At compile time using the value set in @kconfig{CONFIG_ICACHE_LINE_SIZE}. 428 : * - At compile time using the `i-cache-line-size` CPU0 property of the DT. 429 : * - 0 otherwise 430 : * 431 : * @retval size Size of the d-cache line. 432 : * @retval 0 If the d-cache is not enabled. 433 : */ 434 1 : static ALWAYS_INLINE size_t sys_cache_instr_line_size_get(void) 435 : { 436 : #ifdef CONFIG_ICACHE_LINE_SIZE_DETECT 437 : return cache_instr_line_size_get(); 438 : #elif (CONFIG_ICACHE_LINE_SIZE != 0) 439 : return CONFIG_ICACHE_LINE_SIZE; 440 : #else 441 : return DT_PROP_OR(_CPU, i_cache_line_size, 0); 442 : #endif 443 : } 444 : 445 : /** 446 : * @brief Test if a pointer is in cached region. 447 : * 448 : * Some hardware may map the same physical memory twice 449 : * so that it can be seen in both (incoherent) cached mappings 450 : * and a coherent "shared" area. This tests if a particular 451 : * pointer is within the cached, coherent area. 452 : * 453 : * @param ptr Pointer 454 : * 455 : * @retval True if pointer is in cached region. 456 : * @retval False if pointer is not in cached region. 457 : */ 458 1 : static ALWAYS_INLINE bool sys_cache_is_ptr_cached(void *ptr) 459 : { 460 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) 461 : return cache_is_ptr_cached(ptr); 462 : #else 463 : ARG_UNUSED(ptr); 464 : 465 : return false; 466 : #endif 467 : } 468 : 469 : /** 470 : * @brief Test if a pointer is in un-cached region. 471 : * 472 : * Some hardware may map the same physical memory twice 473 : * so that it can be seen in both (incoherent) cached mappings 474 : * and a coherent "shared" area. This tests if a particular 475 : * pointer is within the un-cached, incoherent area. 476 : * 477 : * @param ptr Pointer 478 : * 479 : * @retval True if pointer is not in cached region. 480 : * @retval False if pointer is in cached region. 481 : */ 482 1 : static ALWAYS_INLINE bool sys_cache_is_ptr_uncached(void *ptr) 483 : { 484 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) 485 : return cache_is_ptr_uncached(ptr); 486 : #else 487 : ARG_UNUSED(ptr); 488 : 489 : return false; 490 : #endif 491 : } 492 : 493 : /** 494 : * @brief Return cached pointer to a RAM address 495 : * 496 : * This function takes a pointer to any addressable object (either in 497 : * cacheable memory or not) and returns a pointer that can be used to 498 : * refer to the same memory through the L1 data cache. Data read 499 : * through the resulting pointer will reflect locally cached values on 500 : * the current CPU if they exist, and writes will go first into the 501 : * cache and be written back later. 502 : * 503 : * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not 504 : * enabled. 505 : * 506 : * @see arch_uncached_ptr() 507 : * 508 : * @param ptr A pointer to a valid C object 509 : * @return A pointer to the same object via the L1 dcache 510 : */ 511 1 : static ALWAYS_INLINE void __sparse_cache *sys_cache_cached_ptr_get(void *ptr) 512 : { 513 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) 514 : return cache_cached_ptr(ptr); 515 : #else 516 : return (__sparse_force void __sparse_cache *)ptr; 517 : #endif 518 : } 519 : 520 : /** 521 : * @brief Return uncached pointer to a RAM address 522 : * 523 : * This function takes a pointer to any addressable object (either in 524 : * cacheable memory or not) and returns a pointer that can be used to 525 : * refer to the same memory while bypassing the L1 data cache. Data 526 : * in the L1 cache will not be inspected nor modified by the access. 527 : * 528 : * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not 529 : * enabled. 530 : * 531 : * @see arch_cached_ptr() 532 : * 533 : * @param ptr A pointer to a valid C object 534 : * @return A pointer to the same object bypassing the L1 dcache 535 : */ 536 1 : static ALWAYS_INLINE void *sys_cache_uncached_ptr_get(void __sparse_cache *ptr) 537 : { 538 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP) 539 : return cache_uncached_ptr(ptr); 540 : #else 541 : return (__sparse_force void *)ptr; 542 : #endif 543 : } 544 : 545 : 546 : #ifdef CONFIG_LIBMETAL 547 : static ALWAYS_INLINE void sys_cache_flush(void *addr, size_t size) 548 : { 549 : sys_cache_data_flush_range(addr, size); 550 : } 551 : #endif 552 : 553 : #include <zephyr/syscalls/cache.h> 554 : #ifdef __cplusplus 555 : } 556 : #endif 557 : 558 : /** 559 : * @} 560 : */ 561 : 562 : #endif /* ZEPHYR_INCLUDE_CACHE_H_ */
