Line data Source code
1 1 : /*
2 : * Copyright (c) 2015 Wind River Systems, Inc.
3 : * Copyright (c) 2022 Carlo Caione <ccaione@baylibre.com>
4 : *
5 : * SPDX-License-Identifier: Apache-2.0
6 : */
7 :
8 : #ifndef ZEPHYR_INCLUDE_CACHE_H_
9 : #define ZEPHYR_INCLUDE_CACHE_H_
10 :
11 : /**
12 : * @file
13 : * @brief cache API interface
14 : */
15 :
16 : #include <zephyr/kernel.h>
17 : #include <zephyr/arch/cpu.h>
18 : #include <zephyr/debug/sparse.h>
19 :
20 : #ifdef __cplusplus
21 : extern "C" {
22 : #endif
23 :
24 : #if defined(CONFIG_EXTERNAL_CACHE)
25 : #include <zephyr/drivers/cache.h>
26 :
27 : #elif defined(CONFIG_ARCH_CACHE)
28 : #include <zephyr/arch/cache.h>
29 :
30 : #endif
31 :
32 : /**
33 : * @defgroup cache_interface Cache Interface
34 : * @ingroup os_services
35 : * @{
36 : */
37 :
38 : /**
39 : * @cond INTERNAL_HIDDEN
40 : *
41 : */
42 :
43 : #define _CPU DT_PATH(cpus, cpu_0)
44 :
45 : /** @endcond */
46 :
47 : /**
48 : * @brief Enable the d-cache
49 : *
50 : * Enable the data cache
51 : *
52 : */
53 1 : static ALWAYS_INLINE void sys_cache_data_enable(void)
54 : {
55 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
56 : cache_data_enable();
57 : #endif
58 : }
59 :
60 : /**
61 : * @brief Disable the d-cache
62 : *
63 : * Disable the data cache
64 : *
65 : */
66 1 : static ALWAYS_INLINE void sys_cache_data_disable(void)
67 : {
68 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
69 : cache_data_disable();
70 : #endif
71 : }
72 :
73 : /**
74 : * @brief Enable the i-cache
75 : *
76 : * Enable the instruction cache
77 : *
78 : */
79 1 : static ALWAYS_INLINE void sys_cache_instr_enable(void)
80 : {
81 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
82 : cache_instr_enable();
83 : #endif
84 : }
85 :
86 : /**
87 : * @brief Disable the i-cache
88 : *
89 : * Disable the instruction cache
90 : *
91 : */
92 1 : static ALWAYS_INLINE void sys_cache_instr_disable(void)
93 : {
94 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
95 : cache_instr_disable();
96 : #endif
97 : }
98 :
99 : /**
100 : * @brief Flush the d-cache
101 : *
102 : * Flush the whole data cache.
103 : *
104 : * @retval 0 If succeeded.
105 : * @retval -ENOTSUP If not supported.
106 : * @retval -errno Negative errno for other failures.
107 : */
108 1 : static ALWAYS_INLINE int sys_cache_data_flush_all(void)
109 : {
110 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
111 : return cache_data_flush_all();
112 : #endif
113 : return -ENOTSUP;
114 : }
115 :
116 : /**
117 : * @brief Flush the i-cache
118 : *
119 : * Flush the whole instruction cache.
120 : *
121 : * @retval 0 If succeeded.
122 : * @retval -ENOTSUP If not supported.
123 : * @retval -errno Negative errno for other failures.
124 : */
125 1 : static ALWAYS_INLINE int sys_cache_instr_flush_all(void)
126 : {
127 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
128 : return cache_instr_flush_all();
129 : #endif
130 : return -ENOTSUP;
131 : }
132 :
133 : /**
134 : * @brief Invalidate the d-cache
135 : *
136 : * Invalidate the whole data cache.
137 : *
138 : * @retval 0 If succeeded.
139 : * @retval -ENOTSUP If not supported.
140 : * @retval -errno Negative errno for other failures.
141 : */
142 1 : static ALWAYS_INLINE int sys_cache_data_invd_all(void)
143 : {
144 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
145 : return cache_data_invd_all();
146 : #endif
147 : return -ENOTSUP;
148 : }
149 :
150 : /**
151 : * @brief Invalidate the i-cache
152 : *
153 : * Invalidate the whole instruction cache.
154 : *
155 : * @retval 0 If succeeded.
156 : * @retval -ENOTSUP If not supported.
157 : * @retval -errno Negative errno for other failures.
158 : */
159 1 : static ALWAYS_INLINE int sys_cache_instr_invd_all(void)
160 : {
161 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
162 : return cache_instr_invd_all();
163 : #endif
164 : return -ENOTSUP;
165 : }
166 :
167 : /**
168 : * @brief Flush and Invalidate the d-cache
169 : *
170 : * Flush and Invalidate the whole data cache.
171 : *
172 : * @retval 0 If succeeded.
173 : * @retval -ENOTSUP If not supported.
174 : * @retval -errno Negative errno for other failures.
175 : */
176 1 : static ALWAYS_INLINE int sys_cache_data_flush_and_invd_all(void)
177 : {
178 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
179 : return cache_data_flush_and_invd_all();
180 : #endif
181 : return -ENOTSUP;
182 : }
183 :
184 : /**
185 : * @brief Flush and Invalidate the i-cache
186 : *
187 : * Flush and Invalidate the whole instruction cache.
188 : *
189 : * @retval 0 If succeeded.
190 : * @retval -ENOTSUP If not supported.
191 : * @retval -errno Negative errno for other failures.
192 : */
193 1 : static ALWAYS_INLINE int sys_cache_instr_flush_and_invd_all(void)
194 : {
195 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
196 : return cache_instr_flush_and_invd_all();
197 : #endif
198 : return -ENOTSUP;
199 : }
200 :
201 : /**
202 : * @brief Flush an address range in the d-cache
203 : *
204 : * Flush the specified address range of the data cache.
205 : *
206 : * @note the cache operations act on cache line. When multiple data structures
207 : * share the same cache line being flushed, all the portions of the
208 : * data structures sharing the same line will be flushed. This is usually
209 : * not a problem because writing back is a non-destructive process that
210 : * could be triggered by hardware at any time, so having an aligned
211 : * @p addr or a padded @p size is not strictly necessary.
212 : *
213 : * @param addr Starting address to flush.
214 : * @param size Range size.
215 : *
216 : * @retval 0 If succeeded.
217 : * @retval -ENOTSUP If not supported.
218 : * @retval -errno Negative errno for other failures.
219 : */
220 1 : __syscall_always_inline int sys_cache_data_flush_range(void *addr, size_t size);
221 :
222 : static ALWAYS_INLINE int z_impl_sys_cache_data_flush_range(void *addr, size_t size)
223 : {
224 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
225 : return cache_data_flush_range(addr, size);
226 : #endif
227 : ARG_UNUSED(addr);
228 : ARG_UNUSED(size);
229 :
230 : return -ENOTSUP;
231 : }
232 :
233 : /**
234 : * @brief Flush an address range in the i-cache
235 : *
236 : * Flush the specified address range of the instruction cache.
237 : *
238 : * @note the cache operations act on cache line. When multiple data structures
239 : * share the same cache line being flushed, all the portions of the
240 : * data structures sharing the same line will be flushed. This is usually
241 : * not a problem because writing back is a non-destructive process that
242 : * could be triggered by hardware at any time, so having an aligned
243 : * @p addr or a padded @p size is not strictly necessary.
244 : *
245 : * @param addr Starting address to flush.
246 : * @param size Range size.
247 : *
248 : * @retval 0 If succeeded.
249 : * @retval -ENOTSUP If not supported.
250 : * @retval -errno Negative errno for other failures.
251 : */
252 1 : static ALWAYS_INLINE int sys_cache_instr_flush_range(void *addr, size_t size)
253 : {
254 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
255 : return cache_instr_flush_range(addr, size);
256 : #endif
257 : ARG_UNUSED(addr);
258 : ARG_UNUSED(size);
259 :
260 : return -ENOTSUP;
261 : }
262 :
263 : /**
264 : * @brief Invalidate an address range in the d-cache
265 : *
266 : * Invalidate the specified address range of the data cache.
267 : *
268 : * @note the cache operations act on cache line. When multiple data structures
269 : * share the same cache line being invalidated, all the portions of the
270 : * non-read-only data structures sharing the same line will be
271 : * invalidated as well. This is a destructive process that could lead to
272 : * data loss and/or corruption. When @p addr is not aligned to the cache
273 : * line and/or @p size is not a multiple of the cache line size the
274 : * behaviour is undefined.
275 : *
276 : * @param addr Starting address to invalidate.
277 : * @param size Range size.
278 : *
279 : * @retval 0 If succeeded.
280 : * @retval -ENOTSUP If not supported.
281 : * @retval -errno Negative errno for other failures.
282 : */
283 1 : __syscall_always_inline int sys_cache_data_invd_range(void *addr, size_t size);
284 :
285 : static ALWAYS_INLINE int z_impl_sys_cache_data_invd_range(void *addr, size_t size)
286 : {
287 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
288 : return cache_data_invd_range(addr, size);
289 : #endif
290 : ARG_UNUSED(addr);
291 : ARG_UNUSED(size);
292 :
293 : return -ENOTSUP;
294 : }
295 :
296 : /**
297 : * @brief Invalidate an address range in the i-cache
298 : *
299 : * Invalidate the specified address range of the instruction cache.
300 : *
301 : * @note the cache operations act on cache line. When multiple data structures
302 : * share the same cache line being invalidated, all the portions of the
303 : * non-read-only data structures sharing the same line will be
304 : * invalidated as well. This is a destructive process that could lead to
305 : * data loss and/or corruption. When @p addr is not aligned to the cache
306 : * line and/or @p size is not a multiple of the cache line size the
307 : * behaviour is undefined.
308 : *
309 : * @param addr Starting address to invalidate.
310 : * @param size Range size.
311 : *
312 : * @retval 0 If succeeded.
313 : * @retval -ENOTSUP If not supported.
314 : * @retval -errno Negative errno for other failures.
315 : */
316 1 : static ALWAYS_INLINE int sys_cache_instr_invd_range(void *addr, size_t size)
317 : {
318 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
319 : return cache_instr_invd_range(addr, size);
320 : #endif
321 : ARG_UNUSED(addr);
322 : ARG_UNUSED(size);
323 :
324 : return -ENOTSUP;
325 : }
326 :
327 : /**
328 : * @brief Flush and Invalidate an address range in the d-cache
329 : *
330 : * Flush and Invalidate the specified address range of the data cache.
331 : *
332 : * @note the cache operations act on cache line. When multiple data structures
333 : * share the same cache line being flushed, all the portions of the
334 : * data structures sharing the same line will be flushed before being
335 : * invalidated. This is usually not a problem because writing back is a
336 : * non-destructive process that could be triggered by hardware at any
337 : * time, so having an aligned @p addr or a padded @p size is not strictly
338 : * necessary.
339 : *
340 : * @param addr Starting address to flush and invalidate.
341 : * @param size Range size.
342 : *
343 : * @retval 0 If succeeded.
344 : * @retval -ENOTSUP If not supported.
345 : * @retval -errno Negative errno for other failures.
346 : */
347 1 : __syscall_always_inline int sys_cache_data_flush_and_invd_range(void *addr, size_t size);
348 :
349 : static ALWAYS_INLINE int z_impl_sys_cache_data_flush_and_invd_range(void *addr, size_t size)
350 : {
351 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_DCACHE)
352 : return cache_data_flush_and_invd_range(addr, size);
353 : #endif
354 : ARG_UNUSED(addr);
355 : ARG_UNUSED(size);
356 :
357 : return -ENOTSUP;
358 : }
359 :
360 : /**
361 : * @brief Flush and Invalidate an address range in the i-cache
362 : *
363 : * Flush and Invalidate the specified address range of the instruction cache.
364 : *
365 : * @note the cache operations act on cache line. When multiple data structures
366 : * share the same cache line being flushed, all the portions of the
367 : * data structures sharing the same line will be flushed before being
368 : * invalidated. This is usually not a problem because writing back is a
369 : * non-destructive process that could be triggered by hardware at any
370 : * time, so having an aligned @p addr or a padded @p size is not strictly
371 : * necessary.
372 : *
373 : * @param addr Starting address to flush and invalidate.
374 : * @param size Range size.
375 : *
376 : * @retval 0 If succeeded.
377 : * @retval -ENOTSUP If not supported.
378 : * @retval -errno Negative errno for other failures.
379 : */
380 1 : static ALWAYS_INLINE int sys_cache_instr_flush_and_invd_range(void *addr, size_t size)
381 : {
382 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_ICACHE)
383 : return cache_instr_flush_and_invd_range(addr, size);
384 : #endif
385 : ARG_UNUSED(addr);
386 : ARG_UNUSED(size);
387 :
388 : return -ENOTSUP;
389 : }
390 :
391 : /**
392 : *
393 : * @brief Get the d-cache line size.
394 : *
395 : * The API is provided to get the data cache line.
396 : *
397 : * The cache line size is calculated (in order of priority):
398 : *
399 : * - At run-time when @kconfig{CONFIG_DCACHE_LINE_SIZE_DETECT} is set.
400 : * - At compile time using the value set in @kconfig{CONFIG_DCACHE_LINE_SIZE}.
401 : * - At compile time using the `d-cache-line-size` CPU0 property of the DT.
402 : * - 0 otherwise
403 : *
404 : * @retval size Size of the d-cache line.
405 : * @retval 0 If the d-cache is not enabled.
406 : */
407 1 : static ALWAYS_INLINE size_t sys_cache_data_line_size_get(void)
408 : {
409 : #ifdef CONFIG_DCACHE_LINE_SIZE_DETECT
410 : return cache_data_line_size_get();
411 : #elif (CONFIG_DCACHE_LINE_SIZE != 0)
412 : return CONFIG_DCACHE_LINE_SIZE;
413 : #else
414 : return DT_PROP_OR(_CPU, d_cache_line_size, 0);
415 : #endif
416 : }
417 :
418 : /**
419 : *
420 : * @brief Get the i-cache line size.
421 : *
422 : * The API is provided to get the instruction cache line.
423 : *
424 : * The cache line size is calculated (in order of priority):
425 : *
426 : * - At run-time when @kconfig{CONFIG_ICACHE_LINE_SIZE_DETECT} is set.
427 : * - At compile time using the value set in @kconfig{CONFIG_ICACHE_LINE_SIZE}.
428 : * - At compile time using the `i-cache-line-size` CPU0 property of the DT.
429 : * - 0 otherwise
430 : *
431 : * @retval size Size of the d-cache line.
432 : * @retval 0 If the d-cache is not enabled.
433 : */
434 1 : static ALWAYS_INLINE size_t sys_cache_instr_line_size_get(void)
435 : {
436 : #ifdef CONFIG_ICACHE_LINE_SIZE_DETECT
437 : return cache_instr_line_size_get();
438 : #elif (CONFIG_ICACHE_LINE_SIZE != 0)
439 : return CONFIG_ICACHE_LINE_SIZE;
440 : #else
441 : return DT_PROP_OR(_CPU, i_cache_line_size, 0);
442 : #endif
443 : }
444 :
445 : /**
446 : * @brief Test if a pointer is in cached region.
447 : *
448 : * Some hardware may map the same physical memory twice
449 : * so that it can be seen in both (incoherent) cached mappings
450 : * and a coherent "shared" area. This tests if a particular
451 : * pointer is within the cached, coherent area.
452 : *
453 : * @param ptr Pointer
454 : *
455 : * @retval True if pointer is in cached region.
456 : * @retval False if pointer is not in cached region.
457 : */
458 1 : static ALWAYS_INLINE bool sys_cache_is_ptr_cached(void *ptr)
459 : {
460 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
461 : return cache_is_ptr_cached(ptr);
462 : #else
463 : ARG_UNUSED(ptr);
464 :
465 : return false;
466 : #endif
467 : }
468 :
469 : /**
470 : * @brief Test if a pointer is in un-cached region.
471 : *
472 : * Some hardware may map the same physical memory twice
473 : * so that it can be seen in both (incoherent) cached mappings
474 : * and a coherent "shared" area. This tests if a particular
475 : * pointer is within the un-cached, incoherent area.
476 : *
477 : * @param ptr Pointer
478 : *
479 : * @retval True if pointer is not in cached region.
480 : * @retval False if pointer is in cached region.
481 : */
482 1 : static ALWAYS_INLINE bool sys_cache_is_ptr_uncached(void *ptr)
483 : {
484 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
485 : return cache_is_ptr_uncached(ptr);
486 : #else
487 : ARG_UNUSED(ptr);
488 :
489 : return false;
490 : #endif
491 : }
492 :
493 : /**
494 : * @brief Return cached pointer to a RAM address
495 : *
496 : * This function takes a pointer to any addressable object (either in
497 : * cacheable memory or not) and returns a pointer that can be used to
498 : * refer to the same memory through the L1 data cache. Data read
499 : * through the resulting pointer will reflect locally cached values on
500 : * the current CPU if they exist, and writes will go first into the
501 : * cache and be written back later.
502 : *
503 : * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not
504 : * enabled.
505 : *
506 : * @see arch_uncached_ptr()
507 : *
508 : * @param ptr A pointer to a valid C object
509 : * @return A pointer to the same object via the L1 dcache
510 : */
511 1 : static ALWAYS_INLINE void __sparse_cache *sys_cache_cached_ptr_get(void *ptr)
512 : {
513 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
514 : return cache_cached_ptr(ptr);
515 : #else
516 : return (__sparse_force void __sparse_cache *)ptr;
517 : #endif
518 : }
519 :
520 : /**
521 : * @brief Return uncached pointer to a RAM address
522 : *
523 : * This function takes a pointer to any addressable object (either in
524 : * cacheable memory or not) and returns a pointer that can be used to
525 : * refer to the same memory while bypassing the L1 data cache. Data
526 : * in the L1 cache will not be inspected nor modified by the access.
527 : *
528 : * @note This API returns the same pointer if CONFIG_CACHE_DOUBLEMAP is not
529 : * enabled.
530 : *
531 : * @see arch_cached_ptr()
532 : *
533 : * @param ptr A pointer to a valid C object
534 : * @return A pointer to the same object bypassing the L1 dcache
535 : */
536 1 : static ALWAYS_INLINE void *sys_cache_uncached_ptr_get(void __sparse_cache *ptr)
537 : {
538 : #if defined(CONFIG_CACHE_MANAGEMENT) && defined(CONFIG_CACHE_DOUBLEMAP)
539 : return cache_uncached_ptr(ptr);
540 : #else
541 : return (__sparse_force void *)ptr;
542 : #endif
543 : }
544 :
545 :
546 : #ifdef CONFIG_LIBMETAL
547 : static ALWAYS_INLINE void sys_cache_flush(void *addr, size_t size)
548 : {
549 : sys_cache_data_flush_range(addr, size);
550 : }
551 : #endif
552 :
553 : #include <zephyr/syscalls/cache.h>
554 : #ifdef __cplusplus
555 : }
556 : #endif
557 :
558 : /**
559 : * @}
560 : */
561 :
562 : #endif /* ZEPHYR_INCLUDE_CACHE_H_ */
|
