Line data Source code
1 1 : /*
2 : * Copyright (c) 2022 Intel Corporation
3 : *
4 : * SPDX-License-Identifier: Apache-2.0
5 : */
6 :
7 : /**
8 : * @file
9 : * @brief Real-Time IO device API for moving bytes with low effort
10 : *
11 : * RTIO is a context for asynchronous batch operations using a submission and completion queue.
12 : *
13 : * Asynchronous I/O operation are setup in a submission queue. Each entry in the queue describes
14 : * the operation it wishes to perform with some understood semantics.
15 : *
16 : * These operations may be chained in a such a way that only when the current
17 : * operation is complete the next will be executed. If the current operation fails
18 : * all chained operations will also fail.
19 : *
20 : * Operations may also be submitted as a transaction where a set of operations are considered
21 : * to be one operation.
22 : *
23 : * The completion of these operations typically provide one or more completion queue events.
24 : */
25 :
26 : #ifndef ZEPHYR_INCLUDE_RTIO_RTIO_H_
27 : #define ZEPHYR_INCLUDE_RTIO_RTIO_H_
28 :
29 : #include <string.h>
30 :
31 : #include <zephyr/app_memory/app_memdomain.h>
32 : #include <zephyr/device.h>
33 : #include <zephyr/kernel.h>
34 : #include <zephyr/kernel_structs.h>
35 : #include <zephyr/sys/__assert.h>
36 : #include <zephyr/sys/atomic.h>
37 : #include <zephyr/sys/mem_blocks.h>
38 : #include <zephyr/sys/util.h>
39 : #include <zephyr/sys/iterable_sections.h>
40 : #include <zephyr/sys/mpsc_lockfree.h>
41 :
42 : #ifdef __cplusplus
43 : extern "C" {
44 : #endif
45 :
46 :
47 : /**
48 : * @brief RTIO
49 : * @defgroup rtio RTIO
50 : * @since 3.2
51 : * @version 0.2.0
52 : * @ingroup os_services
53 : * @{
54 : */
55 :
56 : /**
57 : * @brief RTIO Predefined Priorities
58 : * @defgroup rtio_sqe_prio RTIO Priorities
59 : * @ingroup rtio
60 : * @{
61 : */
62 :
63 : /**
64 : * @brief Low priority
65 : */
66 1 : #define RTIO_PRIO_LOW 0U
67 :
68 : /**
69 : * @brief Normal priority
70 : */
71 1 : #define RTIO_PRIO_NORM 127U
72 :
73 : /**
74 : * @brief High priority
75 : */
76 1 : #define RTIO_PRIO_HIGH 255U
77 :
78 : /**
79 : * @}
80 : */
81 :
82 :
83 : /**
84 : * @brief RTIO SQE Flags
85 : * @defgroup rtio_sqe_flags RTIO SQE Flags
86 : * @ingroup rtio
87 : * @{
88 : */
89 :
90 : /**
91 : * @brief The next request in the queue should wait on this one.
92 : *
93 : * Chained SQEs are individual units of work describing patterns of
94 : * ordering and failure cascading. A chained SQE must be started only
95 : * after the one before it. They are given to the iodevs one after another.
96 : */
97 1 : #define RTIO_SQE_CHAINED BIT(0)
98 :
99 : /**
100 : * @brief The next request in the queue is part of a transaction.
101 : *
102 : * Transactional SQEs are sequential parts of a unit of work.
103 : * Only the first transactional SQE is submitted to an iodev, the
104 : * remaining SQEs are never individually submitted but instead considered
105 : * to be part of the transaction to the single iodev. The first sqe in the
106 : * sequence holds the iodev that will be used and the last holds the userdata
107 : * that will be returned in a single completion on failure/success.
108 : */
109 1 : #define RTIO_SQE_TRANSACTION BIT(1)
110 :
111 :
112 : /**
113 : * @brief The buffer should be allocated by the RTIO mempool
114 : *
115 : * This flag can only exist if the CONFIG_RTIO_SYS_MEM_BLOCKS Kconfig was
116 : * enabled and the RTIO context was created via the RTIO_DEFINE_WITH_MEMPOOL()
117 : * macro. If set, the buffer associated with the entry was allocated by the
118 : * internal memory pool and should be released as soon as it is no longer
119 : * needed via a call to rtio_release_mempool().
120 : */
121 1 : #define RTIO_SQE_MEMPOOL_BUFFER BIT(2)
122 :
123 : /**
124 : * @brief The SQE should not execute if possible
125 : *
126 : * If possible (not yet executed), the SQE should be canceled by flagging it as failed and returning
127 : * -ECANCELED as the result.
128 : */
129 1 : #define RTIO_SQE_CANCELED BIT(3)
130 :
131 : /**
132 : * @brief The SQE should continue producing CQEs until canceled
133 : *
134 : * This flag must exist along @ref RTIO_SQE_MEMPOOL_BUFFER and signals that when a read is
135 : * complete. It should be placed back in queue until canceled.
136 : */
137 1 : #define RTIO_SQE_MULTISHOT BIT(4)
138 :
139 : /**
140 : * @brief The SQE does not produce a CQE.
141 : */
142 1 : #define RTIO_SQE_NO_RESPONSE BIT(5)
143 :
144 : /**
145 : * @}
146 : */
147 :
148 : /**
149 : * @brief RTIO CQE Flags
150 : * @defgroup rtio_cqe_flags RTIO CQE Flags
151 : * @ingroup rtio
152 : * @{
153 : */
154 :
155 : /**
156 : * @brief The entry's buffer was allocated from the RTIO's mempool
157 : *
158 : * If this bit is set, the buffer was allocated from the memory pool and should be recycled as
159 : * soon as the application is done with it.
160 : */
161 1 : #define RTIO_CQE_FLAG_MEMPOOL_BUFFER BIT(0)
162 :
163 0 : #define RTIO_CQE_FLAG_GET(flags) FIELD_GET(GENMASK(7, 0), (flags))
164 :
165 : /**
166 : * @brief Get the block index of a mempool flags
167 : *
168 : * @param flags The CQE flags value
169 : * @return The block index portion of the flags field.
170 : */
171 1 : #define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(flags) FIELD_GET(GENMASK(19, 8), (flags))
172 :
173 : /**
174 : * @brief Get the block count of a mempool flags
175 : *
176 : * @param flags The CQE flags value
177 : * @return The block count portion of the flags field.
178 : */
179 1 : #define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(flags) FIELD_GET(GENMASK(31, 20), (flags))
180 :
181 : /**
182 : * @brief Prepare CQE flags for a mempool read.
183 : *
184 : * @param blk_idx The mempool block index
185 : * @param blk_cnt The mempool block count
186 : * @return A shifted and masked value that can be added to the flags field with an OR operator.
187 : */
188 1 : #define RTIO_CQE_FLAG_PREP_MEMPOOL(blk_idx, blk_cnt) \
189 : (FIELD_PREP(GENMASK(7, 0), RTIO_CQE_FLAG_MEMPOOL_BUFFER) | \
190 : FIELD_PREP(GENMASK(19, 8), blk_idx) | FIELD_PREP(GENMASK(31, 20), blk_cnt))
191 :
192 : /**
193 : * @}
194 : */
195 :
196 : /**
197 : * @brief Equivalent to the I2C_MSG_STOP flag
198 : */
199 1 : #define RTIO_IODEV_I2C_STOP BIT(1)
200 :
201 : /**
202 : * @brief Equivalent to the I2C_MSG_RESTART flag
203 : */
204 1 : #define RTIO_IODEV_I2C_RESTART BIT(2)
205 :
206 : /**
207 : * @brief Equivalent to the I2C_MSG_ADDR_10_BITS
208 : */
209 1 : #define RTIO_IODEV_I2C_10_BITS BIT(3)
210 :
211 : /**
212 : * @brief Equivalent to the I3C_MSG_STOP flag
213 : */
214 1 : #define RTIO_IODEV_I3C_STOP BIT(1)
215 :
216 : /**
217 : * @brief Equivalent to the I3C_MSG_RESTART flag
218 : */
219 1 : #define RTIO_IODEV_I3C_RESTART BIT(2)
220 :
221 : /**
222 : * @brief Equivalent to the I3C_MSG_HDR
223 : */
224 1 : #define RTIO_IODEV_I3C_HDR BIT(3)
225 :
226 : /**
227 : * @brief Equivalent to the I3C_MSG_NBCH
228 : */
229 1 : #define RTIO_IODEV_I3C_NBCH BIT(4)
230 :
231 : /**
232 : * @brief I3C HDR Mode Mask
233 : */
234 1 : #define RTIO_IODEV_I3C_HDR_MODE_MASK GENMASK(15, 8)
235 :
236 : /**
237 : * @brief I3C HDR Mode Mask
238 : */
239 1 : #define RTIO_IODEV_I3C_HDR_MODE_SET(flags) \
240 : FIELD_PREP(RTIO_IODEV_I3C_HDR_MODE_MASK, flags)
241 :
242 : /**
243 : * @brief I3C HDR Mode Mask
244 : */
245 1 : #define RTIO_IODEV_I3C_HDR_MODE_GET(flags) \
246 : FIELD_GET(RTIO_IODEV_I3C_HDR_MODE_MASK, flags)
247 :
248 : /**
249 : * @brief I3C HDR 7b Command Code
250 : */
251 1 : #define RTIO_IODEV_I3C_HDR_CMD_CODE_MASK GENMASK(22, 16)
252 :
253 : /**
254 : * @brief I3C HDR 7b Command Code
255 : */
256 1 : #define RTIO_IODEV_I3C_HDR_CMD_CODE_SET(flags) \
257 : FIELD_PREP(RTIO_IODEV_I3C_HDR_CMD_CODE_MASK, flags)
258 :
259 : /**
260 : * @brief I3C HDR 7b Command Code
261 : */
262 1 : #define RTIO_IODEV_I3C_HDR_CMD_CODE_GET(flags) \
263 : FIELD_GET(RTIO_IODEV_I3C_HDR_CMD_CODE_MASK, flags)
264 :
265 : /** @cond ignore */
266 : struct rtio;
267 : struct rtio_cqe;
268 : struct rtio_sqe;
269 : struct rtio_sqe_pool;
270 : struct rtio_cqe_pool;
271 : struct rtio_iodev;
272 : struct rtio_iodev_sqe;
273 : /** @endcond */
274 :
275 : /**
276 : * @typedef rtio_callback_t
277 : * @brief Callback signature for RTIO_OP_CALLBACK
278 : * @param r RTIO context being used with the callback
279 : * @param sqe Submission for the callback op
280 : * @param arg0 Argument option as part of the sqe
281 : */
282 1 : typedef void (*rtio_callback_t)(struct rtio *r, const struct rtio_sqe *sqe, void *arg0);
283 :
284 : /**
285 : * @typedef rtio_signaled_t
286 : * @brief Callback signature for RTIO_OP_AWAIT signaled
287 : * @param iodev_sqe IODEV submission for the await op
288 : * @param userdata Userdata
289 : */
290 1 : typedef void (*rtio_signaled_t)(struct rtio_iodev_sqe *iodev_sqe, void *userdata);
291 :
292 : /**
293 : * @brief A submission queue event
294 : */
295 1 : struct rtio_sqe {
296 1 : uint8_t op; /**< Op code */
297 :
298 1 : uint8_t prio; /**< Op priority */
299 :
300 1 : uint16_t flags; /**< Op Flags */
301 :
302 1 : uint32_t iodev_flags; /**< Op iodev flags */
303 :
304 1 : const struct rtio_iodev *iodev; /**< Device to operation on */
305 :
306 : /**
307 : * User provided data which is returned upon operation completion. Could be a pointer or
308 : * integer.
309 : *
310 : * If unique identification of completions is desired this should be
311 : * unique as well.
312 : */
313 1 : void *userdata;
314 :
315 : union {
316 :
317 : /** OP_TX */
318 : struct {
319 1 : uint32_t buf_len; /**< Length of buffer */
320 1 : const uint8_t *buf; /**< Buffer to write from */
321 1 : } tx;
322 :
323 : /** OP_RX */
324 : struct {
325 : uint32_t buf_len; /**< Length of buffer */
326 1 : uint8_t *buf; /**< Buffer to read into */
327 1 : } rx;
328 :
329 : /** OP_TINY_TX */
330 : struct {
331 1 : uint8_t buf_len; /**< Length of tiny buffer */
332 1 : uint8_t buf[7]; /**< Tiny buffer */
333 1 : } tiny_tx;
334 :
335 : /** OP_CALLBACK */
336 : struct {
337 0 : rtio_callback_t callback;
338 1 : void *arg0; /**< Last argument given to callback */
339 1 : } callback;
340 :
341 : /** OP_TXRX */
342 : struct {
343 : uint32_t buf_len; /**< Length of tx and rx buffers */
344 1 : const uint8_t *tx_buf; /**< Buffer to write from */
345 1 : uint8_t *rx_buf; /**< Buffer to read into */
346 1 : } txrx;
347 :
348 : /** OP_DELAY */
349 : struct {
350 1 : k_timeout_t timeout; /**< Delay timeout. */
351 1 : struct _timeout to; /**< Timeout struct. Used internally. */
352 1 : } delay;
353 :
354 : /** OP_I2C_CONFIGURE */
355 1 : uint32_t i2c_config;
356 :
357 : /** OP_I3C_CONFIGURE */
358 : struct {
359 : /* enum i3c_config_type type; */
360 0 : int type;
361 0 : void *config;
362 1 : } i3c_config;
363 :
364 : /** OP_I3C_CCC */
365 : /* struct i3c_ccc_payload *ccc_payload; */
366 1 : void *ccc_payload;
367 :
368 : /** OP_AWAIT */
369 : struct {
370 0 : atomic_t ok;
371 0 : rtio_signaled_t callback;
372 : void *userdata;
373 1 : } await;
374 0 : };
375 : };
376 :
377 : /** @cond ignore */
378 : /* Ensure the rtio_sqe never grows beyond a common cacheline size of 64 bytes */
379 : BUILD_ASSERT(sizeof(struct rtio_sqe) <= 64);
380 : /** @endcond */
381 :
382 : /**
383 : * @brief A completion queue event
384 : */
385 1 : struct rtio_cqe {
386 0 : struct mpsc_node q;
387 :
388 1 : int32_t result; /**< Result from operation */
389 1 : void *userdata; /**< Associated userdata with operation */
390 1 : uint32_t flags; /**< Flags associated with the operation */
391 : };
392 :
393 0 : struct rtio_sqe_pool {
394 0 : struct mpsc free_q;
395 0 : const uint16_t pool_size;
396 0 : uint16_t pool_free;
397 0 : struct rtio_iodev_sqe *pool;
398 : };
399 :
400 0 : struct rtio_cqe_pool {
401 0 : struct mpsc free_q;
402 0 : const uint16_t pool_size;
403 0 : uint16_t pool_free;
404 0 : struct rtio_cqe *pool;
405 : };
406 :
407 : /**
408 : * @brief An RTIO context containing what can be viewed as a pair of queues.
409 : *
410 : * A queue for submissions (available and in queue to be produced) as well as a queue
411 : * of completions (available and ready to be consumed).
412 : *
413 : * The rtio executor along with any objects implementing the rtio_iodev interface are
414 : * the consumers of submissions and producers of completions.
415 : *
416 : * No work is started until rtio_submit() is called.
417 : */
418 1 : struct rtio {
419 : #ifdef CONFIG_RTIO_SUBMIT_SEM
420 : /* A wait semaphore which may suspend the calling thread
421 : * to wait for some number of completions when calling submit
422 : */
423 : struct k_sem *submit_sem;
424 :
425 : uint32_t submit_count;
426 : #endif
427 :
428 : #ifdef CONFIG_RTIO_CONSUME_SEM
429 : /* A wait semaphore which may suspend the calling thread
430 : * to wait for some number of completions while consuming
431 : * them from the completion queue
432 : */
433 : struct k_sem *consume_sem;
434 : #endif
435 :
436 : /* Total number of completions */
437 0 : atomic_t cq_count;
438 :
439 : /* Number of completions that were unable to be submitted with results
440 : * due to the cq spsc being full
441 : */
442 0 : atomic_t xcqcnt;
443 :
444 : /* Submission queue object pool with free list */
445 0 : struct rtio_sqe_pool *sqe_pool;
446 :
447 : /* Complete queue object pool with free list */
448 0 : struct rtio_cqe_pool *cqe_pool;
449 :
450 : #ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
451 : /* Mem block pool */
452 : struct sys_mem_blocks *block_pool;
453 : #endif
454 :
455 : /* Submission queue */
456 0 : struct mpsc sq;
457 :
458 : /* Completion queue */
459 0 : struct mpsc cq;
460 : };
461 :
462 : /** The memory partition associated with all RTIO context information */
463 1 : extern struct k_mem_partition rtio_partition;
464 :
465 : /**
466 : * @brief Get the mempool block size of the RTIO context
467 : *
468 : * @param[in] r The RTIO context
469 : * @return The size of each block in the context's mempool
470 : * @return 0 if the context doesn't have a mempool
471 : */
472 1 : static inline size_t rtio_mempool_block_size(const struct rtio *r)
473 : {
474 : #ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
475 : ARG_UNUSED(r);
476 : return 0;
477 : #else
478 : if (r == NULL || r->block_pool == NULL) {
479 : return 0;
480 : }
481 : return BIT(r->block_pool->info.blk_sz_shift);
482 : #endif
483 : }
484 :
485 : /**
486 : * @brief Compute the mempool block index for a given pointer
487 : *
488 : * @param[in] r RTIO context
489 : * @param[in] ptr Memory pointer in the mempool
490 : * @return Index of the mempool block associated with the pointer. Or UINT16_MAX if invalid.
491 : */
492 : #ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
493 : static inline uint16_t __rtio_compute_mempool_block_index(const struct rtio *r, const void *ptr)
494 : {
495 : uintptr_t addr = (uintptr_t)ptr;
496 : struct sys_mem_blocks *mem_pool = r->block_pool;
497 : uint32_t block_size = rtio_mempool_block_size(r);
498 :
499 : uintptr_t buff = (uintptr_t)mem_pool->buffer;
500 : uint32_t buff_size = mem_pool->info.num_blocks * block_size;
501 :
502 : if (addr < buff || addr >= buff + buff_size) {
503 : return UINT16_MAX;
504 : }
505 : return (addr - buff) / block_size;
506 : }
507 : #endif
508 :
509 : /**
510 : * @brief IO device submission queue entry
511 : *
512 : * May be cast safely to and from a rtio_sqe as they occupy the same memory provided by the pool
513 : */
514 1 : struct rtio_iodev_sqe {
515 0 : struct rtio_sqe sqe;
516 0 : struct mpsc_node q;
517 0 : struct rtio_iodev_sqe *next;
518 0 : struct rtio *r;
519 : };
520 :
521 : /**
522 : * @brief API that an RTIO IO device should implement
523 : */
524 1 : struct rtio_iodev_api {
525 : /**
526 : * @brief Submit to the iodev an entry to work on
527 : *
528 : * This call should be short in duration and most likely
529 : * either enqueue or kick off an entry with the hardware.
530 : *
531 : * @param iodev_sqe Submission queue entry
532 : */
533 1 : void (*submit)(struct rtio_iodev_sqe *iodev_sqe);
534 : };
535 :
536 : /**
537 : * @brief An IO device with a function table for submitting requests
538 : */
539 1 : struct rtio_iodev {
540 : /* Function pointer table */
541 0 : const struct rtio_iodev_api *api;
542 :
543 : /* Data associated with this iodev */
544 0 : void *data;
545 : };
546 :
547 : /** An operation that does nothing and will complete immediately */
548 1 : #define RTIO_OP_NOP 0
549 :
550 : /** An operation that receives (reads) */
551 1 : #define RTIO_OP_RX (RTIO_OP_NOP+1)
552 :
553 : /** An operation that transmits (writes) */
554 1 : #define RTIO_OP_TX (RTIO_OP_RX+1)
555 :
556 : /** An operation that transmits tiny writes by copying the data to write */
557 1 : #define RTIO_OP_TINY_TX (RTIO_OP_TX+1)
558 :
559 : /** An operation that calls a given function (callback) */
560 1 : #define RTIO_OP_CALLBACK (RTIO_OP_TINY_TX+1)
561 :
562 : /** An operation that transceives (reads and writes simultaneously) */
563 1 : #define RTIO_OP_TXRX (RTIO_OP_CALLBACK+1)
564 :
565 : /** An operation that takes a specified amount of time (asynchronously) before completing */
566 1 : #define RTIO_OP_DELAY (RTIO_OP_TXRX+1)
567 :
568 : /** An operation to recover I2C buses */
569 1 : #define RTIO_OP_I2C_RECOVER (RTIO_OP_DELAY+1)
570 :
571 : /** An operation to configure I2C buses */
572 1 : #define RTIO_OP_I2C_CONFIGURE (RTIO_OP_I2C_RECOVER+1)
573 :
574 : /** An operation to recover I3C buses */
575 1 : #define RTIO_OP_I3C_RECOVER (RTIO_OP_I2C_CONFIGURE+1)
576 :
577 : /** An operation to configure I3C buses */
578 1 : #define RTIO_OP_I3C_CONFIGURE (RTIO_OP_I3C_RECOVER+1)
579 :
580 : /** An operation to sends I3C CCC */
581 1 : #define RTIO_OP_I3C_CCC (RTIO_OP_I3C_CONFIGURE+1)
582 :
583 : /** An operation to suspend bus while awaiting signal */
584 1 : #define RTIO_OP_AWAIT (RTIO_OP_I3C_CCC+1)
585 :
586 : /**
587 : * @brief Prepare a nop (no op) submission
588 : */
589 1 : static inline void rtio_sqe_prep_nop(struct rtio_sqe *sqe,
590 : const struct rtio_iodev *iodev,
591 : void *userdata)
592 : {
593 : memset(sqe, 0, sizeof(struct rtio_sqe));
594 : sqe->op = RTIO_OP_NOP;
595 : sqe->iodev = iodev;
596 : sqe->userdata = userdata;
597 : }
598 :
599 : /**
600 : * @brief Prepare a read op submission
601 : */
602 1 : static inline void rtio_sqe_prep_read(struct rtio_sqe *sqe,
603 : const struct rtio_iodev *iodev,
604 : int8_t prio,
605 : uint8_t *buf,
606 : uint32_t len,
607 : void *userdata)
608 : {
609 : memset(sqe, 0, sizeof(struct rtio_sqe));
610 : sqe->op = RTIO_OP_RX;
611 : sqe->prio = prio;
612 : sqe->iodev = iodev;
613 : sqe->rx.buf_len = len;
614 : sqe->rx.buf = buf;
615 : sqe->userdata = userdata;
616 : }
617 :
618 : /**
619 : * @brief Prepare a read op submission with context's mempool
620 : *
621 : * @see rtio_sqe_prep_read()
622 : */
623 1 : static inline void rtio_sqe_prep_read_with_pool(struct rtio_sqe *sqe,
624 : const struct rtio_iodev *iodev, int8_t prio,
625 : void *userdata)
626 : {
627 : rtio_sqe_prep_read(sqe, iodev, prio, NULL, 0, userdata);
628 : sqe->flags = RTIO_SQE_MEMPOOL_BUFFER;
629 : }
630 :
631 0 : static inline void rtio_sqe_prep_read_multishot(struct rtio_sqe *sqe,
632 : const struct rtio_iodev *iodev, int8_t prio,
633 : void *userdata)
634 : {
635 : rtio_sqe_prep_read_with_pool(sqe, iodev, prio, userdata);
636 : sqe->flags |= RTIO_SQE_MULTISHOT;
637 : }
638 :
639 : /**
640 : * @brief Prepare a write op submission
641 : */
642 1 : static inline void rtio_sqe_prep_write(struct rtio_sqe *sqe,
643 : const struct rtio_iodev *iodev,
644 : int8_t prio,
645 : const uint8_t *buf,
646 : uint32_t len,
647 : void *userdata)
648 : {
649 : memset(sqe, 0, sizeof(struct rtio_sqe));
650 : sqe->op = RTIO_OP_TX;
651 : sqe->prio = prio;
652 : sqe->iodev = iodev;
653 : sqe->tx.buf_len = len;
654 : sqe->tx.buf = buf;
655 : sqe->userdata = userdata;
656 : }
657 :
658 : /**
659 : * @brief Prepare a tiny write op submission
660 : *
661 : * Unlike the normal write operation where the source buffer must outlive the call
662 : * the tiny write data in this case is copied to the sqe. It must be tiny to fit
663 : * within the specified size of a rtio_sqe.
664 : *
665 : * This is useful in many scenarios with RTL logic where a write of the register to
666 : * subsequently read must be done.
667 : */
668 1 : static inline void rtio_sqe_prep_tiny_write(struct rtio_sqe *sqe,
669 : const struct rtio_iodev *iodev,
670 : int8_t prio,
671 : const uint8_t *tiny_write_data,
672 : uint8_t tiny_write_len,
673 : void *userdata)
674 : {
675 : __ASSERT_NO_MSG(tiny_write_len <= sizeof(sqe->tiny_tx.buf));
676 :
677 : memset(sqe, 0, sizeof(struct rtio_sqe));
678 : sqe->op = RTIO_OP_TINY_TX;
679 : sqe->prio = prio;
680 : sqe->iodev = iodev;
681 : sqe->tiny_tx.buf_len = tiny_write_len;
682 : memcpy(sqe->tiny_tx.buf, tiny_write_data, tiny_write_len);
683 : sqe->userdata = userdata;
684 : }
685 :
686 : /**
687 : * @brief Prepare a callback op submission
688 : *
689 : * A somewhat special operation in that it may only be done in kernel mode.
690 : *
691 : * Used where general purpose logic is required in a queue of io operations to do
692 : * transforms or logic.
693 : */
694 1 : static inline void rtio_sqe_prep_callback(struct rtio_sqe *sqe,
695 : rtio_callback_t callback,
696 : void *arg0,
697 : void *userdata)
698 : {
699 : memset(sqe, 0, sizeof(struct rtio_sqe));
700 : sqe->op = RTIO_OP_CALLBACK;
701 : sqe->prio = 0;
702 : sqe->iodev = NULL;
703 : sqe->callback.callback = callback;
704 : sqe->callback.arg0 = arg0;
705 : sqe->userdata = userdata;
706 : }
707 :
708 : /**
709 : * @brief Prepare a callback op submission that does not create a CQE
710 : *
711 : * Similar to @ref rtio_sqe_prep_callback, but the @ref RTIO_SQE_NO_RESPONSE
712 : * flag is set on the SQE to prevent the generation of a CQE upon completion.
713 : *
714 : * This can be useful when the callback is the last operation in a sequence
715 : * whose job is to clean up all the previous CQE's. Without @ref RTIO_SQE_NO_RESPONSE
716 : * the completion itself will result in a CQE that cannot be consumed in the callback.
717 : */
718 1 : static inline void rtio_sqe_prep_callback_no_cqe(struct rtio_sqe *sqe,
719 : rtio_callback_t callback,
720 : void *arg0,
721 : void *userdata)
722 : {
723 : rtio_sqe_prep_callback(sqe, callback, arg0, userdata);
724 : sqe->flags |= RTIO_SQE_NO_RESPONSE;
725 : }
726 :
727 : /**
728 : * @brief Prepare a transceive op submission
729 : */
730 1 : static inline void rtio_sqe_prep_transceive(struct rtio_sqe *sqe,
731 : const struct rtio_iodev *iodev,
732 : int8_t prio,
733 : const uint8_t *tx_buf,
734 : uint8_t *rx_buf,
735 : uint32_t buf_len,
736 : void *userdata)
737 : {
738 : memset(sqe, 0, sizeof(struct rtio_sqe));
739 : sqe->op = RTIO_OP_TXRX;
740 : sqe->prio = prio;
741 : sqe->iodev = iodev;
742 : sqe->txrx.buf_len = buf_len;
743 : sqe->txrx.tx_buf = tx_buf;
744 : sqe->txrx.rx_buf = rx_buf;
745 : sqe->userdata = userdata;
746 : }
747 :
748 0 : static inline void rtio_sqe_prep_await(struct rtio_sqe *sqe,
749 : const struct rtio_iodev *iodev,
750 : int8_t prio,
751 : void *userdata)
752 : {
753 : memset(sqe, 0, sizeof(struct rtio_sqe));
754 : sqe->op = RTIO_OP_AWAIT;
755 : sqe->prio = prio;
756 : sqe->iodev = iodev;
757 : sqe->userdata = userdata;
758 : }
759 :
760 0 : static inline void rtio_sqe_prep_delay(struct rtio_sqe *sqe,
761 : k_timeout_t timeout,
762 : void *userdata)
763 : {
764 : memset(sqe, 0, sizeof(struct rtio_sqe));
765 : sqe->op = RTIO_OP_DELAY;
766 : sqe->prio = 0;
767 : sqe->iodev = NULL;
768 : sqe->delay.timeout = timeout;
769 : sqe->userdata = userdata;
770 : }
771 :
772 0 : static inline struct rtio_iodev_sqe *rtio_sqe_pool_alloc(struct rtio_sqe_pool *pool)
773 : {
774 : struct mpsc_node *node = mpsc_pop(&pool->free_q);
775 :
776 : if (node == NULL) {
777 : return NULL;
778 : }
779 :
780 : struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
781 :
782 : pool->pool_free--;
783 :
784 : return iodev_sqe;
785 : }
786 :
787 0 : static inline void rtio_sqe_pool_free(struct rtio_sqe_pool *pool, struct rtio_iodev_sqe *iodev_sqe)
788 : {
789 : mpsc_push(&pool->free_q, &iodev_sqe->q);
790 :
791 : pool->pool_free++;
792 : }
793 :
794 0 : static inline struct rtio_cqe *rtio_cqe_pool_alloc(struct rtio_cqe_pool *pool)
795 : {
796 : struct mpsc_node *node = mpsc_pop(&pool->free_q);
797 :
798 : if (node == NULL) {
799 : return NULL;
800 : }
801 :
802 : struct rtio_cqe *cqe = CONTAINER_OF(node, struct rtio_cqe, q);
803 :
804 : memset(cqe, 0, sizeof(struct rtio_cqe));
805 :
806 : pool->pool_free--;
807 :
808 : return cqe;
809 : }
810 :
811 0 : static inline void rtio_cqe_pool_free(struct rtio_cqe_pool *pool, struct rtio_cqe *cqe)
812 : {
813 : mpsc_push(&pool->free_q, &cqe->q);
814 :
815 : pool->pool_free++;
816 : }
817 :
818 0 : static inline int rtio_block_pool_alloc(struct rtio *r, size_t min_sz,
819 : size_t max_sz, uint8_t **buf, uint32_t *buf_len)
820 : {
821 : #ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
822 : ARG_UNUSED(r);
823 : ARG_UNUSED(min_sz);
824 : ARG_UNUSED(max_sz);
825 : ARG_UNUSED(buf);
826 : ARG_UNUSED(buf_len);
827 : return -ENOTSUP;
828 : #else
829 : const uint32_t block_size = rtio_mempool_block_size(r);
830 : uint32_t bytes = max_sz;
831 :
832 : /* Not every context has a block pool and the block size may return 0 in
833 : * that case
834 : */
835 : if (block_size == 0) {
836 : return -ENOMEM;
837 : }
838 :
839 : do {
840 : size_t num_blks = DIV_ROUND_UP(bytes, block_size);
841 : int rc = sys_mem_blocks_alloc_contiguous(r->block_pool, num_blks, (void **)buf);
842 :
843 : if (rc == 0) {
844 : *buf_len = num_blks * block_size;
845 : return 0;
846 : }
847 :
848 : if (bytes <= block_size) {
849 : break;
850 : }
851 :
852 : bytes -= block_size;
853 : } while (bytes >= min_sz);
854 :
855 : return -ENOMEM;
856 : #endif
857 : }
858 :
859 0 : static inline void rtio_block_pool_free(struct rtio *r, void *buf, uint32_t buf_len)
860 : {
861 : #ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
862 : ARG_UNUSED(r);
863 : ARG_UNUSED(buf);
864 : ARG_UNUSED(buf_len);
865 : #else
866 : size_t num_blks = buf_len >> r->block_pool->info.blk_sz_shift;
867 :
868 : sys_mem_blocks_free_contiguous(r->block_pool, buf, num_blks);
869 : #endif
870 : }
871 :
872 : /* Do not try and reformat the macros */
873 : /* clang-format off */
874 :
875 : /**
876 : * @brief Statically define and initialize an RTIO IODev
877 : *
878 : * @param name Name of the iodev
879 : * @param iodev_api Pointer to struct rtio_iodev_api
880 : * @param iodev_data Data pointer
881 : */
882 1 : #define RTIO_IODEV_DEFINE(name, iodev_api, iodev_data) \
883 : STRUCT_SECTION_ITERABLE(rtio_iodev, name) = { \
884 : .api = (iodev_api), \
885 : .data = (iodev_data), \
886 : }
887 :
888 : #define Z_RTIO_SQE_POOL_DEFINE(name, sz) \
889 : static struct rtio_iodev_sqe CONCAT(_sqe_pool_, name)[sz]; \
890 : STRUCT_SECTION_ITERABLE(rtio_sqe_pool, name) = { \
891 : .free_q = MPSC_INIT((name.free_q)), \
892 : .pool_size = sz, \
893 : .pool_free = sz, \
894 : .pool = CONCAT(_sqe_pool_, name), \
895 : }
896 :
897 :
898 : #define Z_RTIO_CQE_POOL_DEFINE(name, sz) \
899 : static struct rtio_cqe CONCAT(_cqe_pool_, name)[sz]; \
900 : STRUCT_SECTION_ITERABLE(rtio_cqe_pool, name) = { \
901 : .free_q = MPSC_INIT((name.free_q)), \
902 : .pool_size = sz, \
903 : .pool_free = sz, \
904 : .pool = CONCAT(_cqe_pool_, name), \
905 : }
906 :
907 : /**
908 : * @brief Allocate to bss if available
909 : *
910 : * If CONFIG_USERSPACE is selected, allocate to the rtio_partition bss. Maps to:
911 : * K_APP_BMEM(rtio_partition) static
912 : *
913 : * If CONFIG_USERSPACE is disabled, allocate as plain static:
914 : * static
915 : */
916 1 : #define RTIO_BMEM COND_CODE_1(CONFIG_USERSPACE, (K_APP_BMEM(rtio_partition) static), (static))
917 :
918 : /**
919 : * @brief Allocate as initialized memory if available
920 : *
921 : * If CONFIG_USERSPACE is selected, allocate to the rtio_partition init. Maps to:
922 : * K_APP_DMEM(rtio_partition) static
923 : *
924 : * If CONFIG_USERSPACE is disabled, allocate as plain static:
925 : * static
926 : */
927 1 : #define RTIO_DMEM COND_CODE_1(CONFIG_USERSPACE, (K_APP_DMEM(rtio_partition) static), (static))
928 :
929 : #define Z_RTIO_BLOCK_POOL_DEFINE(name, blk_sz, blk_cnt, blk_align) \
930 : RTIO_BMEM uint8_t __aligned(WB_UP(blk_align)) \
931 : CONCAT(_block_pool_, name)[blk_cnt*WB_UP(blk_sz)]; \
932 : _SYS_MEM_BLOCKS_DEFINE_WITH_EXT_BUF(name, WB_UP(blk_sz), blk_cnt, \
933 : CONCAT(_block_pool_, name), RTIO_DMEM)
934 :
935 : #define Z_RTIO_DEFINE(name, _sqe_pool, _cqe_pool, _block_pool) \
936 : IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, \
937 : (static K_SEM_DEFINE(CONCAT(_submit_sem_, name), 0, K_SEM_MAX_LIMIT))) \
938 : IF_ENABLED(CONFIG_RTIO_CONSUME_SEM, \
939 : (static K_SEM_DEFINE(CONCAT(_consume_sem_, name), 0, K_SEM_MAX_LIMIT))) \
940 : STRUCT_SECTION_ITERABLE(rtio, name) = { \
941 : IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, (.submit_sem = &CONCAT(_submit_sem_, name),)) \
942 : IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, (.submit_count = 0,)) \
943 : IF_ENABLED(CONFIG_RTIO_CONSUME_SEM, (.consume_sem = &CONCAT(_consume_sem_, name),))\
944 : .cq_count = ATOMIC_INIT(0), \
945 : .xcqcnt = ATOMIC_INIT(0), \
946 : .sqe_pool = _sqe_pool, \
947 : .cqe_pool = _cqe_pool, \
948 : IF_ENABLED(CONFIG_RTIO_SYS_MEM_BLOCKS, (.block_pool = _block_pool,)) \
949 : .sq = MPSC_INIT((name.sq)), \
950 : .cq = MPSC_INIT((name.cq)), \
951 : }
952 :
953 : /**
954 : * @brief Statically define and initialize an RTIO context
955 : *
956 : * @param name Name of the RTIO
957 : * @param sq_sz Size of the submission queue entry pool
958 : * @param cq_sz Size of the completion queue entry pool
959 : */
960 1 : #define RTIO_DEFINE(name, sq_sz, cq_sz) \
961 : Z_RTIO_SQE_POOL_DEFINE(CONCAT(name, _sqe_pool), sq_sz); \
962 : Z_RTIO_CQE_POOL_DEFINE(CONCAT(name, _cqe_pool), cq_sz); \
963 : Z_RTIO_DEFINE(name, &CONCAT(name, _sqe_pool), \
964 : &CONCAT(name, _cqe_pool), NULL)
965 :
966 : /* clang-format on */
967 :
968 : /**
969 : * @brief Statically define and initialize an RTIO context
970 : *
971 : * @param name Name of the RTIO
972 : * @param sq_sz Size of the submission queue, must be power of 2
973 : * @param cq_sz Size of the completion queue, must be power of 2
974 : * @param num_blks Number of blocks in the memory pool
975 : * @param blk_size The number of bytes in each block
976 : * @param balign The block alignment
977 : */
978 1 : #define RTIO_DEFINE_WITH_MEMPOOL(name, sq_sz, cq_sz, num_blks, blk_size, balign) \
979 : Z_RTIO_SQE_POOL_DEFINE(name##_sqe_pool, sq_sz); \
980 : Z_RTIO_CQE_POOL_DEFINE(name##_cqe_pool, cq_sz); \
981 : Z_RTIO_BLOCK_POOL_DEFINE(name##_block_pool, blk_size, num_blks, balign); \
982 : Z_RTIO_DEFINE(name, &name##_sqe_pool, &name##_cqe_pool, &name##_block_pool)
983 :
984 : /* clang-format on */
985 :
986 : /**
987 : * @brief Count of acquirable submission queue events
988 : *
989 : * @param r RTIO context
990 : *
991 : * @return Count of acquirable submission queue events
992 : */
993 1 : static inline uint32_t rtio_sqe_acquirable(struct rtio *r)
994 : {
995 : return r->sqe_pool->pool_free;
996 : }
997 :
998 : /**
999 : * @brief Get the next sqe in the transaction
1000 : *
1001 : * @param iodev_sqe Submission queue entry
1002 : *
1003 : * @retval NULL if current sqe is last in transaction
1004 : * @retval struct rtio_sqe * if available
1005 : */
1006 1 : static inline struct rtio_iodev_sqe *rtio_txn_next(const struct rtio_iodev_sqe *iodev_sqe)
1007 : {
1008 : if (iodev_sqe->sqe.flags & RTIO_SQE_TRANSACTION) {
1009 : return iodev_sqe->next;
1010 : } else {
1011 : return NULL;
1012 : }
1013 : }
1014 :
1015 :
1016 : /**
1017 : * @brief Get the next sqe in the chain
1018 : *
1019 : * @param iodev_sqe Submission queue entry
1020 : *
1021 : * @retval NULL if current sqe is last in chain
1022 : * @retval struct rtio_sqe * if available
1023 : */
1024 1 : static inline struct rtio_iodev_sqe *rtio_chain_next(const struct rtio_iodev_sqe *iodev_sqe)
1025 : {
1026 : if (iodev_sqe->sqe.flags & RTIO_SQE_CHAINED) {
1027 : return iodev_sqe->next;
1028 : } else {
1029 : return NULL;
1030 : }
1031 : }
1032 :
1033 : /**
1034 : * @brief Get the next sqe in the chain or transaction
1035 : *
1036 : * @param iodev_sqe Submission queue entry
1037 : *
1038 : * @retval NULL if current sqe is last in chain
1039 : * @retval struct rtio_iodev_sqe * if available
1040 : */
1041 1 : static inline struct rtio_iodev_sqe *rtio_iodev_sqe_next(const struct rtio_iodev_sqe *iodev_sqe)
1042 : {
1043 : return iodev_sqe->next;
1044 : }
1045 :
1046 : /**
1047 : * @brief Acquire a single submission queue event if available
1048 : *
1049 : * @param r RTIO context
1050 : *
1051 : * @retval sqe A valid submission queue event acquired from the submission queue
1052 : * @retval NULL No subsmission queue event available
1053 : */
1054 1 : static inline struct rtio_sqe *rtio_sqe_acquire(struct rtio *r)
1055 : {
1056 : struct rtio_iodev_sqe *iodev_sqe = rtio_sqe_pool_alloc(r->sqe_pool);
1057 :
1058 : if (iodev_sqe == NULL) {
1059 : return NULL;
1060 : }
1061 :
1062 : mpsc_push(&r->sq, &iodev_sqe->q);
1063 :
1064 : return &iodev_sqe->sqe;
1065 : }
1066 :
1067 : /**
1068 : * @brief Drop all previously acquired sqe
1069 : *
1070 : * @param r RTIO context
1071 : */
1072 1 : static inline void rtio_sqe_drop_all(struct rtio *r)
1073 : {
1074 : struct rtio_iodev_sqe *iodev_sqe;
1075 : struct mpsc_node *node = mpsc_pop(&r->sq);
1076 :
1077 : while (node != NULL) {
1078 : iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
1079 : rtio_sqe_pool_free(r->sqe_pool, iodev_sqe);
1080 : node = mpsc_pop(&r->sq);
1081 : }
1082 : }
1083 :
1084 : /**
1085 : * @brief Acquire a complete queue event if available
1086 : */
1087 1 : static inline struct rtio_cqe *rtio_cqe_acquire(struct rtio *r)
1088 : {
1089 : struct rtio_cqe *cqe = rtio_cqe_pool_alloc(r->cqe_pool);
1090 :
1091 : if (cqe == NULL) {
1092 : return NULL;
1093 : }
1094 :
1095 : memset(cqe, 0, sizeof(struct rtio_cqe));
1096 :
1097 : return cqe;
1098 : }
1099 :
1100 : /**
1101 : * @brief Produce a complete queue event if available
1102 : */
1103 1 : static inline void rtio_cqe_produce(struct rtio *r, struct rtio_cqe *cqe)
1104 : {
1105 : mpsc_push(&r->cq, &cqe->q);
1106 : }
1107 :
1108 : /**
1109 : * @brief Consume a single completion queue event if available
1110 : *
1111 : * If a completion queue event is returned rtio_cq_release(r) must be called
1112 : * at some point to release the cqe spot for the cqe producer.
1113 : *
1114 : * @param r RTIO context
1115 : *
1116 : * @retval cqe A valid completion queue event consumed from the completion queue
1117 : * @retval NULL No completion queue event available
1118 : */
1119 1 : static inline struct rtio_cqe *rtio_cqe_consume(struct rtio *r)
1120 : {
1121 : struct mpsc_node *node;
1122 : struct rtio_cqe *cqe = NULL;
1123 :
1124 : #ifdef CONFIG_RTIO_CONSUME_SEM
1125 : if (k_sem_take(r->consume_sem, K_NO_WAIT) != 0) {
1126 : return NULL;
1127 : }
1128 : #endif
1129 :
1130 : node = mpsc_pop(&r->cq);
1131 : if (node == NULL) {
1132 : return NULL;
1133 : }
1134 : cqe = CONTAINER_OF(node, struct rtio_cqe, q);
1135 :
1136 : return cqe;
1137 : }
1138 :
1139 : /**
1140 : * @brief Wait for and consume a single completion queue event
1141 : *
1142 : * If a completion queue event is returned rtio_cq_release(r) must be called
1143 : * at some point to release the cqe spot for the cqe producer.
1144 : *
1145 : * @param r RTIO context
1146 : *
1147 : * @retval cqe A valid completion queue event consumed from the completion queue
1148 : */
1149 1 : static inline struct rtio_cqe *rtio_cqe_consume_block(struct rtio *r)
1150 : {
1151 : struct mpsc_node *node;
1152 : struct rtio_cqe *cqe;
1153 :
1154 : #ifdef CONFIG_RTIO_CONSUME_SEM
1155 : k_sem_take(r->consume_sem, K_FOREVER);
1156 : #endif
1157 : node = mpsc_pop(&r->cq);
1158 : while (node == NULL) {
1159 : Z_SPIN_DELAY(1);
1160 : node = mpsc_pop(&r->cq);
1161 : }
1162 : cqe = CONTAINER_OF(node, struct rtio_cqe, q);
1163 :
1164 : return cqe;
1165 : }
1166 :
1167 : /**
1168 : * @brief Release consumed completion queue event
1169 : *
1170 : * @param r RTIO context
1171 : * @param cqe Completion queue entry
1172 : */
1173 1 : static inline void rtio_cqe_release(struct rtio *r, struct rtio_cqe *cqe)
1174 : {
1175 : rtio_cqe_pool_free(r->cqe_pool, cqe);
1176 : }
1177 :
1178 : /**
1179 : * @brief Flush completion queue
1180 : *
1181 : * @param r RTIO context
1182 : * @return The operation completion result
1183 : * @retval 0 if the queued operations completed with no error
1184 : * @retval <0 on error
1185 : */
1186 1 : static inline int rtio_flush_completion_queue(struct rtio *r)
1187 : {
1188 : struct rtio_cqe *cqe;
1189 : int res = 0;
1190 :
1191 : do {
1192 : cqe = rtio_cqe_consume(r);
1193 : if (cqe != NULL) {
1194 : if ((cqe->result < 0) && (res == 0)) {
1195 : res = cqe->result;
1196 : }
1197 : rtio_cqe_release(r, cqe);
1198 : }
1199 : } while (cqe != NULL);
1200 :
1201 : return res;
1202 : }
1203 :
1204 : /**
1205 : * @brief Compute the CQE flags from the rtio_iodev_sqe entry
1206 : *
1207 : * @param iodev_sqe The SQE entry in question.
1208 : * @return The value that should be set for the CQE's flags field.
1209 : */
1210 1 : static inline uint32_t rtio_cqe_compute_flags(struct rtio_iodev_sqe *iodev_sqe)
1211 : {
1212 : uint32_t flags = 0;
1213 :
1214 : #ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1215 : if (iodev_sqe->sqe.op == RTIO_OP_RX && iodev_sqe->sqe.flags & RTIO_SQE_MEMPOOL_BUFFER) {
1216 : struct rtio *r = iodev_sqe->r;
1217 : struct sys_mem_blocks *mem_pool = r->block_pool;
1218 : unsigned int blk_index = 0;
1219 : unsigned int blk_count = 0;
1220 :
1221 : if (iodev_sqe->sqe.rx.buf) {
1222 : blk_index = (iodev_sqe->sqe.rx.buf - mem_pool->buffer) >>
1223 : mem_pool->info.blk_sz_shift;
1224 : blk_count = iodev_sqe->sqe.rx.buf_len >> mem_pool->info.blk_sz_shift;
1225 : }
1226 : flags = RTIO_CQE_FLAG_PREP_MEMPOOL(blk_index, blk_count);
1227 : }
1228 : #else
1229 : ARG_UNUSED(iodev_sqe);
1230 : #endif
1231 :
1232 : return flags;
1233 : }
1234 :
1235 : /**
1236 : * @brief Retrieve the mempool buffer that was allocated for the CQE.
1237 : *
1238 : * If the RTIO context contains a memory pool, and the SQE was created by calling
1239 : * rtio_sqe_read_with_pool(), this function can be used to retrieve the memory associated with the
1240 : * read. Once processing is done, it should be released by calling rtio_release_buffer().
1241 : *
1242 : * @param[in] r RTIO context
1243 : * @param[in] cqe The CQE handling the event.
1244 : * @param[out] buff Pointer to the mempool buffer
1245 : * @param[out] buff_len Length of the allocated buffer
1246 : * @return 0 on success
1247 : * @return -EINVAL if the buffer wasn't allocated for this cqe
1248 : * @return -ENOTSUP if memory blocks are disabled
1249 : */
1250 1 : __syscall int rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe,
1251 : uint8_t **buff, uint32_t *buff_len);
1252 :
1253 : static inline int z_impl_rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe,
1254 : uint8_t **buff, uint32_t *buff_len)
1255 : {
1256 : #ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1257 : if (RTIO_CQE_FLAG_GET(cqe->flags) == RTIO_CQE_FLAG_MEMPOOL_BUFFER) {
1258 : unsigned int blk_idx = RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(cqe->flags);
1259 : unsigned int blk_count = RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(cqe->flags);
1260 : uint32_t blk_size = rtio_mempool_block_size(r);
1261 :
1262 : *buff_len = blk_count * blk_size;
1263 :
1264 : if (blk_count > 0) {
1265 : *buff = r->block_pool->buffer + blk_idx * blk_size;
1266 :
1267 : __ASSERT_NO_MSG(*buff >= r->block_pool->buffer);
1268 : __ASSERT_NO_MSG(*buff <
1269 : r->block_pool->buffer + blk_size * r->block_pool->info.num_blocks);
1270 : } else {
1271 : *buff = NULL;
1272 : }
1273 : return 0;
1274 : }
1275 : return -EINVAL;
1276 : #else
1277 : ARG_UNUSED(r);
1278 : ARG_UNUSED(cqe);
1279 : ARG_UNUSED(buff);
1280 : ARG_UNUSED(buff_len);
1281 :
1282 : return -ENOTSUP;
1283 : #endif
1284 : }
1285 :
1286 0 : void rtio_executor_submit(struct rtio *r);
1287 0 : void rtio_executor_ok(struct rtio_iodev_sqe *iodev_sqe, int result);
1288 0 : void rtio_executor_err(struct rtio_iodev_sqe *iodev_sqe, int result);
1289 :
1290 : /**
1291 : * @brief Inform the executor of a submission completion with success
1292 : *
1293 : * This may start the next asynchronous request if one is available.
1294 : *
1295 : * @param iodev_sqe IODev Submission that has succeeded
1296 : * @param result Result of the request
1297 : */
1298 1 : static inline void rtio_iodev_sqe_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
1299 : {
1300 : rtio_executor_ok(iodev_sqe, result);
1301 : }
1302 :
1303 : /**
1304 : * @brief Inform the executor of a submissions completion with error
1305 : *
1306 : * This SHALL fail the remaining submissions in the chain.
1307 : *
1308 : * @param iodev_sqe Submission that has failed
1309 : * @param result Result of the request
1310 : */
1311 1 : static inline void rtio_iodev_sqe_err(struct rtio_iodev_sqe *iodev_sqe, int result)
1312 : {
1313 : rtio_executor_err(iodev_sqe, result);
1314 : }
1315 :
1316 : /**
1317 : * Submit a completion queue event with a given result and userdata
1318 : *
1319 : * Called by the executor to produce a completion queue event, no inherent
1320 : * locking is performed and this is not safe to do from multiple callers.
1321 : *
1322 : * @param r RTIO context
1323 : * @param result Integer result code (could be -errno)
1324 : * @param userdata Userdata to pass along to completion
1325 : * @param flags Flags to use for the CEQ see RTIO_CQE_FLAG_*
1326 : */
1327 1 : static inline void rtio_cqe_submit(struct rtio *r, int result, void *userdata, uint32_t flags)
1328 : {
1329 : struct rtio_cqe *cqe = rtio_cqe_acquire(r);
1330 :
1331 : if (cqe == NULL) {
1332 : atomic_inc(&r->xcqcnt);
1333 : } else {
1334 : cqe->result = result;
1335 : cqe->userdata = userdata;
1336 : cqe->flags = flags;
1337 : rtio_cqe_produce(r, cqe);
1338 : #ifdef CONFIG_RTIO_CONSUME_SEM
1339 : k_sem_give(r->consume_sem);
1340 : #endif
1341 : }
1342 :
1343 : /* atomic_t isn't guaranteed to wrap correctly as it could be signed, so
1344 : * we must resort to a cas loop.
1345 : */
1346 : atomic_t val, new_val;
1347 :
1348 : do {
1349 : val = atomic_get(&r->cq_count);
1350 : new_val = (atomic_t)((uintptr_t)val + 1);
1351 : } while (!atomic_cas(&r->cq_count, val, new_val));
1352 :
1353 : #ifdef CONFIG_RTIO_SUBMIT_SEM
1354 : if (r->submit_count > 0) {
1355 : r->submit_count--;
1356 : if (r->submit_count == 0) {
1357 : k_sem_give(r->submit_sem);
1358 : }
1359 : }
1360 : #endif
1361 : }
1362 :
1363 : #define __RTIO_MEMPOOL_GET_NUM_BLKS(num_bytes, blk_size) (((num_bytes) + (blk_size)-1) / (blk_size))
1364 :
1365 : /**
1366 : * @brief Get the buffer associate with the RX submission
1367 : *
1368 : * @param[in] iodev_sqe The submission to probe
1369 : * @param[in] min_buf_len The minimum number of bytes needed for the operation
1370 : * @param[in] max_buf_len The maximum number of bytes needed for the operation
1371 : * @param[out] buf Where to store the pointer to the buffer
1372 : * @param[out] buf_len Where to store the size of the buffer
1373 : *
1374 : * @return 0 if @p buf and @p buf_len were successfully filled
1375 : * @return -ENOMEM Not enough memory for @p min_buf_len
1376 : */
1377 1 : static inline int rtio_sqe_rx_buf(const struct rtio_iodev_sqe *iodev_sqe, uint32_t min_buf_len,
1378 : uint32_t max_buf_len, uint8_t **buf, uint32_t *buf_len)
1379 : {
1380 : struct rtio_sqe *sqe = (struct rtio_sqe *)&iodev_sqe->sqe;
1381 :
1382 : #ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1383 : if (sqe->op == RTIO_OP_RX && sqe->flags & RTIO_SQE_MEMPOOL_BUFFER) {
1384 : struct rtio *r = iodev_sqe->r;
1385 :
1386 : if (sqe->rx.buf != NULL) {
1387 : if (sqe->rx.buf_len < min_buf_len) {
1388 : return -ENOMEM;
1389 : }
1390 : *buf = sqe->rx.buf;
1391 : *buf_len = sqe->rx.buf_len;
1392 : return 0;
1393 : }
1394 :
1395 : int rc = rtio_block_pool_alloc(r, min_buf_len, max_buf_len, buf, buf_len);
1396 : if (rc == 0) {
1397 : sqe->rx.buf = *buf;
1398 : sqe->rx.buf_len = *buf_len;
1399 : return 0;
1400 : }
1401 :
1402 : return -ENOMEM;
1403 : }
1404 : #else
1405 : ARG_UNUSED(max_buf_len);
1406 : #endif
1407 :
1408 : if (sqe->rx.buf_len < min_buf_len) {
1409 : return -ENOMEM;
1410 : }
1411 :
1412 : *buf = sqe->rx.buf;
1413 : *buf_len = sqe->rx.buf_len;
1414 : return 0;
1415 : }
1416 :
1417 : /**
1418 : * @brief Release memory that was allocated by the RTIO's memory pool
1419 : *
1420 : * If the RTIO context was created by a call to RTIO_DEFINE_WITH_MEMPOOL(), then the cqe data might
1421 : * contain a buffer that's owned by the RTIO context. In those cases (if the read request was
1422 : * configured via rtio_sqe_read_with_pool()) the buffer must be returned back to the pool.
1423 : *
1424 : * Call this function when processing is complete. This function will validate that the memory
1425 : * actually belongs to the RTIO context and will ignore invalid arguments.
1426 : *
1427 : * @param r RTIO context
1428 : * @param buff Pointer to the buffer to be released.
1429 : * @param buff_len Number of bytes to free (will be rounded up to nearest memory block).
1430 : */
1431 1 : __syscall void rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len);
1432 :
1433 : static inline void z_impl_rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len)
1434 : {
1435 : #ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1436 : if (r == NULL || buff == NULL || r->block_pool == NULL || buff_len == 0) {
1437 : return;
1438 : }
1439 :
1440 : rtio_block_pool_free(r, buff, buff_len);
1441 : #else
1442 : ARG_UNUSED(r);
1443 : ARG_UNUSED(buff);
1444 : ARG_UNUSED(buff_len);
1445 : #endif
1446 : }
1447 :
1448 : /**
1449 : * Grant access to an RTIO context to a user thread
1450 : *
1451 : * @param r RTIO context
1452 : * @param t Thread to grant permissions to
1453 : */
1454 1 : static inline void rtio_access_grant(struct rtio *r, struct k_thread *t)
1455 : {
1456 : k_object_access_grant(r, t);
1457 :
1458 : #ifdef CONFIG_RTIO_SUBMIT_SEM
1459 : k_object_access_grant(r->submit_sem, t);
1460 : #endif
1461 :
1462 : #ifdef CONFIG_RTIO_CONSUME_SEM
1463 : k_object_access_grant(r->consume_sem, t);
1464 : #endif
1465 : }
1466 :
1467 :
1468 : /**
1469 : * Revoke access to an RTIO context from a user thread
1470 : *
1471 : * @param r RTIO context
1472 : * @param t Thread to revoke permissions from
1473 : */
1474 1 : static inline void rtio_access_revoke(struct rtio *r, struct k_thread *t)
1475 : {
1476 : k_object_access_revoke(r, t);
1477 :
1478 : #ifdef CONFIG_RTIO_SUBMIT_SEM
1479 : k_object_access_revoke(r->submit_sem, t);
1480 : #endif
1481 :
1482 : #ifdef CONFIG_RTIO_CONSUME_SEM
1483 : k_object_access_revoke(r->consume_sem, t);
1484 : #endif
1485 : }
1486 :
1487 : /**
1488 : * @brief Attempt to cancel an SQE
1489 : *
1490 : * If possible (not currently executing), cancel an SQE and generate a failure with -ECANCELED
1491 : * result.
1492 : *
1493 : * @param[in] sqe The SQE to cancel
1494 : * @return 0 if the SQE was flagged for cancellation
1495 : * @return <0 on error
1496 : */
1497 1 : __syscall int rtio_sqe_cancel(struct rtio_sqe *sqe);
1498 :
1499 : static inline int z_impl_rtio_sqe_cancel(struct rtio_sqe *sqe)
1500 : {
1501 : struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe);
1502 :
1503 : do {
1504 : iodev_sqe->sqe.flags |= RTIO_SQE_CANCELED;
1505 : iodev_sqe = rtio_iodev_sqe_next(iodev_sqe);
1506 : } while (iodev_sqe != NULL);
1507 :
1508 : return 0;
1509 : }
1510 :
1511 : /**
1512 : * @brief Signal an AWAIT SQE
1513 : *
1514 : * If the SQE is currently blocking execution, execution is unblocked. If the SQE is not
1515 : * currently blocking execution, The SQE will be skipped.
1516 : *
1517 : * @note To await the AWAIT SQE blocking execution, chain a nop or callback SQE before
1518 : * the await SQE.
1519 : *
1520 : * @param[in] sqe The SQE to signal
1521 : */
1522 1 : __syscall void rtio_sqe_signal(struct rtio_sqe *sqe);
1523 :
1524 : static inline void z_impl_rtio_sqe_signal(struct rtio_sqe *sqe)
1525 : {
1526 : struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe);
1527 :
1528 : if (!atomic_cas(&iodev_sqe->sqe.await.ok, 0, 1)) {
1529 : iodev_sqe->sqe.await.callback(iodev_sqe, iodev_sqe->sqe.await.userdata);
1530 : }
1531 : }
1532 :
1533 : /**
1534 : * @brief Await an AWAIT SQE signal from RTIO IODEV
1535 : *
1536 : * If the SQE is already signaled, the callback is called immediately. Otherwise the
1537 : * callback will be called once the AWAIT SQE is signaled.
1538 : *
1539 : * @param[in] iodev_sqe The IODEV SQE to await signaled
1540 : * @param[in] callback Callback called when SQE is signaled
1541 : * @param[in] userdata User data passed to callback
1542 : */
1543 1 : static inline void rtio_iodev_sqe_await_signal(struct rtio_iodev_sqe *iodev_sqe,
1544 : rtio_signaled_t callback,
1545 : void *userdata)
1546 : {
1547 : iodev_sqe->sqe.await.callback = callback;
1548 : iodev_sqe->sqe.await.userdata = userdata;
1549 :
1550 : if (!atomic_cas(&iodev_sqe->sqe.await.ok, 0, 1)) {
1551 : callback(iodev_sqe, userdata);
1552 : }
1553 : }
1554 :
1555 : /**
1556 : * @brief Copy an array of SQEs into the queue and get resulting handles back
1557 : *
1558 : * Copies one or more SQEs into the RTIO context and optionally returns their generated SQE handles.
1559 : * Handles can be used to cancel events via the rtio_sqe_cancel() call.
1560 : *
1561 : * @param[in] r RTIO context
1562 : * @param[in] sqes Pointer to an array of SQEs
1563 : * @param[out] handle Optional pointer to @ref rtio_sqe pointer to store the handle of the
1564 : * first generated SQE. Use NULL to ignore.
1565 : * @param[in] sqe_count Count of sqes in array
1566 : *
1567 : * @retval 0 success
1568 : * @retval -ENOMEM not enough room in the queue
1569 : */
1570 1 : __syscall int rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes,
1571 : struct rtio_sqe **handle, size_t sqe_count);
1572 :
1573 : static inline int z_impl_rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes,
1574 : struct rtio_sqe **handle,
1575 : size_t sqe_count)
1576 : {
1577 : struct rtio_sqe *sqe;
1578 : uint32_t acquirable = rtio_sqe_acquirable(r);
1579 :
1580 : if (acquirable < sqe_count) {
1581 : return -ENOMEM;
1582 : }
1583 :
1584 : for (unsigned long i = 0; i < sqe_count; i++) {
1585 : sqe = rtio_sqe_acquire(r);
1586 : __ASSERT_NO_MSG(sqe != NULL);
1587 : if (handle != NULL && i == 0) {
1588 : *handle = sqe;
1589 : }
1590 : *sqe = sqes[i];
1591 : }
1592 :
1593 : return 0;
1594 : }
1595 :
1596 : /**
1597 : * @brief Copy an array of SQEs into the queue
1598 : *
1599 : * Useful if a batch of submissions is stored in ROM or
1600 : * RTIO is used from user mode where a copy must be made.
1601 : *
1602 : * Partial copying is not done as chained SQEs need to be submitted
1603 : * as a whole set.
1604 : *
1605 : * @param r RTIO context
1606 : * @param sqes Pointer to an array of SQEs
1607 : * @param sqe_count Count of sqes in array
1608 : *
1609 : * @retval 0 success
1610 : * @retval -ENOMEM not enough room in the queue
1611 : */
1612 1 : static inline int rtio_sqe_copy_in(struct rtio *r, const struct rtio_sqe *sqes, size_t sqe_count)
1613 : {
1614 : return rtio_sqe_copy_in_get_handles(r, sqes, NULL, sqe_count);
1615 : }
1616 :
1617 : /**
1618 : * @brief Copy an array of CQEs from the queue
1619 : *
1620 : * Copies from the RTIO context and its queue completion queue
1621 : * events, waiting for the given time period to gather the number
1622 : * of completions requested.
1623 : *
1624 : * @param r RTIO context
1625 : * @param cqes Pointer to an array of SQEs
1626 : * @param cqe_count Count of sqes in array
1627 : * @param timeout Timeout to wait for each completion event. Total wait time is
1628 : * potentially timeout*cqe_count at maximum.
1629 : *
1630 : * @retval copy_count Count of copied CQEs (0 to cqe_count)
1631 : */
1632 1 : __syscall int rtio_cqe_copy_out(struct rtio *r,
1633 : struct rtio_cqe *cqes,
1634 : size_t cqe_count,
1635 : k_timeout_t timeout);
1636 : static inline int z_impl_rtio_cqe_copy_out(struct rtio *r,
1637 : struct rtio_cqe *cqes,
1638 : size_t cqe_count,
1639 : k_timeout_t timeout)
1640 : {
1641 : size_t copied = 0;
1642 : struct rtio_cqe *cqe;
1643 : k_timepoint_t end = sys_timepoint_calc(timeout);
1644 :
1645 : do {
1646 : cqe = K_TIMEOUT_EQ(timeout, K_FOREVER) ? rtio_cqe_consume_block(r)
1647 : : rtio_cqe_consume(r);
1648 : if (cqe == NULL) {
1649 : Z_SPIN_DELAY(25);
1650 : continue;
1651 : }
1652 : cqes[copied++] = *cqe;
1653 : rtio_cqe_release(r, cqe);
1654 : } while (copied < cqe_count && !sys_timepoint_expired(end));
1655 :
1656 : return copied;
1657 : }
1658 :
1659 : /**
1660 : * @brief Submit I/O requests to the underlying executor
1661 : *
1662 : * Submits the queue of submission queue events to the executor.
1663 : * The executor will do the work of managing tasks representing each
1664 : * submission chain, freeing submission queue events when done, and
1665 : * producing completion queue events as submissions are completed.
1666 : *
1667 : * @warning It is undefined behavior to have re-entrant calls to submit
1668 : *
1669 : * @param r RTIO context
1670 : * @param wait_count Number of submissions to wait for completion of.
1671 : *
1672 : * @retval 0 On success
1673 : */
1674 1 : __syscall int rtio_submit(struct rtio *r, uint32_t wait_count);
1675 :
1676 : #ifdef CONFIG_RTIO_SUBMIT_SEM
1677 : static inline int z_impl_rtio_submit(struct rtio *r, uint32_t wait_count)
1678 : {
1679 : int res = 0;
1680 :
1681 : if (wait_count > 0) {
1682 : __ASSERT(!k_is_in_isr(),
1683 : "expected rtio submit with wait count to be called from a thread");
1684 :
1685 : k_sem_reset(r->submit_sem);
1686 : r->submit_count = wait_count;
1687 : }
1688 :
1689 : rtio_executor_submit(r);
1690 :
1691 : if (wait_count > 0) {
1692 : res = k_sem_take(r->submit_sem, K_FOREVER);
1693 : __ASSERT(res == 0,
1694 : "semaphore was reset or timed out while waiting on completions!");
1695 : }
1696 :
1697 : return res;
1698 : }
1699 : #else
1700 : static inline int z_impl_rtio_submit(struct rtio *r, uint32_t wait_count)
1701 : {
1702 :
1703 : int res = 0;
1704 : uintptr_t cq_count = (uintptr_t)atomic_get(&r->cq_count);
1705 : uintptr_t cq_complete_count = cq_count + wait_count;
1706 : bool wraps = cq_complete_count < cq_count;
1707 :
1708 : rtio_executor_submit(r);
1709 :
1710 : if (wraps) {
1711 : while ((uintptr_t)atomic_get(&r->cq_count) >= cq_count) {
1712 : Z_SPIN_DELAY(10);
1713 : k_yield();
1714 : }
1715 : }
1716 :
1717 : while ((uintptr_t)atomic_get(&r->cq_count) < cq_complete_count) {
1718 : Z_SPIN_DELAY(10);
1719 : k_yield();
1720 : }
1721 :
1722 : return res;
1723 : }
1724 : #endif /* CONFIG_RTIO_SUBMIT_SEM */
1725 :
1726 : /**
1727 : * @brief Pool of RTIO contexts to use with dynamically created threads
1728 : */
1729 1 : struct rtio_pool {
1730 : /** Size of the pool */
1731 1 : size_t pool_size;
1732 :
1733 : /** Array containing contexts of the pool */
1734 1 : struct rtio **contexts;
1735 :
1736 : /** Atomic bitmap to signal a member is used/unused */
1737 1 : atomic_t *used;
1738 : };
1739 :
1740 : /**
1741 : * @brief Obtain an RTIO context from a pool
1742 : *
1743 : * @param pool RTIO pool to acquire a context from
1744 : *
1745 : * @retval NULL no available contexts
1746 : * @retval r Valid context with permissions granted to the calling thread
1747 : */
1748 1 : __syscall struct rtio *rtio_pool_acquire(struct rtio_pool *pool);
1749 :
1750 : static inline struct rtio *z_impl_rtio_pool_acquire(struct rtio_pool *pool)
1751 : {
1752 : struct rtio *r = NULL;
1753 :
1754 : for (size_t i = 0; i < pool->pool_size; i++) {
1755 : if (atomic_test_and_set_bit(pool->used, i) == 0) {
1756 : r = pool->contexts[i];
1757 : break;
1758 : }
1759 : }
1760 :
1761 : if (r != NULL) {
1762 : rtio_access_grant(r, k_current_get());
1763 : }
1764 :
1765 : return r;
1766 : }
1767 :
1768 : /**
1769 : * @brief Return an RTIO context to a pool
1770 : *
1771 : * @param pool RTIO pool to return a context to
1772 : * @param r RTIO context to return to the pool
1773 : */
1774 1 : __syscall void rtio_pool_release(struct rtio_pool *pool, struct rtio *r);
1775 :
1776 : static inline void z_impl_rtio_pool_release(struct rtio_pool *pool, struct rtio *r)
1777 : {
1778 :
1779 : if (k_is_user_context()) {
1780 : rtio_access_revoke(r, k_current_get());
1781 : }
1782 :
1783 : for (size_t i = 0; i < pool->pool_size; i++) {
1784 : if (pool->contexts[i] == r) {
1785 : atomic_clear_bit(pool->used, i);
1786 : break;
1787 : }
1788 : }
1789 : }
1790 :
1791 : /* clang-format off */
1792 :
1793 : /** @cond ignore */
1794 :
1795 : #define Z_RTIO_POOL_NAME_N(n, name) \
1796 : name##_##n
1797 :
1798 : #define Z_RTIO_POOL_DEFINE_N(n, name, sq_sz, cq_sz) \
1799 : RTIO_DEFINE(Z_RTIO_POOL_NAME_N(n, name), sq_sz, cq_sz)
1800 :
1801 : #define Z_RTIO_POOL_REF_N(n, name) \
1802 : &Z_RTIO_POOL_NAME_N(n, name)
1803 :
1804 : /** @endcond */
1805 :
1806 : /**
1807 : * @brief Statically define and initialize a pool of RTIO contexts
1808 : *
1809 : * @param name Name of the RTIO pool
1810 : * @param pool_sz Number of RTIO contexts to allocate in the pool
1811 : * @param sq_sz Size of the submission queue entry pool per context
1812 : * @param cq_sz Size of the completion queue entry pool per context
1813 : */
1814 1 : #define RTIO_POOL_DEFINE(name, pool_sz, sq_sz, cq_sz) \
1815 : LISTIFY(pool_sz, Z_RTIO_POOL_DEFINE_N, (;), name, sq_sz, cq_sz); \
1816 : static struct rtio *name##_contexts[] = { \
1817 : LISTIFY(pool_sz, Z_RTIO_POOL_REF_N, (,), name) \
1818 : }; \
1819 : ATOMIC_DEFINE(name##_used, pool_sz); \
1820 : STRUCT_SECTION_ITERABLE(rtio_pool, name) = { \
1821 : .pool_size = pool_sz, \
1822 : .contexts = name##_contexts, \
1823 : .used = name##_used, \
1824 : }
1825 :
1826 : /* clang-format on */
1827 :
1828 : /**
1829 : * @}
1830 : */
1831 :
1832 : #ifdef __cplusplus
1833 : }
1834 : #endif
1835 :
1836 : #include <zephyr/syscalls/rtio.h>
1837 :
1838 : #endif /* ZEPHYR_INCLUDE_RTIO_RTIO_H_ */
|
