Zephyr API Documentation 4.2.99
A Scalable Open Source RTOS
Loading...
Searching...
No Matches
rtio.h
Go to the documentation of this file.
1/*
2 * Copyright (c) 2022 Intel Corporation
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
25
26#ifndef ZEPHYR_INCLUDE_RTIO_RTIO_H_
27#define ZEPHYR_INCLUDE_RTIO_RTIO_H_
28
29#include <string.h>
30
32#include <zephyr/device.h>
33#include <zephyr/kernel.h>
35#include <zephyr/sys/__assert.h>
36#include <zephyr/sys/atomic.h>
38#include <zephyr/sys/util.h>
41
42#ifdef __cplusplus
43extern "C" {
44#endif
45
46
55
62
66#define RTIO_PRIO_LOW 0U
67
71#define RTIO_PRIO_NORM 127U
72
76#define RTIO_PRIO_HIGH 255U
77
81
82
89
97#define RTIO_SQE_CHAINED BIT(0)
98
109#define RTIO_SQE_TRANSACTION BIT(1)
110
111
121#define RTIO_SQE_MEMPOOL_BUFFER BIT(2)
122
129#define RTIO_SQE_CANCELED BIT(3)
130
137#define RTIO_SQE_MULTISHOT BIT(4)
138
142#define RTIO_SQE_NO_RESPONSE BIT(5)
143
147
154
161#define RTIO_CQE_FLAG_MEMPOOL_BUFFER BIT(0)
162
163#define RTIO_CQE_FLAG_GET(flags) FIELD_GET(GENMASK(7, 0), (flags))
164
171#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(flags) FIELD_GET(GENMASK(19, 8), (flags))
172
179#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(flags) FIELD_GET(GENMASK(31, 20), (flags))
180
188#define RTIO_CQE_FLAG_PREP_MEMPOOL(blk_idx, blk_cnt) \
189 (FIELD_PREP(GENMASK(7, 0), RTIO_CQE_FLAG_MEMPOOL_BUFFER) | \
190 FIELD_PREP(GENMASK(19, 8), blk_idx) | FIELD_PREP(GENMASK(31, 20), blk_cnt))
191
195
199#define RTIO_IODEV_I2C_STOP BIT(1)
200
204#define RTIO_IODEV_I2C_RESTART BIT(2)
205
209#define RTIO_IODEV_I2C_10_BITS BIT(3)
210
214#define RTIO_IODEV_I3C_STOP BIT(1)
215
219#define RTIO_IODEV_I3C_RESTART BIT(2)
220
224#define RTIO_IODEV_I3C_HDR BIT(3)
225
229#define RTIO_IODEV_I3C_NBCH BIT(4)
230
234#define RTIO_IODEV_I3C_HDR_MODE_MASK GENMASK(15, 8)
235
239#define RTIO_IODEV_I3C_HDR_MODE_SET(flags) \
240 FIELD_PREP(RTIO_IODEV_I3C_HDR_MODE_MASK, flags)
241
245#define RTIO_IODEV_I3C_HDR_MODE_GET(flags) \
246 FIELD_GET(RTIO_IODEV_I3C_HDR_MODE_MASK, flags)
247
251#define RTIO_IODEV_I3C_HDR_CMD_CODE_MASK GENMASK(22, 16)
252
256#define RTIO_IODEV_I3C_HDR_CMD_CODE_SET(flags) \
257 FIELD_PREP(RTIO_IODEV_I3C_HDR_CMD_CODE_MASK, flags)
258
262#define RTIO_IODEV_I3C_HDR_CMD_CODE_GET(flags) \
263 FIELD_GET(RTIO_IODEV_I3C_HDR_CMD_CODE_MASK, flags)
264
266struct rtio;
267struct rtio_cqe;
268struct rtio_sqe;
269struct rtio_sqe_pool;
270struct rtio_cqe_pool;
271struct rtio_iodev;
272struct rtio_iodev_sqe;
274
282typedef void (*rtio_callback_t)(struct rtio *r, const struct rtio_sqe *sqe, void *arg0);
283
290typedef void (*rtio_signaled_t)(struct rtio_iodev_sqe *iodev_sqe, void *userdata);
291
295struct rtio_sqe {
297
299
301
303
304 const struct rtio_iodev *iodev;
305
313 void *userdata;
314
315 union {
316
318 struct {
320 const uint8_t *buf;
321 } tx;
322
324 struct {
325 uint32_t buf_len;
327 } rx;
328
330 struct {
333 } tiny_tx;
334
336 struct {
338 void *arg0;
339 } callback;
340
342 struct {
343 uint32_t buf_len;
346 } txrx;
347
349 struct {
351 struct _timeout to;
352 } delay;
353
356
358 struct {
359 /* enum i3c_config_type type; */
360 int type;
361 void *config;
362 } i3c_config;
363
365 /* struct i3c_ccc_payload *ccc_payload; */
367
369 struct {
372 void *userdata;
373 } await;
374 };
375};
376
378/* Ensure the rtio_sqe never grows beyond a common cacheline size of 64 bytes */
379BUILD_ASSERT(sizeof(struct rtio_sqe) <= 64);
381
392
399
406
418struct rtio {
419#ifdef CONFIG_RTIO_SUBMIT_SEM
420 /* A wait semaphore which may suspend the calling thread
421 * to wait for some number of completions when calling submit
422 */
423 struct k_sem *submit_sem;
424
425 uint32_t submit_count;
426#endif
427
428#ifdef CONFIG_RTIO_CONSUME_SEM
429 /* A wait semaphore which may suspend the calling thread
430 * to wait for some number of completions while consuming
431 * them from the completion queue
432 */
433 struct k_sem *consume_sem;
434#endif
435
436 /* Total number of completions */
438
439 /* Number of completions that were unable to be submitted with results
440 * due to the cq spsc being full
441 */
443
444 /* Submission queue object pool with free list */
446
447 /* Complete queue object pool with free list */
449
450#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
451 /* Mem block pool */
452 struct sys_mem_blocks *block_pool;
453#endif
454
455 /* Submission queue */
456 struct mpsc sq;
457
458 /* Completion queue */
459 struct mpsc cq;
460};
461
463extern struct k_mem_partition rtio_partition;
464
472static inline size_t rtio_mempool_block_size(const struct rtio *r)
473{
474#ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
475 ARG_UNUSED(r);
476 return 0;
477#else
478 if (r == NULL || r->block_pool == NULL) {
479 return 0;
480 }
481 return BIT(r->block_pool->info.blk_sz_shift);
482#endif
483}
484
492#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
493static inline uint16_t __rtio_compute_mempool_block_index(const struct rtio *r, const void *ptr)
494{
495 uintptr_t addr = (uintptr_t)ptr;
496 struct sys_mem_blocks *mem_pool = r->block_pool;
497 uint32_t block_size = rtio_mempool_block_size(r);
498
499 uintptr_t buff = (uintptr_t)mem_pool->buffer;
500 uint32_t buff_size = mem_pool->info.num_blocks * block_size;
501
502 if (addr < buff || addr >= buff + buff_size) {
503 return UINT16_MAX;
504 }
505 return (addr - buff) / block_size;
506}
507#endif
508
515 struct rtio_sqe sqe;
516 struct mpsc_node q;
518 struct rtio *r;
519};
520
533 void (*submit)(struct rtio_iodev_sqe *iodev_sqe);
534};
535
540 /* Function pointer table */
541 const struct rtio_iodev_api *api;
542
543 /* Data associated with this iodev */
544 void *data;
545};
546
548#define RTIO_OP_NOP 0
549
551#define RTIO_OP_RX (RTIO_OP_NOP+1)
552
554#define RTIO_OP_TX (RTIO_OP_RX+1)
555
557#define RTIO_OP_TINY_TX (RTIO_OP_TX+1)
558
560#define RTIO_OP_CALLBACK (RTIO_OP_TINY_TX+1)
561
563#define RTIO_OP_TXRX (RTIO_OP_CALLBACK+1)
564
566#define RTIO_OP_DELAY (RTIO_OP_TXRX+1)
567
569#define RTIO_OP_I2C_RECOVER (RTIO_OP_DELAY+1)
570
572#define RTIO_OP_I2C_CONFIGURE (RTIO_OP_I2C_RECOVER+1)
573
575#define RTIO_OP_I3C_RECOVER (RTIO_OP_I2C_CONFIGURE+1)
576
578#define RTIO_OP_I3C_CONFIGURE (RTIO_OP_I3C_RECOVER+1)
579
581#define RTIO_OP_I3C_CCC (RTIO_OP_I3C_CONFIGURE+1)
582
584#define RTIO_OP_AWAIT (RTIO_OP_I3C_CCC+1)
585
589static inline void rtio_sqe_prep_nop(struct rtio_sqe *sqe,
590 const struct rtio_iodev *iodev,
591 void *userdata)
592{
593 memset(sqe, 0, sizeof(struct rtio_sqe));
594 sqe->op = RTIO_OP_NOP;
595 sqe->iodev = iodev;
596 sqe->userdata = userdata;
597}
598
602static inline void rtio_sqe_prep_read(struct rtio_sqe *sqe,
603 const struct rtio_iodev *iodev,
604 int8_t prio,
605 uint8_t *buf,
606 uint32_t len,
607 void *userdata)
608{
609 memset(sqe, 0, sizeof(struct rtio_sqe));
610 sqe->op = RTIO_OP_RX;
611 sqe->prio = prio;
612 sqe->iodev = iodev;
613 sqe->rx.buf_len = len;
614 sqe->rx.buf = buf;
615 sqe->userdata = userdata;
616}
617
623static inline void rtio_sqe_prep_read_with_pool(struct rtio_sqe *sqe,
624 const struct rtio_iodev *iodev, int8_t prio,
625 void *userdata)
626{
627 rtio_sqe_prep_read(sqe, iodev, prio, NULL, 0, userdata);
629}
630
631static inline void rtio_sqe_prep_read_multishot(struct rtio_sqe *sqe,
632 const struct rtio_iodev *iodev, int8_t prio,
633 void *userdata)
634{
635 rtio_sqe_prep_read_with_pool(sqe, iodev, prio, userdata);
637}
638
642static inline void rtio_sqe_prep_write(struct rtio_sqe *sqe,
643 const struct rtio_iodev *iodev,
644 int8_t prio,
645 const uint8_t *buf,
646 uint32_t len,
647 void *userdata)
648{
649 memset(sqe, 0, sizeof(struct rtio_sqe));
650 sqe->op = RTIO_OP_TX;
651 sqe->prio = prio;
652 sqe->iodev = iodev;
653 sqe->tx.buf_len = len;
654 sqe->tx.buf = buf;
655 sqe->userdata = userdata;
656}
657
668static inline void rtio_sqe_prep_tiny_write(struct rtio_sqe *sqe,
669 const struct rtio_iodev *iodev,
670 int8_t prio,
671 const uint8_t *tiny_write_data,
672 uint8_t tiny_write_len,
673 void *userdata)
674{
675 __ASSERT_NO_MSG(tiny_write_len <= sizeof(sqe->tiny_tx.buf));
676
677 memset(sqe, 0, sizeof(struct rtio_sqe));
678 sqe->op = RTIO_OP_TINY_TX;
679 sqe->prio = prio;
680 sqe->iodev = iodev;
681 sqe->tiny_tx.buf_len = tiny_write_len;
682 memcpy(sqe->tiny_tx.buf, tiny_write_data, tiny_write_len);
683 sqe->userdata = userdata;
684}
685
694static inline void rtio_sqe_prep_callback(struct rtio_sqe *sqe,
695 rtio_callback_t callback,
696 void *arg0,
697 void *userdata)
698{
699 memset(sqe, 0, sizeof(struct rtio_sqe));
700 sqe->op = RTIO_OP_CALLBACK;
701 sqe->prio = 0;
702 sqe->iodev = NULL;
703 sqe->callback.callback = callback;
704 sqe->callback.arg0 = arg0;
705 sqe->userdata = userdata;
706}
707
718static inline void rtio_sqe_prep_callback_no_cqe(struct rtio_sqe *sqe,
719 rtio_callback_t callback,
720 void *arg0,
721 void *userdata)
722{
723 rtio_sqe_prep_callback(sqe, callback, arg0, userdata);
725}
726
730static inline void rtio_sqe_prep_transceive(struct rtio_sqe *sqe,
731 const struct rtio_iodev *iodev,
732 int8_t prio,
733 const uint8_t *tx_buf,
734 uint8_t *rx_buf,
735 uint32_t buf_len,
736 void *userdata)
737{
738 memset(sqe, 0, sizeof(struct rtio_sqe));
739 sqe->op = RTIO_OP_TXRX;
740 sqe->prio = prio;
741 sqe->iodev = iodev;
742 sqe->txrx.buf_len = buf_len;
743 sqe->txrx.tx_buf = tx_buf;
744 sqe->txrx.rx_buf = rx_buf;
745 sqe->userdata = userdata;
746}
747
748static inline void rtio_sqe_prep_await(struct rtio_sqe *sqe,
749 const struct rtio_iodev *iodev,
750 int8_t prio,
751 void *userdata)
752{
753 memset(sqe, 0, sizeof(struct rtio_sqe));
754 sqe->op = RTIO_OP_AWAIT;
755 sqe->prio = prio;
756 sqe->iodev = iodev;
757 sqe->userdata = userdata;
758}
759
760static inline void rtio_sqe_prep_delay(struct rtio_sqe *sqe,
761 k_timeout_t timeout,
762 void *userdata)
763{
764 memset(sqe, 0, sizeof(struct rtio_sqe));
765 sqe->op = RTIO_OP_DELAY;
766 sqe->prio = 0;
767 sqe->iodev = NULL;
768 sqe->delay.timeout = timeout;
769 sqe->userdata = userdata;
770}
771
772static inline struct rtio_iodev_sqe *rtio_sqe_pool_alloc(struct rtio_sqe_pool *pool)
773{
774 struct mpsc_node *node = mpsc_pop(&pool->free_q);
775
776 if (node == NULL) {
777 return NULL;
778 }
779
780 struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
781
782 pool->pool_free--;
783
784 return iodev_sqe;
785}
786
787static inline void rtio_sqe_pool_free(struct rtio_sqe_pool *pool, struct rtio_iodev_sqe *iodev_sqe)
788{
789 mpsc_push(&pool->free_q, &iodev_sqe->q);
790
791 pool->pool_free++;
792}
793
794static inline struct rtio_cqe *rtio_cqe_pool_alloc(struct rtio_cqe_pool *pool)
795{
796 struct mpsc_node *node = mpsc_pop(&pool->free_q);
797
798 if (node == NULL) {
799 return NULL;
800 }
801
802 struct rtio_cqe *cqe = CONTAINER_OF(node, struct rtio_cqe, q);
803
804 memset(cqe, 0, sizeof(struct rtio_cqe));
805
806 pool->pool_free--;
807
808 return cqe;
809}
810
811static inline void rtio_cqe_pool_free(struct rtio_cqe_pool *pool, struct rtio_cqe *cqe)
812{
813 mpsc_push(&pool->free_q, &cqe->q);
814
815 pool->pool_free++;
816}
817
818static inline int rtio_block_pool_alloc(struct rtio *r, size_t min_sz,
819 size_t max_sz, uint8_t **buf, uint32_t *buf_len)
820{
821#ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
822 ARG_UNUSED(r);
823 ARG_UNUSED(min_sz);
824 ARG_UNUSED(max_sz);
825 ARG_UNUSED(buf);
826 ARG_UNUSED(buf_len);
827 return -ENOTSUP;
828#else
829 const uint32_t block_size = rtio_mempool_block_size(r);
830 uint32_t bytes = max_sz;
831
832 /* Not every context has a block pool and the block size may return 0 in
833 * that case
834 */
835 if (block_size == 0) {
836 return -ENOMEM;
837 }
838
839 do {
840 size_t num_blks = DIV_ROUND_UP(bytes, block_size);
841 int rc = sys_mem_blocks_alloc_contiguous(r->block_pool, num_blks, (void **)buf);
842
843 if (rc == 0) {
844 *buf_len = num_blks * block_size;
845 return 0;
846 }
847
848 if (bytes <= block_size) {
849 break;
850 }
851
852 bytes -= block_size;
853 } while (bytes >= min_sz);
854
855 return -ENOMEM;
856#endif
857}
858
859static inline void rtio_block_pool_free(struct rtio *r, void *buf, uint32_t buf_len)
860{
861#ifndef CONFIG_RTIO_SYS_MEM_BLOCKS
862 ARG_UNUSED(r);
863 ARG_UNUSED(buf);
864 ARG_UNUSED(buf_len);
865#else
866 size_t num_blks = buf_len >> r->block_pool->info.blk_sz_shift;
867
868 sys_mem_blocks_free_contiguous(r->block_pool, buf, num_blks);
869#endif
870}
871
872/* Do not try and reformat the macros */
873/* clang-format off */
874
882#define RTIO_IODEV_DEFINE(name, iodev_api, iodev_data) \
883 STRUCT_SECTION_ITERABLE(rtio_iodev, name) = { \
884 .api = (iodev_api), \
885 .data = (iodev_data), \
886 }
887
888#define Z_RTIO_SQE_POOL_DEFINE(name, sz) \
889 static struct rtio_iodev_sqe CONCAT(_sqe_pool_, name)[sz]; \
890 STRUCT_SECTION_ITERABLE(rtio_sqe_pool, name) = { \
891 .free_q = MPSC_INIT((name.free_q)), \
892 .pool_size = sz, \
893 .pool_free = sz, \
894 .pool = CONCAT(_sqe_pool_, name), \
895 }
896
897
898#define Z_RTIO_CQE_POOL_DEFINE(name, sz) \
899 static struct rtio_cqe CONCAT(_cqe_pool_, name)[sz]; \
900 STRUCT_SECTION_ITERABLE(rtio_cqe_pool, name) = { \
901 .free_q = MPSC_INIT((name.free_q)), \
902 .pool_size = sz, \
903 .pool_free = sz, \
904 .pool = CONCAT(_cqe_pool_, name), \
905 }
906
916#define RTIO_BMEM COND_CODE_1(CONFIG_USERSPACE, (K_APP_BMEM(rtio_partition) static), (static))
917
927#define RTIO_DMEM COND_CODE_1(CONFIG_USERSPACE, (K_APP_DMEM(rtio_partition) static), (static))
928
929#define Z_RTIO_BLOCK_POOL_DEFINE(name, blk_sz, blk_cnt, blk_align) \
930 RTIO_BMEM uint8_t __aligned(WB_UP(blk_align)) \
931 CONCAT(_block_pool_, name)[blk_cnt*WB_UP(blk_sz)]; \
932 _SYS_MEM_BLOCKS_DEFINE_WITH_EXT_BUF(name, WB_UP(blk_sz), blk_cnt, \
933 CONCAT(_block_pool_, name), RTIO_DMEM)
934
935#define Z_RTIO_DEFINE(name, _sqe_pool, _cqe_pool, _block_pool) \
936 IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, \
937 (static K_SEM_DEFINE(CONCAT(_submit_sem_, name), 0, K_SEM_MAX_LIMIT))) \
938 IF_ENABLED(CONFIG_RTIO_CONSUME_SEM, \
939 (static K_SEM_DEFINE(CONCAT(_consume_sem_, name), 0, K_SEM_MAX_LIMIT))) \
940 STRUCT_SECTION_ITERABLE(rtio, name) = { \
941 IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, (.submit_sem = &CONCAT(_submit_sem_, name),)) \
942 IF_ENABLED(CONFIG_RTIO_SUBMIT_SEM, (.submit_count = 0,)) \
943 IF_ENABLED(CONFIG_RTIO_CONSUME_SEM, (.consume_sem = &CONCAT(_consume_sem_, name),))\
944 .cq_count = ATOMIC_INIT(0), \
945 .xcqcnt = ATOMIC_INIT(0), \
946 .sqe_pool = _sqe_pool, \
947 .cqe_pool = _cqe_pool, \
948 IF_ENABLED(CONFIG_RTIO_SYS_MEM_BLOCKS, (.block_pool = _block_pool,)) \
949 .sq = MPSC_INIT((name.sq)), \
950 .cq = MPSC_INIT((name.cq)), \
951 }
952
960#define RTIO_DEFINE(name, sq_sz, cq_sz) \
961 Z_RTIO_SQE_POOL_DEFINE(CONCAT(name, _sqe_pool), sq_sz); \
962 Z_RTIO_CQE_POOL_DEFINE(CONCAT(name, _cqe_pool), cq_sz); \
963 Z_RTIO_DEFINE(name, &CONCAT(name, _sqe_pool), \
964 &CONCAT(name, _cqe_pool), NULL)
965
966/* clang-format on */
967
978#define RTIO_DEFINE_WITH_MEMPOOL(name, sq_sz, cq_sz, num_blks, blk_size, balign) \
979 Z_RTIO_SQE_POOL_DEFINE(name##_sqe_pool, sq_sz); \
980 Z_RTIO_CQE_POOL_DEFINE(name##_cqe_pool, cq_sz); \
981 Z_RTIO_BLOCK_POOL_DEFINE(name##_block_pool, blk_size, num_blks, balign); \
982 Z_RTIO_DEFINE(name, &name##_sqe_pool, &name##_cqe_pool, &name##_block_pool)
983
984/* clang-format on */
985
993static inline uint32_t rtio_sqe_acquirable(struct rtio *r)
994{
995 return r->sqe_pool->pool_free;
996}
997
1006static inline struct rtio_iodev_sqe *rtio_txn_next(const struct rtio_iodev_sqe *iodev_sqe)
1007{
1008 if (iodev_sqe->sqe.flags & RTIO_SQE_TRANSACTION) {
1009 return iodev_sqe->next;
1010 } else {
1011 return NULL;
1012 }
1013}
1014
1015
1024static inline struct rtio_iodev_sqe *rtio_chain_next(const struct rtio_iodev_sqe *iodev_sqe)
1025{
1026 if (iodev_sqe->sqe.flags & RTIO_SQE_CHAINED) {
1027 return iodev_sqe->next;
1028 } else {
1029 return NULL;
1030 }
1031}
1032
1041static inline struct rtio_iodev_sqe *rtio_iodev_sqe_next(const struct rtio_iodev_sqe *iodev_sqe)
1042{
1043 return iodev_sqe->next;
1044}
1045
1054static inline struct rtio_sqe *rtio_sqe_acquire(struct rtio *r)
1055{
1056 struct rtio_iodev_sqe *iodev_sqe = rtio_sqe_pool_alloc(r->sqe_pool);
1057
1058 if (iodev_sqe == NULL) {
1059 return NULL;
1060 }
1061
1062 mpsc_push(&r->sq, &iodev_sqe->q);
1063
1064 return &iodev_sqe->sqe;
1065}
1066
1072static inline void rtio_sqe_drop_all(struct rtio *r)
1073{
1074 struct rtio_iodev_sqe *iodev_sqe;
1075 struct mpsc_node *node = mpsc_pop(&r->sq);
1076
1077 while (node != NULL) {
1078 iodev_sqe = CONTAINER_OF(node, struct rtio_iodev_sqe, q);
1079 rtio_sqe_pool_free(r->sqe_pool, iodev_sqe);
1080 node = mpsc_pop(&r->sq);
1081 }
1082}
1083
1087static inline struct rtio_cqe *rtio_cqe_acquire(struct rtio *r)
1088{
1089 struct rtio_cqe *cqe = rtio_cqe_pool_alloc(r->cqe_pool);
1090
1091 if (cqe == NULL) {
1092 return NULL;
1093 }
1094
1095 memset(cqe, 0, sizeof(struct rtio_cqe));
1096
1097 return cqe;
1098}
1099
1103static inline void rtio_cqe_produce(struct rtio *r, struct rtio_cqe *cqe)
1104{
1105 mpsc_push(&r->cq, &cqe->q);
1106}
1107
1119static inline struct rtio_cqe *rtio_cqe_consume(struct rtio *r)
1120{
1121 struct mpsc_node *node;
1122 struct rtio_cqe *cqe = NULL;
1123
1124#ifdef CONFIG_RTIO_CONSUME_SEM
1125 if (k_sem_take(r->consume_sem, K_NO_WAIT) != 0) {
1126 return NULL;
1127 }
1128#endif
1129
1130 node = mpsc_pop(&r->cq);
1131 if (node == NULL) {
1132 return NULL;
1133 }
1134 cqe = CONTAINER_OF(node, struct rtio_cqe, q);
1135
1136 return cqe;
1137}
1138
1149static inline struct rtio_cqe *rtio_cqe_consume_block(struct rtio *r)
1150{
1151 struct mpsc_node *node;
1152 struct rtio_cqe *cqe;
1153
1154#ifdef CONFIG_RTIO_CONSUME_SEM
1155 k_sem_take(r->consume_sem, K_FOREVER);
1156#endif
1157 node = mpsc_pop(&r->cq);
1158 while (node == NULL) {
1159 Z_SPIN_DELAY(1);
1160 node = mpsc_pop(&r->cq);
1161 }
1162 cqe = CONTAINER_OF(node, struct rtio_cqe, q);
1163
1164 return cqe;
1165}
1166
1173static inline void rtio_cqe_release(struct rtio *r, struct rtio_cqe *cqe)
1174{
1175 rtio_cqe_pool_free(r->cqe_pool, cqe);
1176}
1177
1186static inline int rtio_flush_completion_queue(struct rtio *r)
1187{
1188 struct rtio_cqe *cqe;
1189 int res = 0;
1190
1191 do {
1192 cqe = rtio_cqe_consume(r);
1193 if (cqe != NULL) {
1194 if ((cqe->result < 0) && (res == 0)) {
1195 res = cqe->result;
1196 }
1197 rtio_cqe_release(r, cqe);
1198 }
1199 } while (cqe != NULL);
1200
1201 return res;
1202}
1203
1210static inline uint32_t rtio_cqe_compute_flags(struct rtio_iodev_sqe *iodev_sqe)
1211{
1212 uint32_t flags = 0;
1213
1214#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1215 if (iodev_sqe->sqe.op == RTIO_OP_RX && iodev_sqe->sqe.flags & RTIO_SQE_MEMPOOL_BUFFER) {
1216 struct rtio *r = iodev_sqe->r;
1217 struct sys_mem_blocks *mem_pool = r->block_pool;
1218 unsigned int blk_index = 0;
1219 unsigned int blk_count = 0;
1220
1221 if (iodev_sqe->sqe.rx.buf) {
1222 blk_index = (iodev_sqe->sqe.rx.buf - mem_pool->buffer) >>
1223 mem_pool->info.blk_sz_shift;
1224 blk_count = iodev_sqe->sqe.rx.buf_len >> mem_pool->info.blk_sz_shift;
1225 }
1226 flags = RTIO_CQE_FLAG_PREP_MEMPOOL(blk_index, blk_count);
1227 }
1228#else
1229 ARG_UNUSED(iodev_sqe);
1230#endif
1231
1232 return flags;
1233}
1234
1250__syscall int rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe,
1251 uint8_t **buff, uint32_t *buff_len);
1252
1253static inline int z_impl_rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe,
1254 uint8_t **buff, uint32_t *buff_len)
1255{
1256#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1258 unsigned int blk_idx = RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(cqe->flags);
1259 unsigned int blk_count = RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(cqe->flags);
1261
1262 *buff_len = blk_count * blk_size;
1263
1264 if (blk_count > 0) {
1265 *buff = r->block_pool->buffer + blk_idx * blk_size;
1266
1267 __ASSERT_NO_MSG(*buff >= r->block_pool->buffer);
1268 __ASSERT_NO_MSG(*buff <
1269 r->block_pool->buffer + blk_size * r->block_pool->info.num_blocks);
1270 } else {
1271 *buff = NULL;
1272 }
1273 return 0;
1274 }
1275 return -EINVAL;
1276#else
1277 ARG_UNUSED(r);
1278 ARG_UNUSED(cqe);
1279 ARG_UNUSED(buff);
1280 ARG_UNUSED(buff_len);
1281
1282 return -ENOTSUP;
1283#endif
1284}
1285
1287void rtio_executor_ok(struct rtio_iodev_sqe *iodev_sqe, int result);
1288void rtio_executor_err(struct rtio_iodev_sqe *iodev_sqe, int result);
1289
1298static inline void rtio_iodev_sqe_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
1299{
1300 rtio_executor_ok(iodev_sqe, result);
1301}
1302
1311static inline void rtio_iodev_sqe_err(struct rtio_iodev_sqe *iodev_sqe, int result)
1312{
1313 rtio_executor_err(iodev_sqe, result);
1314}
1315
1327static inline void rtio_cqe_submit(struct rtio *r, int result, void *userdata, uint32_t flags)
1328{
1329 struct rtio_cqe *cqe = rtio_cqe_acquire(r);
1330
1331 if (cqe == NULL) {
1332 atomic_inc(&r->xcqcnt);
1333 } else {
1334 cqe->result = result;
1335 cqe->userdata = userdata;
1336 cqe->flags = flags;
1337 rtio_cqe_produce(r, cqe);
1338#ifdef CONFIG_RTIO_CONSUME_SEM
1339 k_sem_give(r->consume_sem);
1340#endif
1341 }
1342
1343 /* atomic_t isn't guaranteed to wrap correctly as it could be signed, so
1344 * we must resort to a cas loop.
1345 */
1346 atomic_t val, new_val;
1347
1348 do {
1349 val = atomic_get(&r->cq_count);
1350 new_val = (atomic_t)((uintptr_t)val + 1);
1351 } while (!atomic_cas(&r->cq_count, val, new_val));
1352
1353#ifdef CONFIG_RTIO_SUBMIT_SEM
1354 if (r->submit_count > 0) {
1355 r->submit_count--;
1356 if (r->submit_count == 0) {
1357 k_sem_give(r->submit_sem);
1358 }
1359 }
1360#endif
1361}
1362
1363#define __RTIO_MEMPOOL_GET_NUM_BLKS(num_bytes, blk_size) (((num_bytes) + (blk_size)-1) / (blk_size))
1364
1377static inline int rtio_sqe_rx_buf(const struct rtio_iodev_sqe *iodev_sqe, uint32_t min_buf_len,
1378 uint32_t max_buf_len, uint8_t **buf, uint32_t *buf_len)
1379{
1380 struct rtio_sqe *sqe = (struct rtio_sqe *)&iodev_sqe->sqe;
1381
1382#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1383 if (sqe->op == RTIO_OP_RX && sqe->flags & RTIO_SQE_MEMPOOL_BUFFER) {
1384 struct rtio *r = iodev_sqe->r;
1385
1386 if (sqe->rx.buf != NULL) {
1387 if (sqe->rx.buf_len < min_buf_len) {
1388 return -ENOMEM;
1389 }
1390 *buf = sqe->rx.buf;
1391 *buf_len = sqe->rx.buf_len;
1392 return 0;
1393 }
1394
1395 int rc = rtio_block_pool_alloc(r, min_buf_len, max_buf_len, buf, buf_len);
1396 if (rc == 0) {
1397 sqe->rx.buf = *buf;
1398 sqe->rx.buf_len = *buf_len;
1399 return 0;
1400 }
1401
1402 return -ENOMEM;
1403 }
1404#else
1405 ARG_UNUSED(max_buf_len);
1406#endif
1407
1408 if (sqe->rx.buf_len < min_buf_len) {
1409 return -ENOMEM;
1410 }
1411
1412 *buf = sqe->rx.buf;
1413 *buf_len = sqe->rx.buf_len;
1414 return 0;
1415}
1416
1431__syscall void rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len);
1432
1433static inline void z_impl_rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len)
1434{
1435#ifdef CONFIG_RTIO_SYS_MEM_BLOCKS
1436 if (r == NULL || buff == NULL || r->block_pool == NULL || buff_len == 0) {
1437 return;
1438 }
1439
1440 rtio_block_pool_free(r, buff, buff_len);
1441#else
1442 ARG_UNUSED(r);
1443 ARG_UNUSED(buff);
1444 ARG_UNUSED(buff_len);
1445#endif
1446}
1447
1454static inline void rtio_access_grant(struct rtio *r, struct k_thread *t)
1455{
1457
1458#ifdef CONFIG_RTIO_SUBMIT_SEM
1459 k_object_access_grant(r->submit_sem, t);
1460#endif
1461
1462#ifdef CONFIG_RTIO_CONSUME_SEM
1463 k_object_access_grant(r->consume_sem, t);
1464#endif
1465}
1466
1467
1474static inline void rtio_access_revoke(struct rtio *r, struct k_thread *t)
1475{
1477
1478#ifdef CONFIG_RTIO_SUBMIT_SEM
1479 k_object_access_revoke(r->submit_sem, t);
1480#endif
1481
1482#ifdef CONFIG_RTIO_CONSUME_SEM
1483 k_object_access_revoke(r->consume_sem, t);
1484#endif
1485}
1486
1497__syscall int rtio_sqe_cancel(struct rtio_sqe *sqe);
1498
1499static inline int z_impl_rtio_sqe_cancel(struct rtio_sqe *sqe)
1500{
1501 struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe);
1502
1503 do {
1504 iodev_sqe->sqe.flags |= RTIO_SQE_CANCELED;
1505 iodev_sqe = rtio_iodev_sqe_next(iodev_sqe);
1506 } while (iodev_sqe != NULL);
1507
1508 return 0;
1509}
1510
1522__syscall void rtio_sqe_signal(struct rtio_sqe *sqe);
1523
1524static inline void z_impl_rtio_sqe_signal(struct rtio_sqe *sqe)
1525{
1526 struct rtio_iodev_sqe *iodev_sqe = CONTAINER_OF(sqe, struct rtio_iodev_sqe, sqe);
1527
1528 if (!atomic_cas(&iodev_sqe->sqe.await.ok, 0, 1)) {
1529 iodev_sqe->sqe.await.callback(iodev_sqe, iodev_sqe->sqe.await.userdata);
1530 }
1531}
1532
1543static inline void rtio_iodev_sqe_await_signal(struct rtio_iodev_sqe *iodev_sqe,
1544 rtio_signaled_t callback,
1545 void *userdata)
1546{
1547 iodev_sqe->sqe.await.callback = callback;
1548 iodev_sqe->sqe.await.userdata = userdata;
1549
1550 if (!atomic_cas(&iodev_sqe->sqe.await.ok, 0, 1)) {
1551 callback(iodev_sqe, userdata);
1552 }
1553}
1554
1570__syscall int rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes,
1571 struct rtio_sqe **handle, size_t sqe_count);
1572
1573static inline int z_impl_rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes,
1574 struct rtio_sqe **handle,
1575 size_t sqe_count)
1576{
1577 struct rtio_sqe *sqe;
1578 uint32_t acquirable = rtio_sqe_acquirable(r);
1579
1580 if (acquirable < sqe_count) {
1581 return -ENOMEM;
1582 }
1583
1584 for (unsigned long i = 0; i < sqe_count; i++) {
1585 sqe = rtio_sqe_acquire(r);
1586 __ASSERT_NO_MSG(sqe != NULL);
1587 if (handle != NULL && i == 0) {
1588 *handle = sqe;
1589 }
1590 *sqe = sqes[i];
1591 }
1592
1593 return 0;
1594}
1595
1612static inline int rtio_sqe_copy_in(struct rtio *r, const struct rtio_sqe *sqes, size_t sqe_count)
1613{
1614 return rtio_sqe_copy_in_get_handles(r, sqes, NULL, sqe_count);
1615}
1616
1632__syscall int rtio_cqe_copy_out(struct rtio *r,
1633 struct rtio_cqe *cqes,
1634 size_t cqe_count,
1636static inline int z_impl_rtio_cqe_copy_out(struct rtio *r,
1637 struct rtio_cqe *cqes,
1638 size_t cqe_count,
1640{
1641 size_t copied = 0;
1642 struct rtio_cqe *cqe;
1643 k_timepoint_t end = sys_timepoint_calc(timeout);
1644
1645 do {
1648 if (cqe == NULL) {
1649 Z_SPIN_DELAY(25);
1650 continue;
1651 }
1652 cqes[copied++] = *cqe;
1653 rtio_cqe_release(r, cqe);
1654 } while (copied < cqe_count && !sys_timepoint_expired(end));
1655
1656 return copied;
1657}
1658
1674__syscall int rtio_submit(struct rtio *r, uint32_t wait_count);
1675
1676#ifdef CONFIG_RTIO_SUBMIT_SEM
1677static inline int z_impl_rtio_submit(struct rtio *r, uint32_t wait_count)
1678{
1679 int res = 0;
1680
1681 if (wait_count > 0) {
1682 __ASSERT(!k_is_in_isr(),
1683 "expected rtio submit with wait count to be called from a thread");
1684
1685 k_sem_reset(r->submit_sem);
1686 r->submit_count = wait_count;
1687 }
1688
1690
1691 if (wait_count > 0) {
1692 res = k_sem_take(r->submit_sem, K_FOREVER);
1693 __ASSERT(res == 0,
1694 "semaphore was reset or timed out while waiting on completions!");
1695 }
1696
1697 return res;
1698}
1699#else
1700static inline int z_impl_rtio_submit(struct rtio *r, uint32_t wait_count)
1701{
1702
1703 int res = 0;
1704 uintptr_t cq_count = (uintptr_t)atomic_get(&r->cq_count);
1705 uintptr_t cq_complete_count = cq_count + wait_count;
1706 bool wraps = cq_complete_count < cq_count;
1707
1709
1710 if (wraps) {
1711 while ((uintptr_t)atomic_get(&r->cq_count) >= cq_count) {
1712 Z_SPIN_DELAY(10);
1713 k_yield();
1714 }
1715 }
1716
1717 while ((uintptr_t)atomic_get(&r->cq_count) < cq_complete_count) {
1718 Z_SPIN_DELAY(10);
1719 k_yield();
1720 }
1721
1722 return res;
1723}
1724#endif /* CONFIG_RTIO_SUBMIT_SEM */
1725
1732
1734 struct rtio **contexts;
1735
1738};
1739
1748__syscall struct rtio *rtio_pool_acquire(struct rtio_pool *pool);
1749
1750static inline struct rtio *z_impl_rtio_pool_acquire(struct rtio_pool *pool)
1751{
1752 struct rtio *r = NULL;
1753
1754 for (size_t i = 0; i < pool->pool_size; i++) {
1755 if (atomic_test_and_set_bit(pool->used, i) == 0) {
1756 r = pool->contexts[i];
1757 break;
1758 }
1759 }
1760
1761 if (r != NULL) {
1763 }
1764
1765 return r;
1766}
1767
1774__syscall void rtio_pool_release(struct rtio_pool *pool, struct rtio *r);
1775
1776static inline void z_impl_rtio_pool_release(struct rtio_pool *pool, struct rtio *r)
1777{
1778
1779 if (k_is_user_context()) {
1781 }
1782
1783 for (size_t i = 0; i < pool->pool_size; i++) {
1784 if (pool->contexts[i] == r) {
1785 atomic_clear_bit(pool->used, i);
1786 break;
1787 }
1788 }
1789}
1790
1791/* clang-format off */
1792
1794
1795#define Z_RTIO_POOL_NAME_N(n, name) \
1796 name##_##n
1797
1798#define Z_RTIO_POOL_DEFINE_N(n, name, sq_sz, cq_sz) \
1799 RTIO_DEFINE(Z_RTIO_POOL_NAME_N(n, name), sq_sz, cq_sz)
1800
1801#define Z_RTIO_POOL_REF_N(n, name) \
1802 &Z_RTIO_POOL_NAME_N(n, name)
1803
1805
1814#define RTIO_POOL_DEFINE(name, pool_sz, sq_sz, cq_sz) \
1815 LISTIFY(pool_sz, Z_RTIO_POOL_DEFINE_N, (;), name, sq_sz, cq_sz); \
1816 static struct rtio *name##_contexts[] = { \
1817 LISTIFY(pool_sz, Z_RTIO_POOL_REF_N, (,), name) \
1818 }; \
1819 ATOMIC_DEFINE(name##_used, pool_sz); \
1820 STRUCT_SECTION_ITERABLE(rtio_pool, name) = { \
1821 .pool_size = pool_sz, \
1822 .contexts = name##_contexts, \
1823 .used = name##_used, \
1824 }
1825
1826/* clang-format on */
1827
1831
1832#ifdef __cplusplus
1833}
1834#endif
1835
1836#include <zephyr/syscalls/rtio.h>
1837
1838#endif /* ZEPHYR_INCLUDE_RTIO_RTIO_H_ */
workaround assembler barfing for ST r
Definition asm-macro-32-bit-gnu.h:24
long atomic_t
Definition atomic_types.h:15
static void atomic_clear_bit(atomic_t *target, int bit)
Atomically clear a bit.
Definition atomic.h:191
atomic_val_t atomic_get(const atomic_t *target)
Atomic get.
static bool atomic_test_and_set_bit(atomic_t *target, int bit)
Atomically set a bit and test it.
Definition atomic.h:170
atomic_val_t atomic_inc(atomic_t *target)
Atomic increment.
bool atomic_cas(atomic_t *target, atomic_val_t old_value, atomic_val_t new_value)
Atomic compare-and-set.
#define K_FOREVER
Generate infinite timeout delay.
Definition kernel.h:1481
#define K_NO_WAIT
Generate null timeout delay.
Definition kernel.h:1371
k_timepoint_t sys_timepoint_calc(k_timeout_t timeout)
Calculate a timepoint value.
static bool sys_timepoint_expired(k_timepoint_t timepoint)
Indicates if timepoint is expired.
Definition clock.h:339
#define K_TIMEOUT_EQ(a, b)
Compare timeouts for equality.
Definition clock.h:80
bool k_is_in_isr(void)
Determine if code is running at interrupt level.
int sys_mem_blocks_free_contiguous(sys_mem_blocks_t *mem_block, void *block, size_t count)
Free contiguous multiple memory blocks.
int sys_mem_blocks_alloc_contiguous(sys_mem_blocks_t *mem_block, size_t count, void **out_block)
Allocate a contiguous set of memory blocks.
static ALWAYS_INLINE void mpsc_push(struct mpsc *q, struct mpsc_node *n)
Push a node.
Definition mpsc_lockfree.h:126
static struct mpsc_node * mpsc_pop(struct mpsc *q)
Pop a node off of the list.
Definition mpsc_lockfree.h:145
#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_CNT(flags)
Get the block count of a mempool flags.
Definition rtio.h:179
#define RTIO_CQE_FLAG_MEMPOOL_GET_BLK_IDX(flags)
Get the block index of a mempool flags.
Definition rtio.h:171
#define RTIO_CQE_FLAG_MEMPOOL_BUFFER
The entry's buffer was allocated from the RTIO's mempool.
Definition rtio.h:161
#define RTIO_CQE_FLAG_PREP_MEMPOOL(blk_idx, blk_cnt)
Prepare CQE flags for a mempool read.
Definition rtio.h:188
#define RTIO_CQE_FLAG_GET(flags)
Definition rtio.h:163
#define RTIO_SQE_MULTISHOT
The SQE should continue producing CQEs until canceled.
Definition rtio.h:137
#define RTIO_SQE_TRANSACTION
The next request in the queue is part of a transaction.
Definition rtio.h:109
#define RTIO_SQE_MEMPOOL_BUFFER
The buffer should be allocated by the RTIO mempool.
Definition rtio.h:121
#define RTIO_SQE_CANCELED
The SQE should not execute if possible.
Definition rtio.h:129
#define RTIO_SQE_NO_RESPONSE
The SQE does not produce a CQE.
Definition rtio.h:142
#define RTIO_SQE_CHAINED
The next request in the queue should wait on this one.
Definition rtio.h:97
void rtio_pool_release(struct rtio_pool *pool, struct rtio *r)
Return an RTIO context to a pool.
static void rtio_sqe_prep_read_with_pool(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, void *userdata)
Prepare a read op submission with context's mempool.
Definition rtio.h:623
void rtio_executor_err(struct rtio_iodev_sqe *iodev_sqe, int result)
#define RTIO_OP_CALLBACK
An operation that calls a given function (callback)
Definition rtio.h:560
static uint32_t rtio_sqe_acquirable(struct rtio *r)
Count of acquirable submission queue events.
Definition rtio.h:993
static void rtio_sqe_prep_delay(struct rtio_sqe *sqe, k_timeout_t timeout, void *userdata)
Definition rtio.h:760
static void rtio_cqe_pool_free(struct rtio_cqe_pool *pool, struct rtio_cqe *cqe)
Definition rtio.h:811
static void rtio_sqe_prep_tiny_write(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, const uint8_t *tiny_write_data, uint8_t tiny_write_len, void *userdata)
Prepare a tiny write op submission.
Definition rtio.h:668
struct rtio * rtio_pool_acquire(struct rtio_pool *pool)
Obtain an RTIO context from a pool.
static size_t rtio_mempool_block_size(const struct rtio *r)
Get the mempool block size of the RTIO context.
Definition rtio.h:472
static void rtio_cqe_submit(struct rtio *r, int result, void *userdata, uint32_t flags)
Submit a completion queue event with a given result and userdata.
Definition rtio.h:1327
static void rtio_sqe_prep_nop(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, void *userdata)
Prepare a nop (no op) submission.
Definition rtio.h:589
void rtio_release_buffer(struct rtio *r, void *buff, uint32_t buff_len)
Release memory that was allocated by the RTIO's memory pool.
static int rtio_sqe_copy_in(struct rtio *r, const struct rtio_sqe *sqes, size_t sqe_count)
Copy an array of SQEs into the queue.
Definition rtio.h:1612
static void rtio_cqe_produce(struct rtio *r, struct rtio_cqe *cqe)
Produce a complete queue event if available.
Definition rtio.h:1103
#define RTIO_OP_TINY_TX
An operation that transmits tiny writes by copying the data to write.
Definition rtio.h:557
static uint32_t rtio_cqe_compute_flags(struct rtio_iodev_sqe *iodev_sqe)
Compute the CQE flags from the rtio_iodev_sqe entry.
Definition rtio.h:1210
static void rtio_iodev_sqe_await_signal(struct rtio_iodev_sqe *iodev_sqe, rtio_signaled_t callback, void *userdata)
Await an AWAIT SQE signal from RTIO IODEV.
Definition rtio.h:1543
void rtio_executor_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
static int rtio_block_pool_alloc(struct rtio *r, size_t min_sz, size_t max_sz, uint8_t **buf, uint32_t *buf_len)
Definition rtio.h:818
static void rtio_sqe_prep_write(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, const uint8_t *buf, uint32_t len, void *userdata)
Prepare a write op submission.
Definition rtio.h:642
int rtio_sqe_copy_in_get_handles(struct rtio *r, const struct rtio_sqe *sqes, struct rtio_sqe **handle, size_t sqe_count)
Copy an array of SQEs into the queue and get resulting handles back.
static struct rtio_cqe * rtio_cqe_pool_alloc(struct rtio_cqe_pool *pool)
Definition rtio.h:794
struct k_mem_partition rtio_partition
The memory partition associated with all RTIO context information.
static void rtio_sqe_prep_read(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, uint8_t *buf, uint32_t len, void *userdata)
Prepare a read op submission.
Definition rtio.h:602
static struct rtio_sqe * rtio_sqe_acquire(struct rtio *r)
Acquire a single submission queue event if available.
Definition rtio.h:1054
#define RTIO_OP_TX
An operation that transmits (writes)
Definition rtio.h:554
static void rtio_sqe_drop_all(struct rtio *r)
Drop all previously acquired sqe.
Definition rtio.h:1072
static void rtio_sqe_prep_read_multishot(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, void *userdata)
Definition rtio.h:631
int rtio_cqe_copy_out(struct rtio *r, struct rtio_cqe *cqes, size_t cqe_count, k_timeout_t timeout)
Copy an array of CQEs from the queue.
static int rtio_flush_completion_queue(struct rtio *r)
Flush completion queue.
Definition rtio.h:1186
static void rtio_access_revoke(struct rtio *r, struct k_thread *t)
Revoke access to an RTIO context from a user thread.
Definition rtio.h:1474
static void rtio_sqe_prep_callback(struct rtio_sqe *sqe, rtio_callback_t callback, void *arg0, void *userdata)
Prepare a callback op submission.
Definition rtio.h:694
static void rtio_access_grant(struct rtio *r, struct k_thread *t)
Grant access to an RTIO context to a user thread.
Definition rtio.h:1454
#define RTIO_OP_TXRX
An operation that transceives (reads and writes simultaneously)
Definition rtio.h:563
static void rtio_cqe_release(struct rtio *r, struct rtio_cqe *cqe)
Release consumed completion queue event.
Definition rtio.h:1173
static int rtio_sqe_rx_buf(const struct rtio_iodev_sqe *iodev_sqe, uint32_t min_buf_len, uint32_t max_buf_len, uint8_t **buf, uint32_t *buf_len)
Get the buffer associate with the RX submission.
Definition rtio.h:1377
static void rtio_iodev_sqe_err(struct rtio_iodev_sqe *iodev_sqe, int result)
Inform the executor of a submissions completion with error.
Definition rtio.h:1311
void(* rtio_signaled_t)(struct rtio_iodev_sqe *iodev_sqe, void *userdata)
Callback signature for RTIO_OP_AWAIT signaled.
Definition rtio.h:290
static void rtio_sqe_prep_transceive(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, const uint8_t *tx_buf, uint8_t *rx_buf, uint32_t buf_len, void *userdata)
Prepare a transceive op submission.
Definition rtio.h:730
int rtio_sqe_cancel(struct rtio_sqe *sqe)
Attempt to cancel an SQE.
static void rtio_sqe_pool_free(struct rtio_sqe_pool *pool, struct rtio_iodev_sqe *iodev_sqe)
Definition rtio.h:787
static void rtio_iodev_sqe_ok(struct rtio_iodev_sqe *iodev_sqe, int result)
Inform the executor of a submission completion with success.
Definition rtio.h:1298
void(* rtio_callback_t)(struct rtio *r, const struct rtio_sqe *sqe, void *arg0)
Callback signature for RTIO_OP_CALLBACK.
Definition rtio.h:282
#define RTIO_OP_NOP
An operation that does nothing and will complete immediately.
Definition rtio.h:548
#define RTIO_OP_AWAIT
An operation to suspend bus while awaiting signal.
Definition rtio.h:584
static struct rtio_cqe * rtio_cqe_acquire(struct rtio *r)
Acquire a complete queue event if available.
Definition rtio.h:1087
static struct rtio_iodev_sqe * rtio_chain_next(const struct rtio_iodev_sqe *iodev_sqe)
Get the next sqe in the chain.
Definition rtio.h:1024
static struct rtio_cqe * rtio_cqe_consume(struct rtio *r)
Consume a single completion queue event if available.
Definition rtio.h:1119
static struct rtio_iodev_sqe * rtio_sqe_pool_alloc(struct rtio_sqe_pool *pool)
Definition rtio.h:772
void rtio_sqe_signal(struct rtio_sqe *sqe)
Signal an AWAIT SQE.
static struct rtio_iodev_sqe * rtio_iodev_sqe_next(const struct rtio_iodev_sqe *iodev_sqe)
Get the next sqe in the chain or transaction.
Definition rtio.h:1041
static void rtio_sqe_prep_callback_no_cqe(struct rtio_sqe *sqe, rtio_callback_t callback, void *arg0, void *userdata)
Prepare a callback op submission that does not create a CQE.
Definition rtio.h:718
#define RTIO_OP_DELAY
An operation that takes a specified amount of time (asynchronously) before completing.
Definition rtio.h:566
int rtio_cqe_get_mempool_buffer(const struct rtio *r, struct rtio_cqe *cqe, uint8_t **buff, uint32_t *buff_len)
Retrieve the mempool buffer that was allocated for the CQE.
static struct rtio_iodev_sqe * rtio_txn_next(const struct rtio_iodev_sqe *iodev_sqe)
Get the next sqe in the transaction.
Definition rtio.h:1006
void rtio_executor_submit(struct rtio *r)
static struct rtio_cqe * rtio_cqe_consume_block(struct rtio *r)
Wait for and consume a single completion queue event.
Definition rtio.h:1149
static void rtio_block_pool_free(struct rtio *r, void *buf, uint32_t buf_len)
Definition rtio.h:859
static void rtio_sqe_prep_await(struct rtio_sqe *sqe, const struct rtio_iodev *iodev, int8_t prio, void *userdata)
Definition rtio.h:748
#define RTIO_OP_RX
An operation that receives (reads)
Definition rtio.h:551
int rtio_submit(struct rtio *r, uint32_t wait_count)
Submit I/O requests to the underlying executor.
void k_sem_reset(struct k_sem *sem)
Resets a semaphore's count to zero.
void k_sem_give(struct k_sem *sem)
Give a semaphore.
int k_sem_take(struct k_sem *sem, k_timeout_t timeout)
Take a semaphore.
#define BIT(n)
Unsigned integer with bit position n set (signed in assembly language).
Definition util_macro.h:44
#define CONTAINER_OF(ptr, type, field)
Get a pointer to a structure containing the element.
Definition util.h:285
#define DIV_ROUND_UP(n, d)
Divide and round up.
Definition util.h:353
#define EINVAL
Invalid argument.
Definition errno.h:60
#define ENOMEM
Not enough core.
Definition errno.h:50
#define ENOTSUP
Unsupported value.
Definition errno.h:114
void k_yield(void)
Yield the current thread.
static __attribute_const__ k_tid_t k_current_get(void)
Get thread ID of the current thread.
Definition kernel.h:676
void k_object_access_grant(const void *object, struct k_thread *thread)
Grant a thread access to a kernel object.
void k_object_access_revoke(const void *object, struct k_thread *thread)
Revoke a thread's access to a kernel object.
#define NULL
Definition iar_missing_defs.h:20
Public kernel APIs.
Memory Blocks Allocator.
A wait-free intrusive multi producer single consumer (MPSC) queue using a singly linked list.
flags
Definition parser.h:97
__UINT32_TYPE__ uint32_t
Definition stdint.h:90
__INT32_TYPE__ int32_t
Definition stdint.h:74
__UINT8_TYPE__ uint8_t
Definition stdint.h:88
#define UINT16_MAX
Definition stdint.h:28
__UINTPTR_TYPE__ uintptr_t
Definition stdint.h:105
__UINT16_TYPE__ uint16_t
Definition stdint.h:89
__INT8_TYPE__ int8_t
Definition stdint.h:72
void * memset(void *buf, int c, size_t n)
void * memcpy(void *ZRESTRICT d, const void *ZRESTRICT s, size_t n)
Memory Partition.
Definition mem_domain.h:55
Semaphore structure.
Definition kernel.h:3275
Thread Structure.
Definition thread.h:262
Kernel timeout type.
Definition clock.h:65
Kernel timepoint type.
Definition clock.h:242
Queue member.
Definition mpsc_lockfree.h:79
MPSC Queue.
Definition mpsc_lockfree.h:86
Definition rtio.h:400
struct rtio_cqe * pool
Definition rtio.h:404
struct mpsc free_q
Definition rtio.h:401
const uint16_t pool_size
Definition rtio.h:402
uint16_t pool_free
Definition rtio.h:403
A completion queue event.
Definition rtio.h:385
void * userdata
Associated userdata with operation.
Definition rtio.h:389
struct mpsc_node q
Definition rtio.h:386
uint32_t flags
Flags associated with the operation.
Definition rtio.h:390
int32_t result
Result from operation.
Definition rtio.h:388
API that an RTIO IO device should implement.
Definition rtio.h:524
void(* submit)(struct rtio_iodev_sqe *iodev_sqe)
Submit to the iodev an entry to work on.
Definition rtio.h:533
Compute the mempool block index for a given pointer.
Definition rtio.h:514
struct rtio_iodev_sqe * next
Definition rtio.h:517
struct rtio_sqe sqe
Definition rtio.h:515
struct rtio * r
Definition rtio.h:518
struct mpsc_node q
Definition rtio.h:516
An IO device with a function table for submitting requests.
Definition rtio.h:539
const struct rtio_iodev_api * api
Definition rtio.h:541
void * data
Definition rtio.h:544
Pool of RTIO contexts to use with dynamically created threads.
Definition rtio.h:1729
struct rtio ** contexts
Array containing contexts of the pool.
Definition rtio.h:1734
atomic_t * used
Atomic bitmap to signal a member is used/unused.
Definition rtio.h:1737
size_t pool_size
Size of the pool.
Definition rtio.h:1731
Definition rtio.h:393
struct rtio_iodev_sqe * pool
Definition rtio.h:397
const uint16_t pool_size
Definition rtio.h:395
struct mpsc free_q
Definition rtio.h:394
uint16_t pool_free
Definition rtio.h:396
A submission queue event.
Definition rtio.h:295
uint32_t i2c_config
OP_I2C_CONFIGURE.
Definition rtio.h:355
void * userdata
User provided data which is returned upon operation completion.
Definition rtio.h:313
const uint8_t * tx_buf
Buffer to write from.
Definition rtio.h:344
struct rtio_sqe::@346015370260157122324174060242055067274246076272::@156103043324343070111363144011175302340145332235 tiny_tx
OP_TINY_TX.
uint8_t op
Op code.
Definition rtio.h:296
struct rtio_sqe::@346015370260157122324174060242055067274246076272::@133276262235321357167246345002275273273102345261 rx
OP_RX.
void * arg0
Last argument given to callback.
Definition rtio.h:338
struct rtio_sqe::@346015370260157122324174060242055067274246076272::@004235137221060376063310265133374117312204154130 tx
OP_TX.
atomic_t ok
Definition rtio.h:370
uint8_t * rx_buf
Buffer to read into.
Definition rtio.h:345
uint8_t prio
Op priority.
Definition rtio.h:298
k_timeout_t timeout
Delay timeout.
Definition rtio.h:350
struct _timeout to
Timeout struct.
Definition rtio.h:351
struct rtio_sqe::@346015370260157122324174060242055067274246076272::@324335144354230362066357340057163170350241057343 txrx
OP_TXRX.
uint32_t buf_len
Length of buffer.
Definition rtio.h:319
const struct rtio_iodev * iodev
Device to operation on.
Definition rtio.h:304
struct rtio_sqe::@346015370260157122324174060242055067274246076272::@377160164316265207047041027316236163055075120351 await
OP_AWAIT.
uint32_t iodev_flags
Op iodev flags.
Definition rtio.h:302
void * ccc_payload
OP_I3C_CCC.
Definition rtio.h:366
struct rtio_sqe::@346015370260157122324174060242055067274246076272::@242110314202367053245316142070211303126322345320 delay
OP_DELAY.
int type
Definition rtio.h:360
uint16_t flags
Op Flags.
Definition rtio.h:300
const uint8_t * buf
Buffer to write from.
Definition rtio.h:320
void * config
Definition rtio.h:361
rtio_callback_t callback
Definition rtio.h:337
An RTIO context containing what can be viewed as a pair of queues.
Definition rtio.h:418
struct rtio_cqe_pool * cqe_pool
Definition rtio.h:448
struct mpsc sq
Definition rtio.h:456
atomic_t cq_count
Definition rtio.h:437
struct rtio_sqe_pool * sqe_pool
Definition rtio.h:445
atomic_t xcqcnt
Definition rtio.h:442
struct mpsc cq
Definition rtio.h:459
Misc utilities.
static __pinned_func bool k_is_user_context(void)
Indicate whether the CPU is currently in user mode.
Definition syscall.h:115