/* * IO uring */ #pragma once typedef enum : int32_t { KPL_IO_URING_QUEUE_SIZE_POW2_MIN = 12, KPL_IO_URING_QUEUE_SIZE_POW2_MAX = 20, KPL_IO_URING_QUEUE_SIZE_POW2_DEFAULT = 15 } kpl_io_uring_enum; /* * Per thread io_uring container, do not use across threads * Operations are added to the submit queue by a functions specific to that operation * Once one or more operations are added, call kpl_io_uring_submit to send the operations to the kernel * Only one completed operation can be read at a time by kpl_io_uring_get_completed * Uses defaults that require kernel version of 6.6 or greater */ typedef struct { int32_t ring_fd, ring_queue_entries; // fd for submit, number of queue slots uint32_t added_to_queue, submitted_to_kernel; //added to submission queue, added with io uring enter uint32_t sq_cq_size, sqes_size; // using single mmap for kernel submission and completion uint32_t *sring_head, *sring_tail, *sring_mask; uint32_t *cring_head, *cring_tail, *cring_mask; void *sring_ptr, *cring_ptr; struct io_uring_sqe *sqes; struct io_uring_cqe *cqes; struct io_uring_params params; } kpl_io_uring; // must be zero initialized /* * Called by owner thread */ int32_t kpl_io_uring_init(kpl_io_uring *io_uring, int32_t queue_size_pow2); /* * Be sure no operations are in queue before calling * Called by owner thread */ inline void kpl_io_uring_free(kpl_io_uring *io_uring) { munmap(io_uring->sring_ptr, io_uring->sq_cq_size); munmap(io_uring->sqes, io_uring->sqes_size); } /* * Reads one from cqes, will call io uring enter if none are complete and submitted_to_kernel is not zero * This will wait if io uring enter is called * NULL for none completed * If an error occurred, the error details will be on the kpl_task_object */ kpl_object_task *kpl_io_uring_get_completed(kpl_io_uring *io_uring); /* * Submits all sqes (added_to_queue has total) in one io uring eneter * This does not wait */ int32_t kpl_io_uring_submit(kpl_io_uring *io_uring); /* * Only call this though one of the functions below to add an operation to the queue */ struct io_uring_sqe *kpl_io_uring_op(kpl_io_uring *io_uring, kpl_object_task *task, int32_t op, int32_t fd, const void *addr, uint32_t len, uint64_t offset); inline bool kpl_io_uring_openat(kpl_io_uring *io_uring, kpl_object_task *task, int32_t dir_fd, const char *path, int32_t flags, mode_t mode) { struct io_uring_sqe *sqe = kpl_io_uring_op(io_uring, task, IORING_OP_OPENAT, dir_fd, path, mode, 0); if (sqe) sqe->open_flags = (uint32_t) flags; return sqe; } inline bool kpl_io_uring_close(kpl_io_uring *io_uring, kpl_object_task *task, int32_t fd) { return kpl_io_uring_op(io_uring, task, IORING_OP_CLOSE, fd, NULL, 0, 0); } inline bool kpl_io_uring_read(kpl_io_uring *io_uring, kpl_object_task *task, int32_t fd, void *buffer, uint32_t byte_size, uint64_t offset) { return kpl_io_uring_op(io_uring, task, IORING_OP_READ, fd, buffer, byte_size, offset); } inline bool kpl_io_uring_write(kpl_io_uring *uring, kpl_object_task *task, int32_t fd, void *buffer, uint32_t byte_size, uint64_t offset) { return kpl_io_uring_op(uring, task, IORING_OP_WRITE, fd, buffer, byte_size, offset); } inline bool kpl_io_uring_statx(kpl_io_uring *io_uring, kpl_object_task *task, int32_t dir_fd, const char *path, int32_t flags, uint32_t mask, struct statx *statx_object) { struct io_uring_sqe *sqe = kpl_io_uring_op(io_uring, task, IORING_OP_STATX, dir_fd, path, mask, (uint64_t) statx_object); if (sqe) sqe->statx_flags = (uint32_t) flags; return sqe; }