1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
/*
* IO uring
*/
#pragma once
typedef enum : int32_t {
KPL_IO_URING_QUEUE_SIZE_POW2_MIN = 12,
KPL_IO_URING_QUEUE_SIZE_POW2_MAX = 20,
KPL_IO_URING_QUEUE_SIZE_POW2_DEFAULT = 15
} kpl_io_uring_enum;
/*
* Per thread io_uring container, do not use across threads
* Operations are added to the submit queue by a functions specific to that operation
* Once one or more operations are added, call kpl_io_uring_submit to send the operations to the kernel
* Only one completed operation can be read at a time by kpl_io_uring_get_completed
* Uses defaults that require kernel version of 6.6 or greater
*/
typedef struct {
int32_t ring_fd, ring_queue_entries; // fd for submit, number of queue slots
uint32_t added_to_queue, submitted_to_kernel; //added to submission queue, added with io uring enter
uint32_t sq_cq_size, sqes_size; // using single mmap for kernel submission and completion
uint32_t *sring_head, *sring_tail, *sring_mask;
uint32_t *cring_head, *cring_tail, *cring_mask;
void *sring_ptr, *cring_ptr;
struct io_uring_sqe *sqes;
struct io_uring_cqe *cqes;
struct io_uring_params params;
} kpl_io_uring; // must be zero initialized
/*
* Called by owner thread
*/
int32_t kpl_io_uring_init(kpl_io_uring *io_uring, int32_t queue_size_pow2);
/*
* Be sure no operations are in queue before calling
* Called by owner thread
*/
inline void kpl_io_uring_free(kpl_io_uring *io_uring) {
munmap(io_uring->sring_ptr, io_uring->sq_cq_size);
munmap(io_uring->sqes, io_uring->sqes_size);
}
/*
* Reads one from cqes, will call io uring enter if none are complete and submitted_to_kernel is not zero
* This will wait if io uring enter is called
* NULL for none completed
* If an error occurred, the error details will be on the kpl_task_object
*/
kpl_object_task *kpl_io_uring_get_completed(kpl_io_uring *io_uring);
/*
* Submits all sqes (added_to_queue has total) in one io uring eneter
* This does not wait
*/
int32_t kpl_io_uring_submit(kpl_io_uring *io_uring);
/*
* Only call this though one of the functions below to add an operation to the queue
*/
struct io_uring_sqe *kpl_io_uring_op(kpl_io_uring *io_uring, kpl_object_task *task, int32_t op, int32_t fd,
const void *addr, uint32_t len, uint64_t offset);
inline bool kpl_io_uring_openat(kpl_io_uring *io_uring, kpl_object_task *task, int32_t dir_fd,
const char *path, int32_t flags, mode_t mode) {
struct io_uring_sqe *sqe = kpl_io_uring_op(io_uring, task, IORING_OP_OPENAT, dir_fd, path, mode, 0);
if (sqe)
sqe->open_flags = (uint32_t) flags;
return sqe;
}
inline bool kpl_io_uring_close(kpl_io_uring *io_uring, kpl_object_task *task, int32_t fd) {
return kpl_io_uring_op(io_uring, task, IORING_OP_CLOSE, fd, NULL, 0, 0);
}
inline bool kpl_io_uring_read(kpl_io_uring *io_uring, kpl_object_task *task, int32_t fd, void *buffer,
uint32_t byte_size, uint64_t offset) {
return kpl_io_uring_op(io_uring, task, IORING_OP_READ, fd, buffer, byte_size, offset);
}
inline bool kpl_io_uring_write(kpl_io_uring *uring, kpl_object_task *task, int32_t fd, void *buffer,
uint32_t byte_size, uint64_t offset) {
return kpl_io_uring_op(uring, task, IORING_OP_WRITE, fd, buffer, byte_size, offset);
}
inline bool kpl_io_uring_statx(kpl_io_uring *io_uring, kpl_object_task *task, int32_t dir_fd,
const char *path, int32_t flags, uint32_t mask, struct statx *statx_object) {
struct io_uring_sqe *sqe = kpl_io_uring_op(io_uring, task, IORING_OP_STATX, dir_fd, path, mask,
(uint64_t) statx_object);
if (sqe)
sqe->statx_flags = (uint32_t) flags;
return sqe;
}
|
