commit d9c598e309b7a1cda332a374760e13963df4f6de
parent aee0f267ee7674fbc2055ae338c642cb9de7bdda
Author: Drew DeVault <sir@cmpwn.com>
Date: Fri, 14 May 2021 14:49:05 -1000
linux::uring: implement setup
Diffstat:
4 files changed, 179 insertions(+), 46 deletions(-)
diff --git a/linux/uring/setup.ha b/linux/uring/setup.ha
@@ -0,0 +1,82 @@
+use errors;
+use rt;
+
+
+// Sets up an io_uring. The params parameter must be initialized with the
+// desired flags, sq_thread_cpu, and sq_thread_idle parameters; the remaining
+// fields are initialized by the kernel.
+export fn setup(entries: u32, params: *params) (io_uring | error) = {
+ const fd = match (rt::io_uring_setup(entries, params)) {
+ err: rt::errno => return errors::errno(err),
+ fd: int => fd,
+ };
+
+ let uring = io_uring {
+ sq = sq { ... },
+ cq = cq { ... },
+ fd = fd,
+ flags = params.flags,
+ features = params.features,
+ };
+ let sq = &uring.sq, cq = &uring.cq;
+
+ sq.ring_sz = params.sq_off.array + params.sq_entries * size(uint);
+ cq.ring_sz = params.cq_off.cqes + params.cq_entries * size(cqe);
+
+ if (uring.features & features::SINGLE_MMAP == features::SINGLE_MMAP) {
+ if (cq.ring_sz > sq.ring_sz) {
+ sq.ring_sz = cq.ring_sz;
+ };
+ cq.ring_sz = sq.ring_sz;
+ };
+
+ sq.ring_ptr = match (rt::mmap(null,
+ params.sq_off.array + entries * size(u32),
+ rt::PROT_READ | rt::PROT_WRITE,
+ rt::MAP_SHARED | rt::MAP_POPULATE,
+ fd, OFF_SQ_RING)) {
+ err: rt::errno => return errors::errno(err),
+ ptr: *void => ptr,
+ };
+
+ cq.ring_ptr = if (uring.features & features::SINGLE_MMAP == features::SINGLE_MMAP) {
+ sq.ring_ptr;
+ } else match (rt::mmap(null, cq.ring_sz,
+ rt::PROT_READ | rt::PROT_WRITE,
+ rt::MAP_SHARED | rt::MAP_POPULATE,
+ fd, OFF_CQ_RING)) {
+ err: rt::errno => return errors::errno(err),
+ ptr: *void => ptr,
+ };
+
+ const ring_ptr = sq.ring_ptr: uintptr;
+ sq.khead = (ring_ptr + params.sq_off.head: uintptr): *uint;
+ sq.ktail = (ring_ptr + params.sq_off.tail: uintptr): *uint;
+ sq.kring_mask = (ring_ptr + params.sq_off.ring_mask: uintptr): *uint;
+ sq.kring_entries = (ring_ptr + params.sq_off.ring_entries: uintptr): *uint;
+ sq.kflags = (ring_ptr + params.sq_off.flags: uintptr): *sqring_flags;
+ sq.kdropped = (ring_ptr + params.sq_off.dropped: uintptr): *uint;
+ sq.array = (ring_ptr + params.sq_off.array: uintptr): *uint;
+ sq.sqes = match (rt::mmap(null,
+ params.sq_entries * size(sqe),
+ rt::PROT_READ | rt::PROT_WRITE,
+ rt::MAP_SHARED | rt::MAP_POPULATE,
+ fd, OFF_SQES)) {
+ err: rt::errno => return errors::errno(err),
+ ptr: *void => ptr: *[*]sqe,
+ };
+
+ const ring_ptr = cq.ring_ptr: uintptr;
+ cq.khead = (ring_ptr + params.cq_off.head: uintptr): *uint;
+ cq.ktail = (ring_ptr + params.cq_off.tail: uintptr): *uint;
+ cq.kring_mask = (ring_ptr + params.cq_off.ring_mask: uintptr): *uint;
+ cq.kring_entries = (ring_ptr + params.cq_off.ring_entries: uintptr): *uint;
+ cq.koverflow = (ring_ptr + params.cq_off.overflow: uintptr): *uint;
+ cq.cqes = (ring_ptr + params.cq_off.cqes: uintptr): *[*]cqe;
+
+ if (params.cq_off.flags != 0) {
+ cq.kflags = (ring_ptr + params.cq_off.flags: uintptr): *cqring_flags;
+ };
+
+ return uring;
+};
diff --git a/linux/uring/uring.ha b/linux/uring/uring.ha
@@ -1,7 +1,17 @@
-export def CQE_BUFFER_SHIFT: uint = 16;
-export def OFF_SQ_RING: u64 = 0;
-export def OFF_CQ_RING: u64 = 0x8000000;
-export def OFF_SQES: u64 = 0x10000000;
+use errors;
+
+// All errors which may be returned by this module.
+export type error = errors::opaque;
+
+// Converts an [[error]] into a human-readable string.
+export fn strerror(err: error) const str = {
+ return errors::strerror(err);
+};
+
+def CQE_BUFFER_SHIFT: uint = 16;
+def OFF_SQ_RING: u64 = 0;
+def OFF_CQ_RING: u64 = 0x8000000;
+def OFF_SQES: u64 = 0x10000000;
// An io_uring [[sqe]] operation.
export type op = enum u8 {
@@ -41,6 +51,42 @@ export type op = enum u8 {
TEE,
};
+// Flags for an [[sqe]].
+export type sqe_flags = enum u8 {
+ // Use fixed fileset
+ FIXED_FILE = 1 << 0,
+ // Issue after inflight IO
+ IO_DRAIN = 1 << 1,
+ // Links next sqe
+ IO_LINK = 1 << 2,
+ // Like LINK, but stronger
+ IO_HARDLINK = 1 << 3,
+ // Always go async
+ ASYNC = 1 << 4,
+ // Select buffer from sqe.buf_group
+ BUFFER_SELECT = 1 << 5,
+};
+
+// Flags for an fsync operation.
+export type fsync_flags = enum u32 {
+ DATASYNC = 1 << 0,
+};
+
+// Flags for a timeout operation.
+export type timeout_flags = enum u32 {
+ ABS = 1 << 0,
+};
+
+// Flags for a splice operation.
+export type splice_flags = enum u32 {
+ F_FD_IN_FIXED = 1 << 31,
+};
+
+// Flags for a [[cqe]].
+export type cqe_flags = enum u32 {
+ F_BUFFER = 1 << 0,
+};
+
// A submission queue entry.
export type sqe = struct {
opcode: op,
@@ -85,37 +131,6 @@ export type sqe = struct {
},
};
-// Flags for an [[sqe]].
-export type sqe_flags = enum u8 {
- // Use fixed fileset
- FIXED_FILE = 1 << 0,
- // Issue after inflight IO
- IO_DRAIN = 1 << 1,
- // Links next sqe
- IO_LINK = 1 << 2,
- // Like LINK, but stronger
- IO_HARDLINK = 1 << 3,
- // Always go async
- ASYNC = 1 << 4,
- // Select buffer from sqe.buf_group
- BUFFER_SELECT = 1 << 5,
-};
-
-// Flags for an fsync operation.
-export type fsync_flags = enum u32 {
- DATASYNC = 1 << 0,
-};
-
-// Flags for a timeout operation.
-export type timeout_flags = enum u32 {
- ABS = 1 << 0,
-};
-
-// Flags for a splice operation.
-export type splice_flags = enum u32 {
- F_FD_IN_FIXED = 1 << 31,
-};
-
// A completion queue entry.
export type cqe = struct {
user_data: u64,
@@ -123,18 +138,13 @@ export type cqe = struct {
flags: cqe_flags,
};
-// Flags for a [[cqe]].
-export type cqe_flags = enum u32 {
- F_BUFFER = 1 << 0,
-};
-
// Filled with the offset for mmap(2)
export type sqring_offsets = struct {
head: u32,
tail: u32,
ring_mask: u32,
ring_entries: u32,
- flags: sqring_flags,
+ flags: u32,
dropped: u32,
array: u32,
resv1: u32,
@@ -157,7 +167,7 @@ export type cqring_offsets = struct {
ring_entries: u32,
overflow: u32,
cqes: u32,
- flags: cqring_flags,
+ flags: u32,
resv1: u32,
resv2: u64,
};
@@ -189,7 +199,7 @@ export type setup_flags = enum u32 {
export type params = struct {
sq_entries: u32,
cq_entries: u32,
- flags: u32,
+ flags: setup_flags,
sq_thread_cpu: u32,
sq_thread_idle: u32,
features: features,
@@ -287,3 +297,41 @@ export type restriction_op = enum u16 {
// Require sqe flags (these flags must be set on each submission)
SQE_FLAGS_REQUIRED = 3,
};
+
+// State for an io_uring.
+export type io_uring = struct {
+ sq: sq,
+ cq: cq,
+ fd: int,
+ flags: setup_flags,
+ features: features,
+};
+
+// Submission queue state.
+export type sq = struct {
+ khead: *uint,
+ ktail: *uint,
+ kring_mask: *uint,
+ kring_entries: *uint,
+ kflags: *sqring_flags,
+ kdropped: *uint,
+ array: *uint,
+ sqes: *[*]sqe,
+ sqe_head: uint,
+ sqe_tail: uint,
+ ring_sz: size,
+ ring_ptr: *void,
+};
+
+// Completion queue state.
+export type cq = struct {
+ khead: *uint,
+ ktail: *uint,
+ kring_mask: *uint,
+ kring_entries: *uint,
+ kflags: *cqring_flags,
+ koverflow: *uint,
+ cqes: *[*]cqe,
+ ring_sz: size,
+ ring_ptr: *void,
+};
diff --git a/scripts/gen-stdlib b/scripts/gen-stdlib
@@ -519,8 +519,9 @@ linux() {
linux_uring() {
gen_srcs linux::uring \
+ setup.ha \
uring.ha
- gen_ssa linux::uring
+ gen_ssa linux::uring errors
}
linux_vdso() {
diff --git a/stdlib.mk b/stdlib.mk
@@ -698,9 +698,10 @@ $(HARECACHE)/linux/linux.ssa: $(stdlib_linux_srcs) $(stdlib_rt) $(stdlib_format_
# linux::uring
stdlib_linux_uring_srcs= \
+ $(STDLIB)/linux/uring/setup.ha \
$(STDLIB)/linux/uring/uring.ha
-$(HARECACHE)/linux/uring/linux_uring.ssa: $(stdlib_linux_uring_srcs) $(stdlib_rt)
+$(HARECACHE)/linux/uring/linux_uring.ssa: $(stdlib_linux_uring_srcs) $(stdlib_rt) $(stdlib_errors)
@printf 'HAREC \t$@\n'
@mkdir -p $(HARECACHE)/linux/uring
@HARECACHE=$(HARECACHE) $(HAREC) $(HAREFLAGS) -o $@ -Nlinux::uring \
@@ -1687,9 +1688,10 @@ $(TESTCACHE)/linux/linux.ssa: $(testlib_linux_srcs) $(testlib_rt) $(testlib_form
# linux::uring
testlib_linux_uring_srcs= \
+ $(STDLIB)/linux/uring/setup.ha \
$(STDLIB)/linux/uring/uring.ha
-$(TESTCACHE)/linux/uring/linux_uring.ssa: $(testlib_linux_uring_srcs) $(testlib_rt)
+$(TESTCACHE)/linux/uring/linux_uring.ssa: $(testlib_linux_uring_srcs) $(testlib_rt) $(testlib_errors)
@printf 'HAREC \t$@\n'
@mkdir -p $(TESTCACHE)/linux/uring
@HARECACHE=$(TESTCACHE) $(HAREC) $(TESTHAREFLAGS) -o $@ -Nlinux::uring \
