sparkles · defer-taskrun.d

#!/usr/bin/env dub
/+ dub.sdl:
    name "io_uring_defer_taskrun"
    dependency "during" version="~>0.5.0"
    platforms "linux"
    targetPath "build"
+/
/**
 * `io_uring` — the modern low-overhead ring config: `SINGLE_ISSUER` +
 * `DEFER_TASKRUN` + `COOP_TASKRUN` (Linux 6.1, building on 6.0 and 5.19).
 *
 * This is the ring setup most thread-per-core async runtimes (tokio-uring-style,
 * one ring pinned to one thread) reach for. It combines three setup flags:
 *
 *   - `IORING_SETUP_COOP_TASKRUN` (5.19) — completion task-work runs
 *     cooperatively, only when the issuing task is already in the kernel, rather
 *     than the kernel forcing it via an inter-processor interrupt (IPI). Fewer
 *     IPIs == less cross-CPU noise.
 *   - `IORING_SETUP_SINGLE_ISSUER` (6.0) — a promise that exactly one task ever
 *     submits to this ring. That lets the kernel drop a chunk of internal
 *     locking/synchronization on the submission path.
 *   - `IORING_SETUP_DEFER_TASKRUN` (6.1) — completion task-work is *deferred*
 *     entirely until the owning task next enters the kernel asking for events
 *     (an `io_uring_enter` with `GETEVENTS`). This is the big win: completions
 *     are batched and processed at a single, predictable point, eliminating
 *     wakeups/IPIs between submit and reap. `DEFER_TASKRUN` *requires*
 *     `SINGLE_ISSUER` (only one task can be the one that "enters", so the
 *     ownership must be unambiguous), and it only makes progress when that task
 *     calls in with `GETEVENTS` — which is exactly what `io.wait()` does here.
 *
 * The demo: set the ring up with all three flags, submit a NOP plus a short
 * relative TIMEOUT, then `io.wait()` (entering with `GETEVENTS`, which is what
 * runs the deferred completion task-work) and verify both complete.
 *
 * Companion to the io_uring chronology:
 * see docs/research/async-io/io-uring/timeline.md
 *     § "6.1 — Zero-copy sendmsg, deferred task-run (December 2022)".
 *
 * Run with: `dub run --single defer-taskrun.d`
 *
 * Portability: a kernel older than 6.1 rejects this flag combination with
 * `-EINVAL`; older still has no io_uring at all (`setup` fails). In both cases
 * the program prints a `SKIP:` line and exits 0 so it stays green in CI
 * regardless of the host kernel.
 */
module 
(module) io_uring_defer_taskrunio_uring — the modern low-overhead ring config: SINGLE_ISSUER +
DEFER_TASKRUN + COOP_TASKRUN (Linux 6.1, building on 6.0 and 5.19).
This is the ring setup most thread-per-core async runtimes (tokio-uring-style,
one ring pinned to one thread) reach for. It combines three setup flags:
IORING_SETUP_COOP_TASKRUN (5.19) — completion task-work runs
    cooperatively, only when the issuing task is already in the kernel, rather
    than the kernel forcing it via an inter-processor interrupt (IPI). Fewer
    IPIs == less cross-CPU noise.
IORING_SETUP_SINGLE_ISSUER (6.0) — a promise that exactly one task ever
    submits to this ring. That lets the kernel drop a chunk of internal
    locking/synchronization on the submission path.
IORING_SETUP_DEFER_TASKRUN (6.1) — completion task-work is deferred
    entirely until the owning task next enters the kernel asking for events
    (an io_uring_enter with GETEVENTS). This is the big win: completions
    are batched and processed at a single, predictable point, eliminating
    wakeups/IPIs between submit and reap. DEFER_TASKRUN requires
    SINGLE_ISSUER (only one task can be the one that "enters", so the
    ownership must be unambiguous), and it only makes progress when that task
    calls in with GETEVENTS — which is exactly what io.wait() does here.
The demo: set the ring up with all three flags, submit a NOP plus a short
relative TIMEOUT, then io.wait() (entering with GETEVENTS, which is what
runs the deferred completion task-work) and verify both complete.
Companion to the io_uring chronology:
see docs/research/async-io/io-uring/timeline.md
§ "6.1 — Zero-copy sendmsg, deferred task-run (December 2022)".
Run with: dub run --single defer-taskrun.d
Portability
a kernel older than 6.1 rejects this flag combination with
-EINVAL; older still has no io_uring at all (setup fails). In both cases
the program prints a SKIP: line and exits 0 so it stays green in CI
regardless of the host kernel.io_uring_defer_taskrun;

import 
(module) duringSimple idiomatic dlang wrapper around linux io_uring
(see: https://kernel.dk/io_uring.pdf) asynchronous API.during;

import 
(package) stdstd.
(module) std.stdioCategory Symbols 
File handles _popen File isFileHandle openNetwork stderr stdin stdout 
Reading chunks lines readf readfln readln 
Writing toFile write writef writefln writeln 
Misc KeepTerminator LockType StdioException 
Standard I/O functions that extend core.stdc.stdio.  core.stdc.stdio
is publically imported when importing std.stdio.
There are three layers of I/O:
The lowest layer is the operating system layer. The two main schemes are Windows and Posix.
C's stdio.h which unifies the two operating system schemes.
std.stdio, this module, unifies the various stdio.h implementations into
    a high level package for D programs.
Source
std/stdio.d
@copyrightCopyright The D Language Foundation 2007-.@licenseBoost License 1.0.@authorsWalter Bright,
Andrei Alexandrescu,
Alex Rønne Petersenstdio : 
(alias template) io_uring_defer_taskrun.writefln = std.stdio.writefln(alias fmt, A...)(A args) if (isSomeString!(typeof(fmt)))Equivalent to writef(fmt, args, '\n').writefln, stderr;

int 
int D main()main()
{
    enum ulong 
(constant) ulong io_uring_defer_taskrun.main.nopCookie = 913378LUnopCookie = 0xDEFE2;
    enum ulong 
(constant) ulong io_uring_defer_taskrun.main.timeoutCookie = 7417863LUtimeoutCookie = 0x71_3007;

    // The modern thread-per-core config. The order of detection matters: we first
    // probe whether io_uring exists at all (plain setup), then whether *this*
    // flag combination is accepted. That keeps the two SKIP reasons distinct.
    
(struct) during.UringMain entry point to work with io_uring.
It hides SubmissionQueue and CompletionQueue behind standard range interface.
We put in SubmissionEntry entries and take out CompletionEntry entries.
Use predefined prepXX methods to fill required fields of SubmissionEntry before put or during putWith.
Note
prepXX functions doesn't touch previous entry state, just fills in operation properties. This is because for
less error prone interface it is cleared automatically when prepared using putWith. So when using on own SubmissionEntry
(outside submission queue), that would be added to the submission queue using put, be sure its cleared if it's
reused for multiple operations.Uring 
(local variable) during.Uring ioio;
    const 
(local variable) const(int) deferRetdeferRet = 
(local variable) during.Uring ioio.
int during.setup(ref during.Uring uring, uint entries = 128u, during.io_uring.SetupFlags flags = SetupFlags.NONE) nothrow @nogc @safeSetup new instance of io_uring into provided Uring structure.
@paramuring Uring structure to be initialized (must not be already initialized)@paramentries Number of entries to initialize uring with@paramflags SetupFlags to use to initialize uring.@returnsOn succes it returns 0, -errno otherwise.setup(
        8,
        
(enum) during.io_uring.SetupFlagsio_uring_setup() flagsSetupFlags.
(enum value) during.io_uring.SetupFlags.SINGLE_ISSUER = 4096uIORING_SETUP_SINGLE_ISSUER (from Linux 6.0)
Hint that only one task / thread will ever submit on this ring. Lets the kernel skip
synchronisation that would otherwise be needed for shared submission. Misuse (multiple
submitters) is detected and returns -EEXIST.SINGLE_ISSUER | 
(enum) during.io_uring.SetupFlagsio_uring_setup() flagsSetupFlags.
(enum value) during.io_uring.SetupFlags.DEFER_TASKRUN = 8192uIORING_SETUP_DEFER_TASKRUN (from Linux 6.1)
Defer task_work to run only when the ring is being entered, rather than at the next
kernel/user transition on the submitting task. Eliminates a class of interrupts and
cuts latency for many workloads. Requires SINGLE_ISSUER.DEFER_TASKRUN | 
(enum) during.io_uring.SetupFlagsio_uring_setup() flagsSetupFlags.
(enum value) during.io_uring.SetupFlags.COOP_TASKRUN = 256uIORING_SETUP_COOP_TASKRUN
By default, io_uring will interrupt a task running in userspace when a
completion event comes in. This is to ensure that completions run in a timely
manner. For a lot of use cases, this is overkill and can cause reduced
performance from both the inter-processor interrupt used to do this, the
kernel/user transition, the needless interruption of the tasks userspace
activities, and reduced batching if completions come in at a rapid rate. Most
applications don't need the forceful interruption, as the events are processed
at any kernel/user transition. The exception are setups where the application
uses multiple threads operating on the same ring, where the application
waiting on completions isn't the one that submitted them. For most other
use cases, setting this flag will improve performance.
Note
Available since 5.19.COOP_TASKRUN,
    );
    if (
(local variable) const(int) deferRetdeferRet < 0)
    {
        // -EINVAL here means the kernel knows io_uring but rejects one of these
        // flags (DEFER_TASKRUN < 6.1, SINGLE_ISSUER < 6.0). Distinguish "no
        // io_uring at all" from "this feature is too new" by retrying a plain setup.
        
(struct) during.UringMain entry point to work with io_uring.
It hides SubmissionQueue and CompletionQueue behind standard range interface.
We put in SubmissionEntry entries and take out CompletionEntry entries.
Use predefined prepXX methods to fill required fields of SubmissionEntry before put or during putWith.
Note
prepXX functions doesn't touch previous entry state, just fills in operation properties. This is because for
less error prone interface it is cleared automatically when prepared using putWith. So when using on own SubmissionEntry
(outside submission queue), that would be added to the submission queue using put, be sure its cleared if it's
reused for multiple operations.Uring 
(local variable) during.Uring plainplain;
        const 
(local variable) const(int) plainRetplainRet = 
(local variable) during.Uring plainplain.
int during.setup(ref during.Uring uring, uint entries = 128u, during.io_uring.SetupFlags flags = SetupFlags.NONE) nothrow @nogc @safeSetup new instance of io_uring into provided Uring structure.
@paramuring Uring structure to be initialized (must not be already initialized)@paramentries Number of entries to initialize uring with@paramflags SetupFlags to use to initialize uring.@returnsOn succes it returns 0, -errno otherwise.setup(8);
        if (
(local variable) const(int) plainRetplainRet < 0)
        {
            
void std.stdio.writefln!(char, const(int))(in char[] fmt, const(int) __param_1) @safeEquivalent to writef(fmt, args, '\n').writefln(
                "SKIP: io_uring_setup failed (errno %d) — io_uring unavailable on this host",
                -
(local variable) const(int) plainRetplainRet);
            return 0;
        }
        
void std.stdio.writefln!(char, const(int))(in char[] fmt, const(int) __param_1) @safeEquivalent to writef(fmt, args, '\n').writefln(
            "SKIP: SINGLE_ISSUER|DEFER_TASKRUN|COOP_TASKRUN rejected (errno %d) — needs Linux >= 6.1",
            -
(local variable) const(int) deferRetdeferRet);
        return 0;
    }

    // A relative timeout that fires almost immediately (1ms). Under DEFER_TASKRUN
    // its completion task-work is deferred until we enter with GETEVENTS below.
    
(struct) during.io_uring.KernelTimespecTime specification as defined in kernel headers (used by TIMEOUT operations)KernelTimespec 
(local variable) during.io_uring.KernelTimespec tsts;
    
(local variable) during.io_uring.KernelTimespec tsts.
(field) long during.io_uring.KernelTimespec.tv_secsecondstv_sec = 0;
    
(local variable) during.io_uring.KernelTimespec tsts.
(field) long during.io_uring.KernelTimespec.tv_nsecnanosecondstv_nsec = 1_000_000; // 1ms

    // Enqueue a NOP and a short TIMEOUT. With deferred task-run, neither posts a
    // completion eagerly — the kernel parks the task-work until the owning task
    // (us) next enters the ring asking for events.
    
(local variable) during.Uring ioio.putWith!((ref SubmissionEntry e) {
        e.prepNop();
        e.user_data = nopCookie;
    })();
    
(local variable) during.Uring ioio.putWith!((ref SubmissionEntry e, ref KernelTimespec t) {
        e.prepTimeout(t, 0, TimeoutFlags.REL);
        e.user_data = timeoutCookie;
    })(
during.Uring during.Uring.putWith!(function (ref during.io_uring.SubmissionEntry e, ref during.io_uring.KernelTimespec t) nothrow @nogc @safe
{
prepTimeout(e, t, 0LU, TimeoutFlags.REL);
e.user_data = 7417863LU;
}
, during.io_uring.KernelTimespec)(ref during.io_uring.KernelTimespec __param_0) nothrow @nogc return ref @safeAdds new entry to the SubmissionQueue.
Note that this just adds entry to the queue and doesn't advance the tail
marker kernel sees. For that finishSq() is needed to be called next.
Also note that to actually enter new entries to kernel,
it's needed to call submit().
@paramFN Function to fill next entry in queue by ref (should be faster). It is expected to be in a form of void function(ARGS)(ref SubmissionEntry, auto ref ARGS). Note that in this case queue entry is cleaned first before function is called.@paramentry Custom built SubmissionEntry to be posted as is. Note that in this case it is copied whole over one in the SubmissionQueue.@paramargs Optional arguments passed to the function@returnsreference to Uring structure so it's possible to chain multiple commands.ts);

    // submit(0) flushes the SQ without asking the kernel to wait on a count; it
    // returns the number of SQEs consumed.
    const 
(local variable) const(int) submittedsubmitted = 
(local variable) during.Uring ioio.
int during.Uring.submit(uint want) nothrow @nogc @safeSubmits qued SubmissionEntry to be processed by kernel.
@paramwant number of CompletionEntries to wait for. If 0, this just submits queued entries and returns. If > 0, it blocks until at least wanted number of entries were completed.@paramsig See io_uring_enter(2) man page@returnsNumber of submitted entries on success, -errno on errorsubmit(0);
    if (
(local variable) const(int) submittedsubmitted < 0)
    {
        stderr.
std.stdio.File std.stdio.makeGlobal!"core.stdc.stdio.stderr"() nothrow @nogc @property ref @systemwritefln("submit failed: errno %d", -
(local variable) const(int) submittedsubmitted);
        return 1;
    }
    if (
(local variable) const(int) submittedsubmitted != 2)
    {
        stderr.
std.stdio.File std.stdio.makeGlobal!"core.stdc.stdio.stderr"() nothrow @nogc @property ref @systemwritefln("expected to submit 2 SQEs, submitted %d", 
(local variable) const(int) submittedsubmitted);
        return 1;
    }

    // Reap both completions. Each `io.wait` performs an io_uring_enter with
    // GETEVENTS — the single, deferred point at which the kernel runs the parked
    // completion task-work for this ring. A TIMEOUT with count 0 reports -ETIME.
    bool 
(local variable) bool sawNopsawNop;
    bool 
(local variable) bool sawTimeoutsawTimeout;
    foreach (
(local variable) int __; 0 .. 2)
    {
        
(local variable) during.Uring ioio.
int during.Uring.wait(uint want = 1u) nothrow @nogcSimmilar to submit but with this method we just wait for required number
of CompletionEntries.
@returns0 on success, -errno on errorwait(1);
        const 
(local variable) const(during.io_uring.CompletionEntry) cc = 
(local variable) during.Uring ioio.
during.io_uring.CompletionEntry during.Uring.front() pure nothrow @nogc return ref @safeGet first CompletionEntry from cq ringfront;
        const 
(local variable) const(int) resres = 
(local variable) const(during.io_uring.CompletionEntry) cc.
(field) int during.io_uring.CompletionEntry.resresult code for this eventres;
        const 
(local variable) const(ulong) udud = 
(local variable) const(during.io_uring.CompletionEntry) cc.
(field) ulong during.io_uring.CompletionEntry.user_datasqe->data submission passed backuser_data;
        
(local variable) during.Uring ioio.
void during.Uring.popFront() pure nothrow @nogc @safeMove to next CompletionEntrypopFront();

        if (
(local variable) const(ulong) udud == 
(constant) ulong io_uring_defer_taskrun.main.nopCookie = 913378LUnopCookie)
        {
            if (
(local variable) const(int) resres < 0)
            {
                stderr.
std.stdio.File std.stdio.makeGlobal!"core.stdc.stdio.stderr"() nothrow @nogc @property ref @systemwritefln("NOP completed with error: errno %d", -
(local variable) const(int) resres);
                return 1;
            }
            
(local variable) bool sawNopsawNop = true;
        }
        else if (
(local variable) const(ulong) udud == 
(constant) ulong io_uring_defer_taskrun.main.timeoutCookie = 7417863LUtimeoutCookie)
        {
            import 
(package) corecore.
(package) core.syssys.
(package) core.sys.linuxlinux.
(module) core.sys.linux.errnoD header file for GNU/Linux
glibc stdlib/errno.herrno : 
(alias constant) ETIME = int core.stdc.errno.ETIME = 62ETIME;
            // A relative timeout with count 0 elapses and reports -ETIME; that is
            // the expected success signal here, not a failure.
            if (
(local variable) const(int) resres != -
(constant) int core.stdc.errno.ETIME = 62ETIME && 
(local variable) const(int) resres != 0)
            {
                stderr.
std.stdio.File std.stdio.makeGlobal!"core.stdc.stdio.stderr"() nothrow @nogc @property ref @systemwritefln("TIMEOUT completed unexpectedly: errno %d", -
(local variable) const(int) resres);
                return 1;
            }
            
(local variable) bool sawTimeoutsawTimeout = true;
        }
        else
        {
            stderr.
std.stdio.File std.stdio.makeGlobal!"core.stdc.stdio.stderr"() nothrow @nogc @property ref @systemwritefln("unexpected completion: user_data 0x%X", 
(local variable) const(ulong) udud);
            return 1;
        }
    }

    if (!
(local variable) bool sawNopsawNop || !
(local variable) bool sawTimeoutsawTimeout)
    {
        stderr.
std.stdio.File std.stdio.makeGlobal!"core.stdc.stdio.stderr"() nothrow @nogc @property ref @systemwritefln(
            "missing completions: sawNop=%s sawTimeout=%s", 
(local variable) bool sawNopsawNop, 
(local variable) bool sawTimeoutsawTimeout);
        return 1;
    }

    
void std.stdio.writefln!char(in char[] fmt) @safeEquivalent to writef(fmt, args, '\n').writefln(
        "ok: SINGLE_ISSUER|DEFER_TASKRUN|COOP_TASKRUN ring set up; NOP + TIMEOUT "
        ~ "reaped via GETEVENTS-driven deferred task-run");
    return 0;
}