loop_int.h

/*-
 * Copyright (c) 2026, The XTC Project
 * Use of this source code is governed by the ISC License.
 *
 * src/inc/loop_int.h
 *      Internal definitions for the L2 event loop.  Not part of the
 *      public ABI.
 */
 
#ifndef XTC_LOOP_INT_H
#define XTC_LOOP_INT_H
 
#include <stdatomic.h>
#include <stdint.h>
 
#include "xtc_loop.h"
#include "xtc_io.h"
#include "xtc_res.h"
#include "deque.h"
#include "os_thread.h"
 
/*
 * Task state machine.  Transitions:
 *      SCHEDULED -> RUNNING            loop pops from queue
 *      RUNNING   -> SCHEDULED          fn returned RESCHED
 *      RUNNING   -> PARKED             fn returned PENDING
 *      RUNNING   -> DONE               fn returned DONE
 *      PARKED    -> SCHEDULED          waker fired (or timer / fd ready)
 *      PARKED    -> DONE               (not reachable; PENDING tasks
 *                                       are reaped only after they
 *                                       next return DONE)
 */
enum xtc_task_state {
        XTC_TS_SCHEDULED = 0,
        XTC_TS_RUNNING   = 1,
        XTC_TS_PARKED    = 2,
        XTC_TS_DONE      = 3
};
 
struct xtc_task {
        xtc_task_fn  fn;
        void        *user;
        xtc_loop_t  *loop;
        int          state;
        /* Monotonic time (ns) this run quantum was dispatched, recorded
         * by the scheduler when loop->yield_budget_ns > 0.  xtc_yield_check
         * compares against it; 0 means not yet recorded this quantum. */
        int64_t      run_start_ns;
        /* Pinned tasks run only on their home loop -- they go on the
         * owner-only FIFO, never the stealable deque.  Used for explicit
         * placement (xtc_exec_spawn_on) and for processes, which keep a
         * shard-style affinity to one loop.  Unpinned tasks (the general
         * pool) go on the Chase-Lev deque and may be work-stolen. */
        int          pinned;
        /* Run-queue intrusive next pointer. */
        struct xtc_task *q_next;
 
        /* Park bookkeeping.  At most one of these is active at a time
         * while the task is in PARKED state. */
        xtc_timer_t *park_timer;
        int          park_fd;       /* -1 when not parked on fd */
        /* Voluntary park: when set by a primitive (e.g. xtc_amutex) just
         * before yielding, the coro step returns PENDING instead of
         * RESCHED, so the task sleeps until a waker re-enqueues it rather
         * than busy-spinning.  Read-and-cleared by the step. */
        int          park_requested;
 
        /* Wakeup-cause flags, set by the dispatcher / timer callback /
         * mbox_deliver when the task is unparked.  Sampled and cleared
         * by the parker on resume (e.g. xtc_proc_wait_fd).  Encodes
         * XTC_IO_* flags from the dispatched event plus the synthetic
         * XTC_WAIT_MAILBOX (set by __mbox_deliver) and XTC_WAIT_TIMEOUT
         * (set by the timer callback).  Atomic: a cross-thread xtc_send
         * ORs XTC_WAIT_MAILBOX in from a FOREIGN thread (proc.c) while the
         * owning loop ORs/reads/zeros it (task.c/loop.c/proc.c), so a plain
         * uint32_t RMW here is a data race (TSan-reportable).  All accesses
         * use relaxed atomics -- ordering is provided by the waker/mailbox
         * lock; the atomic only makes the OR itself race-free. */
        _Atomic uint32_t wake_revents;
 
        /* Doubly linked into loop->all_tasks so a completed task can be
         * unlinked in O(1) and recycled to the loop's task_slab (instead of
         * lingering until loop_fini).  all_prev == NULL means the head. */
        struct xtc_task *all_next;
        struct xtc_task *all_prev;
 
        /* 1 if this task struct is eligible to be recycled onto the loop's
         * task free-list when it completes (a plain task on its home loop);
         * cleared for tasks that must not be recycled.  All task structs
         * are __os_calloc'd regardless -- this only gates the free-list
         * push, not the allocation source. */
        int              recyclable;
 
        /* Optional cleanup hook invoked by xtc_loop_fini before the task
         * struct is freed.  The coroutine layer sets this to release the
         * fiber stack + coro struct that wrap a task; plain tasks leave
         * it NULL.  Keeps task lifetime owned by the loop while letting
         * higher layers reclaim what they attached. */
        void       (*cleanup)(void *cleanup_arg);
        void        *cleanup_arg;
};
 
/*
 * Timer record.  Kept in a binary min-heap inside the loop.
 *
 * Cancel is lazy: we mark cancelled and skip on pop.  Cancel is O(1)
 * by cost; the heap may carry up to N stale entries until the next
 * extraction reaches them.  For M3 this is good enough; M5 may
 * upgrade to a hierarchical wheel.
 */
struct xtc_timer {
        int64_t      deadline_ns;
        xtc_timer_fn cb;
        void        *user;
        xtc_task_t  *waiter;        /* task to wake when fired (NULL if pure cb) */
        int          heap_idx;      /* current position in heap (-1 if not in) */
        int          cancelled;
        int          fired;
        int          sim_late;      /* DST: 1 once a buggify late-fire bumped
                                     * this timer's deadline (bump at most
                                     * once so a late fire cannot spin);
                                     * always 0 outside sim. */
        xtc_loop_t  *loop;          /* back-pointer for cancel-by-handle */
        struct xtc_timer *all_next; /* per-loop linked list for cleanup */
};
 
/*
 * Inbox message kinds.  All inbox traffic is cross-thread; the loop
 * owner drains its inbox at the top of every step.
 */
enum xtc_inbox_kind {
        XTC_INB_WAKE = 0,        /* re-queue a parked task */
        XTC_INB_PUBLISH = 1,     /* publish a freshly-allocated task */
};
 
struct xtc_inbox_msg {
        enum xtc_inbox_kind  kind;
        xtc_task_t          *task;
        struct xtc_inbox_msg *next;
};
 
struct xtc_inbox {
        __os_mutex_t          lock;
        struct xtc_inbox_msg *head;
        struct xtc_inbox_msg *tail;
        int                   inited;
};
 
struct xtc_loop {
        xtc_io_t *io;
 
        /* Local run queue (Chase-Lev deque, owner pushes/pops). */
        xtc_deque_t deque;
 
        /* Slow-path overflow when the deque is full.  Owner-only. */
        struct xtc_task *q_head;
        struct xtc_task *q_tail;
 
        /* Timer min-heap. */
        xtc_timer_t **timers;
        int           n_timers;
        int           cap_timers;
 
        /* All tasks ever spawned, for cleanup.  Owner-only after init. */
        struct xtc_task *all_tasks;
 
        /* All timers ever created, for cleanup at fini. */
        xtc_timer_t *all_timers;
 
        /* M11.5b: per-loop slab cache for xtc_timer_t.  Created lazily
         * by xtc_timer_set; freed in loop_fini.  Per-loop = single-
         * threaded ownership = magazine fast path is lock-free. */
        struct xtc_slab *timer_slab;
 
        /* Per-loop task-struct free-list: a plain single-threaded LIFO of
         * recycled task structs (linked through their q_next while free).
         * xtc_task_spawn pops from it instead of malloc'ing, and a
         * completed plain task on its home loop is pushed back instead of
         * freed -- the spawn-heavy hot path, with no allocator call and no
         * accumulation.  Only the owning loop thread touches it, so it is
         * lock-free.  Drained (structs __os_free'd) at loop_fini. */
        struct xtc_task *task_free;
        int              task_free_n;
 
        /* Live-task counter.  Atomic so cross-thread spawns/completions
         * can update it without lock. */
        _Atomic int n_alive;
 
        /* Per-loop work statistics (executor observability).  tasks_run
         * counts task steps executed on this loop; steals counts tasks
         * this loop successfully stole from a peer.  Relaxed atomics:
         * read-mostly counters, exactness across a concurrent read is
         * not required. */
        _Atomic uint64_t n_tasks_run;
        _Atomic uint64_t n_steals;
 
        /* I/O fairness: counts task runs since the last I/O poll.  When the
         * run queue never empties (busy-yielding fibers, e.g. a buffer
         * manager spinning on eviction), the loop would otherwise never
         * poll I/O and parked completions would starve.  Every
         * IO_FAIRNESS_QUANTUM runs the step does a non-blocking poll. */
        unsigned int     runs_since_poll;
 
        /* Cooperative yield watchdog (opt-in).  When yield_budget_ns > 0
         * the scheduler records each quantum's start time on the task and
         * xtc_yield_check reports a task over budget; n_yield_due counts
         * over-budget reports (telemetry). */
        int64_t          yield_budget_ns;
        _Atomic uint64_t n_yield_due;
 
        int stop_requested;
 
        /* Cross-thread inbox: wakers and remote spawns deposit here;
         * the owner drains in __xtc_loop_drain_inbox. */
        struct xtc_inbox inbox;
 
        /* For the multi-loop executor: 0-based index in xtc_exec; -1 if
         * this loop is standalone (M3 single-thread mode). */
        int exec_id;
 
        /* Back-pointer to the executor (NULL if standalone). */
        struct xtc_exec *exec;
 
        /*
         * Resource accountant.  Either owned by the loop (allocated and
         * freed at init/fini) or borrowed from the executor.  Tracks
         * tasks-alive, inbox messages, channels, etc.
         */
        xtc_res_t *res;
        int        owns_res;
};
 
/* Internal helpers shared between loop.c, task.c, timer.c. */
int  __xtc_loop_enqueue(xtc_loop_t *loop, xtc_task_t *t);
/* Spawn with explicit pinned-ness: pinned tasks stay on `loop` (FIFO,
 * never work-stolen); unpinned tasks may migrate via the deque. */
int  __xtc_task_spawn_ex(xtc_loop_t *loop, xtc_task_fn fn, void *user,
                         int pinned, xtc_task_t **out_task);
int  __xtc_timer_heap_push(xtc_loop_t *loop, xtc_timer_t *t);
xtc_timer_t *__xtc_timer_heap_pop_due(xtc_loop_t *loop, int64_t now_ns);
int64_t      __xtc_timer_heap_next_deadline(xtc_loop_t *loop);
int  __xtc_loop_dispatch_event(xtc_loop_t *loop, xtc_io_event_t *ev);
 
/* Implemented in proc.c.  Called from loop_fini to release the
 * proc-table side struct hashed against this loop pointer.  Must be
 * idempotent. */
void __xtc_proc_loop_unregister(xtc_loop_t *loop);
 
/* Inbox API.  Producer-side functions are thread-safe. */
int  __xtc_inbox_init(struct xtc_inbox *ib);
void __xtc_inbox_fini(struct xtc_inbox *ib);
int  __xtc_inbox_push(struct xtc_inbox *ib, enum xtc_inbox_kind k, xtc_task_t *t);
int  __xtc_inbox_drain(xtc_loop_t *loop);  /* owner-only; drains into local queue */
 
/* Per-thread cursor: which loop the calling thread is running.
 * NULL on threads that aren't loop owners. */
extern XTC_THREAD_LOCAL xtc_loop_t *__xtc_current_loop;
 
/*
 * Fiber-context preservation hook.  The L3 process layer keeps a
 * per-thread "current proc" pointer that must survive a yield (the
 * scheduler runs other fibers in between, which overwrite it).  The
 * L2 coro layer cannot depend on L3, so every yield/await jump saves
 * the opaque context before jumping to the scheduler and restores it
 * on resume through these hooks.  proc.c installs them on first
 * spawn; while NULL (no process layer in use) the calls are no-ops.
 * Set once to stable function addresses, so a plain pointer load is
 * safe without synchronization.
 */
extern void *(*__xtc_fiber_ctx_save)(void);
extern void  (*__xtc_fiber_ctx_restore)(void *);
 
/* Post-resume cancellation hook.  Installed by the process layer
 * (proc.c).  Called at the universal fiber resume point (after a yield
 * returns) so a fiber that had a kill/cancel requested while it was
 * NOT at a cooperative point -- e.g. a pure CPU loop that was
 * involuntarily preempted -- honors the kill the instant the scheduler
 * resumes it, by unwinding via xtc_exit_self.  NULL when no process
 * layer is present (bare coroutine use).  Returns without effect if no
 * kill is pending. */
extern void  (*__xtc_fiber_kill_check)(void);
 
/* Forward declaration for back-pointer in xtc_loop. */
struct xtc_exec;
 
#endif /* XTC_LOOP_INT_H */