CONTROL_IMAGES := control flow exiting faulting spinning admin remote

CONTROL_RIL := $(addprefix build/,$(addsuffix .ril,$(CONTROL_IMAGES)))

IMAGE_MODULES_control := user/control/lifecycle.rad user/control/authority.rad

CONTROL_CATALOG := -zero-bss -pkg images -mod build/control/images.rad -mod user/sys.ras $(addprefix -image ,$(CONTROL_RIL))

MACHINE := $(EMU) -machine -no-guard-stack -max-steps=1000000000 -count-instructions

build/control/images.rad: $(CONTROL_RIL) $(COMPILER)

	mkdir -p $(@D)

	$(COMPILE) -catalog $@ $(addprefix -image ,$(CONTROL_RIL))

kernel.rv64: main.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(COMPILER)

	$(COMPILE) $(CONTROL_CATALOG) $(CORE) -pkg kernel -start arch/start.ras -mod main.rad -entry kernel -o $@

parallel.rv64: parallel.rad parallel/mailboxes.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(COMPILER)

	$(COMPILE) $(CONTROL_CATALOG) $(CORE) -pkg parallel -start arch/start.ras -mod parallel.rad -mod parallel/mailboxes.rad -entry parallel -o $@

	@output="$$( $(MACHINE) -harts=2 -run kernel.rv64 2>build/boot.log )"; status=$$?; \

	if test "$$status" -ne 2 || test "$$output" != CONTROL-OK; then cat build/boot.log; exit 1; fi

record the contracts established through step 22 of the 22-step plan.

## Budgeted multicore execution and boot

- core::control applies trap effects under the shared lock. Each hart owns one

  architectural frame and one exact-use accounting interval outside that lock.

  Charge completed U-mode execution before processing its trap. Queries observe

  retained state and conservative balances without charging another hart's live

  interval.

- At the architectural boundary, failed metadata queries terminate the caller

  with Fault code 512 + Error. Ordinary call failures return Error to U-mode.

- Run grants explicit admission. Local Run suspends its caller until the callee

  yields, waits, exhausts its budget, or terminates. Remote Run uses the bounded

  request queue. Events may resume an admitted waiting context with positive

  retained budget. Yield requires another explicit Run.

- Dying domains lose lookup and dispatch authority. Each active architectural

  owner receives a reserved Stop request. Reclaim only after every frame returns

  and accounting completes. Destroy waits for every selected domain incarnation,

  including creation descendants, before replying. Stale requests cannot target

  a reused identity.

- Boot reserves bounded kernel tables from physical RAM before user allocation.

  Root receives device, interrupt, per-hart reserve-budget, startup-Page, and stack

  authority; a separate finite budget starts it on hart zero. Root Exit reclaims

  root normally and leaves the machine running.

- Root Device handles have Read, Write, Grant, and Transfer rights. Root can

  share device authority or transfer it to a driver domain.

- Control workloads cover local continuations, Page sharing, delegation after

  issuer death, reparenting, query faults, budget exhaustion, and cross-hart event

  handshakes. Two contexts in one domain change and observe shared state on

  different harts. After Destroy returns, a sentinel must remain unchanged

  through a timeout, exercising remote quiescence before reclamation.

## Source size

- Kernel mechanisms and machine entry contain 3,449 nonblank, non-comment source

  lines. All kernel Radiance/RAS files, including checks and workloads, contain

  6,977 such lines. The largest module has 458 lines.

- These are source-size metrics, not evidence of hostile-code isolation, hardware

  timing, or cache-coherence behavior.

make std-test bin-test

Run hosted mechanisms, context/timer entry, two-hart PLIC/UART delivery, and the eight-image native workload. Run the complete control workload on 1, 2, and 8 harts. Production boot must print CONTROL-OK, then reach the emulator tick limit with the machine still running after root exit. The check expects that deliberate limit exit rather than firmware shutdown.

The shared-backend smoke graph also exercises repeated, unaligned, and

transitive private pointers in independent instances. Oversized writable

addresses must return a capacity error before backend address patching. Machine

checks use emulator tick limits and deterministic interleaving; they are not

real-hardware tests. The trusted-input verification exclusions still apply.

export mod boot;

export mod calls;

export mod execution;

export mod control;

export mod teardown;

export mod runtime;

//! One-shot physical provisioning of kernel tables and explicit root authority.

//!

//! Kernel tables consume permanent frame allocations, never Page resources or

//! domain-lifetime grants. Their slices remain stable until the machine resets.

use core::abi;

use core::activation;

use core::atomic;

use core::budget_caps;

use core::budgets;

use core::contexts;

use core::domains;

use core::execution;

use core::fdt;

use core::frames;

use core::handles;

use core::interrupts;

use core::memory;

use core::pages;

use core::physical;

use core::platform;

use core::resources;

use core::smp;

use core::state;

use images;

/// Space for one Page per tracked frame plus all bounded non-Page resources.

constant RESOURCE_CAPACITY: u32 = frames::MAX_FRAMES + 512 + platform::MAX_DEVICES + platform::MAX_IRQS;

/// Root's authorized Page stack; activation allocates a separate native stack.

constant ROOT_STACK_FRAMES: u32 = 16;

/// Initial budget handles permit inspection, splitting, binding and delegation.

constant BUDGET_RIGHTS: u16 = abi::READ | abi::WRITE | abi::EXECUTE | abi::GRANT | abi::TRANSFER;

/// Release-published only after root admission and every shared table is ready.

static READY: u32 = 0;

/// Decoder output must not consume the hart's private 64 KiB machine stack.

/// Node properties borrow the firmware blob, which stays in reserved boot RAM.

static TREE: fdt::Tree = undefined;

/// Validated topology is immutable after publication.

static PLATFORM: platform::Platform = undefined;

/// Machine-lifetime frame ownership, never copied after tables borrow its address.

static POOL: frames::Pool = undefined;

/// Stable roots of permanently allocated capability and context tables.

static KERNEL: state::State = undefined;

/// Global interrupt state, protected by RUNTIME.lock after publication.

static CONTROLLER: interrupts::Controller = undefined;

/// Stable lock and pointers to permanent hardware-hart and admission tables.

static RUNTIME: execution::State = undefined;

/// Provision shared state once and acquire its publication on every online hart.

/// Firmware reserves the image, machine stacks and borrowed FDT for machine life.

/// Each caller must initialize its local runtime interrupt state before dispatch.

export unsafe fn start(hardware: u64, description: *u8) -> *mut execution::State {

    assert hardware < platform::MAX_HARTS as u64;

    let hart = hardware as u32;

    assert smp::hart() == hart;

    if hart == 0 {

        let size = try! fdt::word(@sliceOf(description, 40), 4);

        assert size >= 40 and size <= fdt::MAX_BYTES;

        try! fdt::decode(@sliceOf(description, size), &mut TREE);

        try! platform::discover(&TREE, &mut PLATFORM);

        try! frames::init(&mut POOL, &PLATFORM);

        let root = init(&mut KERNEL, &mut POOL, &PLATFORM, &mut CONTROLLER);

        initRuntime(&mut RUNTIME, &mut KERNEL, &PLATFORM, &mut CONTROLLER);

        // Activation and binding alone grant no dispatch admission.

        execution::root(&mut RUNTIME, root);

        atomic::store(&mut READY, 1);

    } else {

        // This is the only shared access permitted before acquiring publication.

        while atomic::load(&READY) == 0 {}

    }

    return &mut RUNTIME;

}

/// Allocate permanent, cleared kernel storage without a stack-local backing array.

/// The caller supplies a concrete type layout and retains the resulting slice.

unsafe fn storage(pool: *mut frames::Pool, count: u32, size: u32, alignment: u32) -> *mut opaque {

    assert count > 0 and size > 0;

    assert alignment > 0 and alignment <= frames::PAGE_SIZE;

    assert alignment & (alignment - 1) == 0;

    let bytes = count as u64 * size as u64;

    let frameCount = (bytes + frames::PAGE_SIZE as u64 - 1) / frames::PAGE_SIZE as u64;

    assert frameCount <= frames::MAX_FRAMES as u64;

    let run = frames::install(try! frames::allocate(pool, frameCount as u32, physical::clear));

    // No resource object or grant ever owns these frames. Only the availability

    // bitmap and permanent kernel slices retain their machine-lifetime ownership.

    return physical::bytes(run.first as u64 * frames::PAGE_SIZE as u64, run.count * frames::PAGE_SIZE).ptr as *mut opaque;

}

/// Initialize full-capacity tables directly in owned physical storage.

unsafe fn tables(kernel: *mut state::State, pool: *mut frames::Pool) {

    let domainStore = storage(pool, abi::MAX_DOMAINS, @sizeOf(domains::Domain), @alignOf(domains::Domain));

    let resourceStore = storage(pool, RESOURCE_CAPACITY, @sizeOf(resources::Slot), @alignOf(resources::Slot));

    let contextStore = storage(pool, contexts::CAPACITY, @sizeOf(contexts::Context), @alignOf(contexts::Context));

    let pinStore = storage(pool, frames::MAX_FRAMES, @sizeOf(u16), @alignOf(u16));

    let objectStore = storage(pool, frames::MAX_FRAMES, @sizeOf(bool), @alignOf(bool));

    let grantCount = frames::MAX_FRAMES / 64 * abi::MAX_DOMAINS;

    let grantStore = storage(pool, grantCount, @sizeOf(u64), @alignOf(u64));

    set kernel.domains = @sliceOf(domainStore as *mut domains::Domain, abi::MAX_DOMAINS);

    set kernel.resources = @sliceOf(resourceStore as *mut resources::Slot, RESOURCE_CAPACITY);

    set kernel.contexts = @sliceOf(contextStore as *mut contexts::Context, contexts::CAPACITY);

    domains::init(kernel.domains);

    resources::init(kernel.resources);

    for i in 0..kernel.contexts.len { contexts::init(&mut kernel.contexts[i], i); }

    memory::init(&mut kernel.memory, pool,

        @sliceOf(pinStore as *mut u16, frames::MAX_FRAMES),

        @sliceOf(objectStore as *mut bool, frames::MAX_FRAMES),

        @sliceOf(grantStore as *mut u64, grantCount), abi::MAX_DOMAINS);

}

/// Allocate machine-lifetime routing and hart-owned frames before publication.

/// kernel has been initialized by init. runtime and its lock must never move;

/// no domain Page, private-image allocation, or teardown owns these table frames.

/// Root still needs an explicit execution::root admission after this returns.

export unsafe fn initRuntime(runtime: *mut execution::State, kernel: *mut state::State,

    machine: *platform::Platform, controller: *mut interrupts::Controller) {

    let pool = kernel.memory.pool;

    let hartStore = storage(pool, platform::MAX_HARTS, @sizeOf(execution::Hart), @alignOf(execution::Hart));

    let routeStore = storage(pool, kernel.contexts.len, @sizeOf(execution::Route), @alignOf(execution::Route));

    execution::init(runtime, kernel, machine, controller,

        @sliceOf(hartStore as *mut execution::Hart, platform::MAX_HARTS),

        @sliceOf(routeStore as *mut execution::Route, kernel.contexts.len));

}

/// Publish one boot resource with its first reference in an exact root slot.

fn install(kernel: *mut state::State, slot: u32, value: resources::Value, rights: u16) -> abi::Handle {

    let object = try! resources::create(kernel.resources, value);

    resources::retain(kernel.resources, object);

    return handles::install(&mut kernel.domains[0].handles, slot, object, rights);

}

/// Provision a validated machine once, before secondary or user kernel entry.

/// pool is already initialized, excluding kernel/image/boot reservations; kernel,

/// pool, machine and controller remain kernel-owned and mapped throughout use.

/// Initialize the PLIC on hardware hart 0 and return root's activated context,

/// bound to a distinct finite bootstrap Budget. Allocation failure halts boot.

/// Startup is consecutive u64 words: version, deviceCount, irqCount, hartCount,

/// budgetBaseSlot, bootstrapBudgetSlot, timebaseFrequency, imageCount; then

/// (hardwareHartId, reserveBudgetSlot) pairs and catalog Image indices in order.

export unsafe fn init(kernel: *mut state::State, pool: *mut frames::Pool, machine: *platform::Platform, controller: *mut interrupts::Controller) -> u64 {

    assert pool.limit > 0 and pool.limit <= frames::MAX_FRAMES;

    assert machine.deviceCount <= platform::MAX_DEVICES;

    assert machine.irqCount > 0 and machine.irqCount < platform::MAX_IRQS;

    assert machine.hartCount > 0 and machine.hartCount <= platform::MAX_HARTS;

    assert machine.frequency > 0 and images::ROOT < images::COUNT;

    let budgetBaseSlot = 2 + machine.deviceCount + machine.irqCount;

    let bootstrapBudgetSlot = budgetBaseSlot + machine.hartCount;

    assert bootstrapBudgetSlot + 3 <= abi::MAX_HANDLES;

    let mut bootHartIndex = machine.hartCount;

    for i in 0..machine.hartCount {

        assert machine.harts[i].id < platform::MAX_HARTS;

        if machine.harts[i].id == 0 { set bootHartIndex = i; }

    }

    assert bootHartIndex < machine.hartCount;

    tables(kernel, pool);

    interrupts::init(controller, machine, bootHartIndex);

    let root = domains::root(kernel.domains, images::ROOT);

    let owner = try! domains::resolve(kernel.domains, 0, root, 0);

    let initial = try! activation::initialize(pool, images::ROOT, physical::clear, physical::copy);

    set kernel.domains[0].private = initial.run;

    set kernel.domains[0].env.stateBase = initial.base;

    // Fixed slots contain no implicit discovery or ambient resource authority.

    for i in 0..machine.deviceCount {

        let _device = install(kernel, 2 + i, resources::Value::Device(machine.devices[i].region), abi::READ | abi::WRITE | abi::GRANT | abi::TRANSFER);

    }

    for source in 1..machine.irqCount + 1 {

        let slot = 2 + machine.deviceCount + source - 1;

        let irq = install(kernel, slot, resources::Value::Interrupt(resources::Interrupt {

            number: source, target: owner,

        }), abi::RIGHTS);

        let entry = try! handles::require(&kernel.domains[0].handles, irq, abi::Kind::Interrupt, 0);

        interrupts::register(controller, source, entry.object);

    }

    let frequency = machine.frequency as u64;

    for i in 0..machine.hartCount {

        let reserve = try! budgets::init(machine.harts[i].id, frequency * 3600);

        let _budget = install(kernel, budgetBaseSlot + i, resources::Value::Budget(reserve), BUDGET_RIGHTS);

    }

    let bootstrap = try! budgets::init(0, frequency * 60);

    let bootBudget = install(kernel, bootstrapBudgetSlot, resources::Value::Budget(bootstrap), BUDGET_RIGHTS);

    // These two Page objects follow all fixed authority slots: startup, then stack.

    // Large catalogs may span several frames but still occupy one startup Page.

    let wordCount = 8 as u64 + machine.hartCount as u64 * 2 + images::COUNT as u64;

    let byteCount = wordCount * 8;

    let startupFrames = (byteCount + frames::PAGE_SIZE as u64 - 1) / frames::PAGE_SIZE as u64;

    let startup = try! pages::allocate(kernel, 0, root, startupFrames, physical::clear);

    let args = try! pages::access(kernel, 0, startup, abi::READ | abi::WRITE);

    let words = @sliceOf(physical::bytes(args.base, byteCount as u32).ptr as *mut u64, wordCount as u32);

    set words[0] = 1;

    set words[1] = machine.deviceCount as u64;

    set words[2] = machine.irqCount as u64;

    set words[3] = machine.hartCount as u64;

    set words[4] = budgetBaseSlot as u64;

    set words[5] = bootstrapBudgetSlot as u64;

    set words[6] = frequency;

    set words[7] = images::COUNT as u64;

    for i in 0..machine.hartCount {

        set words[8 + i * 2] = machine.harts[i].id as u64;

        set words[9 + i * 2] = (budgetBaseSlot + i) as u64;

    }

    for i in 0..images::COUNT {

        set words[8 + machine.hartCount * 2 + i] = i as u64;

    }

    let stack = try! pages::allocate(kernel, 0, root, ROOT_STACK_FRAMES as u64, physical::clear);

    let stackRange = try! pages::access(kernel, 0, stack, abi::WRITE);

    let id = try! activation::activate(kernel, 0, root, stackRange.base + stackRange.size,

        args.base, byteCount, physical::clear);

    try! budget_caps::bind(kernel, 0, bootBudget, root, id);

    return id;

}

//! Direct-call effects completed under the shared kernel critical section.

use core::abi;

use core::budget_caps;

use core::calls;

use core::contexts;

use core::cpu;

use core::domains;

use core::events;

use core::execution;

use core::lifecycle;

use core::notifications;

use core::smp;

use core::teardown;

use core::timers;

/// Query faults preserve the exact ABI error above the architectural fault range.

export constant QUERY_FAULT: u32 = 0x200;

/// An undefined direct-call number is a program fault.

export constant UNKNOWN_CALL: u32 = 0x102;

/// Determine whether this receiver has an admitted, budgeted waiter on a hart.

fn waiting(runtime: *mut execution::State, owner: abi::Object, hart: u32) -> bool {

    for i in 0..runtime.kernel.contexts.len {

        let context = &runtime.kernel.contexts[i];

        if context.hart <> hart or context.status <> contexts::Status::Waiting

            or not execution::same(context.owner, owner) or context.budget.kind <> abi::Kind::Budget { continue; }

        if execution::route(runtime, i).admitted and budget_caps::value(runtime.kernel, context.budget).remaining > 0 {

            return true;

        }

    }

    return false;

}

/// Construct a domain-wide notification hint; it contains no execution authority.

fn wake(owner: abi::Object) -> smp::Request {

    return smp::Request { kind: smp::RequestKind::Wake, domain: owner, context: 0 };

}

/// Check every required remote slot before committing a Wakeup event.

fn prepareWake(runtime: *mut execution::State, index: u32, arguments: *[u64]) -> abi::Object throws (abi::Error) {

    if arguments[1] > 0xffffffff { throw abi::Error::InvalidArg; }

    let source = &runtime.kernel.contexts[index];

    let owner = try domains::resolve(runtime.kernel.domains, source.owner.index,

        abi::Handle { bits: arguments[0] }, abi::WAKE);

    try events::refresh(&mut runtime.kernel.domains[owner.index].events) catch {

        lifecycle::terminate(runtime.kernel, owner, 3, teardown::BAD_EVENTS);

        throw abi::Error::BadHandle;

    };

    for hart in 0..runtime.harts.len {

        if hart <> source.hart and runtime.harts[hart].online and waiting(runtime, owner, hart)

            and not smp::canPost(&runtime.harts[hart].mailbox, wake(owner)) { throw abi::Error::Busy; }

    }

    return owner;

}

/// Publish all preflighted wake requests in the same critical section as the event.

unsafe fn publishWake(runtime: *mut execution::State, source: u32, owner: abi::Object) {

    for hart in 0..runtime.harts.len {

        if hart == source or not runtime.harts[hart].online or not waiting(runtime, owner, hart) { continue; }

        try! smp::post(&mut runtime.harts[hart].mailbox, wake(owner));

        smp::kick(runtime.machine.clint.base, runtime.harts[hart].hardware.clintIndex);

    }

}

/// Identify direct calls whose typed interface cannot expose an error result.

fn query(number: u64) -> bool {

    return (number >= 44 and number <= 48) or number == 62 or number == 74 or number == 75;

}

/// Distinguish undefined calls from errors returned by defined control operations.

fn defined(number: u64) -> bool {

    return (number >= 10 and number <= 12) or (number >= 20 and number <= 23)

        or number == 30 or number == 31 or (number >= 40 and number <= 50)

        or (number >= 60 and number <= 62) or (number >= 70 and number <= 76);

}

/// Commit one call. Architectural return and budget charging must precede this.

unsafe fn apply(runtime: *mut execution::State, index: u32, now: u64) throws (abi::Error) {

    let kernel = runtime.kernel;

    let context = &mut kernel.contexts[index];

    let frame = &mut context.frame;

    let number = frame.registers[17];

    let arguments = &frame.registers[10..17];

    let mut receiver = domains::none();

    if number == 50 { set receiver = try prepareWake(runtime, index, arguments); }

    let outcome = try calls::dispatch(kernel, context.owner.index, number, arguments, now) catch error {

        // A remote consumer can publish its head between preflight and push.

        // Invalid progress faults that receiver, not the authorized sender.

        if number == 50 and error == abi::Error::InvalidArg {

            lifecycle::terminate(kernel, receiver, 3, teardown::BAD_EVENTS);

            throw abi::Error::BadHandle;

        }

        throw error;

    };

    match outcome {

        case calls::Outcome::Reply { values } => {

            if number == 50 { publishWake(runtime, context.hart, receiver); }

            cpu::reply(frame, abi::Error::Ok, values[0], values[1], values[2], values[3]);

        },

        case calls::Outcome::Yield => {

            cpu::reply(frame, abi::Error::Ok, 0, 0, 0, 0);

            set context.status = contexts::Status::Stopped;

            set execution::route(runtime, index).admitted = false;

            execution::relinquish(runtime, index);

        },

        case calls::Outcome::Wait => {

            try notifications::wait(kernel, index);

            cpu::reply(frame, abi::Error::Ok, 0, 0, 0, 0);

            if context.status == contexts::Status::Waiting { execution::relinquish(runtime, index); }

        },

        case calls::Outcome::Timeout { ticks, token } => {

            try timers::arm(&mut runtime.harts[context.hart].timers, context.owner, now, ticks, token);

            cpu::reply(frame, abi::Error::Ok, 0, 0, 0, 0);

        },

        case calls::Outcome::Terminate { kind, code } => lifecycle::terminate(kernel, context.owner, kind, code),

        case calls::Outcome::Destroy { handle, flags } => try teardown::request(runtime, index, handle, flags),

        case calls::Outcome::Run { context: target } => {

            try execution::run(runtime, index, target);

            cpu::reply(frame, abi::Error::Ok, 0, 0, 0, 0);

        },

    }

}

/// Dispatch a returned U-mode ecall, faulting queries and undefined operations.

export unsafe fn invoke(runtime: *mut execution::State, index: u32, now: u64) {

    let context = &mut runtime.kernel.contexts[index];

    let number = context.frame.registers[17];

    assert context.frame.cause == 8 and context.status == contexts::Status::Ready;

    try apply(runtime, index, now) catch error {

        if query(number) {

            lifecycle::terminate(runtime.kernel, context.owner, 3, QUERY_FAULT + error as u32);

        } else if not defined(number) {

            lifecycle::terminate(runtime.kernel, context.owner, 3, UNKNOWN_CALL);

        } else if number == 42 {

            lifecycle::terminate(runtime.kernel, context.owner, 3, teardown::BAD_EVENTS);

        } else {

            cpu::reply(&mut context.frame, error, 0, 0, 0, 0);

        }

    };

}

//! Explicit execution admission, continuations, and hart-owned remote requests.

use core::abi;

use core::budget_caps;

use core::contexts;

use core::cpu;

use core::domains;

use core::interrupts;

use core::platform;

use core::smp;

use core::state;

use core::timers;

/// Private admission state for one incarnation-qualified context.

export record Route {

    /// Context identity to which all remaining fields belong.

    identity: u64,

    /// An explicit Run or boot admission remains in force.

    admitted: bool,

    /// The owning hart has not consumed its Run request.

    queued: bool,

    /// Same-hart callee that must relinquish execution before this caller resumes.

    suspended: u64,

    /// A successful Destroy reply awaits all snapshotted incarnations.

    destroying: bool,

    /// Selected domain epochs; zero excludes a slot. Read only while destroying.

    targets: [u32; abi::MAX_DOMAINS],

}

/// Private architectural routing and bounded asynchronous state for one hart.

export record Hart {

    /// Firmware admitted this hardware hart.

    online: bool,

    /// Validated CLINT and PLIC routing.

    hardware: platform::Hart,

    /// Context whose architectural frame this hart owns outside the lock.

    current: u64,

    /// Explicit continuation or interrupted context preferred at the next dispatch.

    preferred: u64,

    /// Bounded ordinary Run and Wake transport.

    mailbox: smp::Mailbox,

    /// Reserved stop request for this hart's single architectural owner.

    stop: ?smp::Request,

    /// Accepted one-shot timers, serialized by the global lock.

    timers: timers::Store,

    /// Private trap frame used only by this hart's idle entry.

    idle: cpu::Frame,

}

/// Stable shared runtime storage; every field except owned frames needs lock.

export record State {

    /// Capability, physical-memory, and context mechanisms.

    kernel: *mut state::State,

    /// Immutable firmware description.

    machine: *platform::Platform,

    /// Global PLIC service state.

    controller: *mut interrupts::Controller,

    /// Hardware-ID-indexed hart records.

    harts: *mut [Hart],

    /// Context-indexed admission records.

    routes: *mut [Route],

    /// Exact-use critical-section ownership.

    lock: smp::Lock,

}

/// Initialize one unshared admission record or discard an expired incarnation.

fn reset(route: *mut Route, identity: u64) {

    set route.identity = identity;

    set route.admitted = false;

    set route.queued = false;

    set route.suspended = 0;

    set route.destroying = false;

    set route.targets = [0; abi::MAX_DOMAINS];

}

/// Initialize stable storage once before publication to secondary harts.

export fn init(runtime: *mut State, kernel: *mut state::State, machine: *platform::Platform,

    controller: *mut interrupts::Controller, harts: *mut [Hart], routes: *mut [Route]) {

    assert harts.len == platform::MAX_HARTS and routes.len == kernel.contexts.len;

    set runtime.kernel = kernel;

    set runtime.machine = machine;

    set runtime.controller = controller;

    set runtime.harts = harts;

    set runtime.routes = routes;

    smp::init(&mut runtime.lock);

    for i in 0..routes.len { reset(&mut routes[i], 0); }

    for i in 0..harts.len {

        let hart = &mut harts[i];

        set hart.online = false;

        set hart.hardware = platform::Hart { id: i, controller: 0, clintIndex: 0, plicContext: 0 };

        set hart.current = 0;

        set hart.preferred = 0;

        set hart.stop = nil;

        smp::initMailbox(&mut hart.mailbox);

        timers::init(&mut hart.timers);

        cpu::init(&mut hart.idle, 0, 0, 0, 0);

    }

    for i in 0..machine.hartCount {

        let hardware = machine.harts[i];

        assert hardware.id < harts.len;

        set harts[hardware.id].online = true;

        set harts[hardware.id].hardware = hardware;

    }

}

/// Borrow admission state for the current live context incarnation under lock.

export fn route(runtime: *mut State, index: u32) -> *mut Route {

    let id = contexts::identity(&runtime.kernel.contexts[index]);

    let entry = &mut runtime.routes[index];

    if entry.identity <> id { reset(entry, id); }

    return entry;

}

/// Check complete domain identity without treating a transport message as authority.

export fn same(left: abi::Object, right: abi::Object) -> bool {

    return left.kind == right.kind and left.index == right.index and left.epoch == right.epoch;

}

/// Release a same-hart caller after yield, wait, exhaustion, or termination.

/// Skip dead or exhausted callers without depending on context-table ordering.

/// Every link names both context incarnations, and Run cannot form a cycle.

export fn relinquish(runtime: *mut State, index: u32) {

    let mut child = index;

    for depth in 0..runtime.kernel.contexts.len {

        let context = &mut runtime.kernel.contexts[child];

        assert context.status <> contexts::Status::Running;

        let caller = context.continuation;

        set context.continuation = 0;

        if caller == 0 { return; }

        let parent = try contexts::resolve(runtime.kernel.contexts, caller) catch { return; };

        let entry = route(runtime, parent);

        if entry.suspended <> contexts::identity(context) { return; }

        set entry.suspended = 0;

        let resumed = &mut runtime.kernel.contexts[parent];

        assert resumed.hart == context.hart and resumed.status == contexts::Status::Stopped;

        if domains::live(runtime.kernel.domains, resumed.owner) and entry.admitted {

            assert not entry.queued and not entry.destroying;

            if resumed.budget.kind == abi::Kind::Budget

                and budget_caps::value(runtime.kernel, resumed.budget).remaining > 0 {

                set resumed.status = contexts::Status::Ready;

                set runtime.harts[resumed.hart].preferred = caller;

                return;

            }

        }

        set entry.admitted = false;

        set child = parent;

    }

}

/// Admit boot's already activated, budget-bound root without an implicit caller.

export fn root(runtime: *mut State, id: u64) {

    let index = try! contexts::resolve(runtime.kernel.contexts, id);

    let context = &runtime.kernel.contexts[index];

    assert context.hart == 0 and context.status == contexts::Status::Ready;

    assert runtime.harts[0].online and domains::live(runtime.kernel.domains, context.owner);

    assert runtime.kernel.domains[context.owner.index].state == domains::Lifecycle::Active;

    let budget = budget_caps::value(runtime.kernel, context.budget);

    assert budget.context == id and budget.hart == 0 and budget.remaining > 0;

    set route(runtime, index).admitted = true;

    set runtime.harts[0].preferred = id;

}

/// Admit an authorized Run atomically with transport capacity and continuation state.

/// The control dispatcher has checked Execute rights and the retained budget.

export unsafe fn run(runtime: *mut State, caller: u32, id: u64) throws (abi::Error) {

    let index = try contexts::resolve(runtime.kernel.contexts, id);

    let context = &mut runtime.kernel.contexts[index];

    let entry = route(runtime, index);

    if entry.admitted or entry.destroying or entry.suspended <> 0 { throw abi::Error::Busy; }

    let hardware = context.hart;

    if hardware >= runtime.harts.len or not runtime.harts[hardware].online { throw abi::Error::Denied; }

    let source = &mut runtime.kernel.contexts[caller];

    assert source.status == contexts::Status::Ready and context.continuation == 0;

    let sourceRoute = route(runtime, caller);

    assert sourceRoute.admitted and sourceRoute.suspended == 0 and not sourceRoute.destroying;

    let message = smp::Request { kind: smp::RequestKind::Run, domain: context.owner, context: id };

    if source.hart <> hardware and not smp::canPost(&runtime.harts[hardware].mailbox, message) {

        throw abi::Error::Busy;

    }

    set entry.admitted = true;

    set context.status = contexts::Status::Ready;

    if source.hart == hardware {

        set context.continuation = contexts::identity(source);

        set sourceRoute.suspended = id;

        set source.status = contexts::Status::Stopped;

        set runtime.harts[hardware].preferred = id;

    } else {

        set entry.queued = true;

        try! smp::post(&mut runtime.harts[hardware].mailbox, message);

        smp::kick(runtime.machine.clint.base, runtime.harts[hardware].hardware.clintIndex);

    }

}

/// Test whether explicit admission and finite retained authority permit dispatch.

fn ready(runtime: *mut State, index: u32, hart: u32) -> bool {

    let context = &runtime.kernel.contexts[index];

    if context.status <> contexts::Status::Ready or context.hart <> hart

        or not domains::live(runtime.kernel.domains, context.owner) { return false; }

    if runtime.kernel.domains[context.owner.index].state <> domains::Lifecycle::Active { return false; }

    let entry = route(runtime, index);

    if not entry.admitted or entry.queued or entry.destroying or entry.suspended <> 0 { return false; }

    if context.budget.kind <> abi::Kind::Budget { return false; }

    return budget_caps::value(runtime.kernel, context.budget).remaining > 0;

}

/// Resolve a preferred incarnation without admitting any other table occupant.

fn preference(runtime: *mut State, hart: u32, id: u64) -> ?u32 {

    let index = try contexts::resolve(runtime.kernel.contexts, id) catch { return nil; };

    if ready(runtime, index, hart) { return index; }

    return nil;

}

/// Select an explicit continuation first, then retained ready admissions on this hart.

/// This adds no authority, quantum, replenishment, or priority to a context.

export fn select(runtime: *mut State, hart: u32) -> ?u32 {

    let preferred = runtime.harts[hart].preferred;

    set runtime.harts[hart].preferred = 0;

    if preferred <> 0 {

        if let index = preference(runtime, hart, preferred) { return index; }

    }

    for i in 0..runtime.kernel.contexts.len {

        if ready(runtime, i, hart) { return i; }

    }

    return nil;

}

/// Acknowledge under the publication lock, then reject stale request identities.

export unsafe fn drain(runtime: *mut State, hart: u32) {

    let local = &mut runtime.harts[hart];

    assert local.current == 0;

    smp::acknowledge(runtime.machine.clint.base, local.hardware.clintIndex);

    // The owning hart reaches drain only after consuming its Running obligation.

    // A stale stop can be discarded, but a matching frame must be quiescent.

    if let request = local.stop {

        assert request.kind == smp::RequestKind::Stop;

        let index = request.context as u32;

        if index < runtime.kernel.contexts.len {

            let context = &runtime.kernel.contexts[index];

            if contexts::identity(context) == request.context and same(context.owner, request.domain) {

                assert context.hart == hart and context.status <> contexts::Status::Running;

            }

        }

    }

    set local.stop = nil;

    while let request = smp::take(&mut local.mailbox) {

        if not domains::live(runtime.kernel.domains, request.domain) { continue; }

        if request.kind == smp::RequestKind::Wake { continue; }

        let index = try contexts::resolve(runtime.kernel.contexts, request.context) catch { continue; };

        let context = &runtime.kernel.contexts[index];

        if context.hart <> hart or not same(context.owner, request.domain) { continue; }

        if request.kind == smp::RequestKind::Run { set route(runtime, index).queued = false; }

    }

}