radiance · commit

kernel: Complete cross-hart transactions

e654149f88afb6f0ed0cd40ed0f32ea386f397bb0c2106e3321cba2af63e450d

Execute remote budget changes at the owning hart after saving its
context. Reserve bounded readiness batches before activation or
wakeup, retain preemptible callers through acknowledgement, and notify
migrated timeout receivers.

Validate queue capacity, stale replies, queued expiration, context
retention, and atomic rejection. Native 1/2/8-hart checks cover
concurrent calls, context migration, idle-hart activation and wakeup,
and remote timeout delivery. Full make test passes with the sibling
kernel-emulator.

Alexis Sellier committed 13 days ago 1 parent 8aaf2eee

kernel/kernel.rad +1 -0

export mod instances;
export mod domains;
export mod budgets;
export mod dispatch;
export mod timers;
export mod remote;
export mod calls;
export mod sys;
export mod boot;
@test export mod tests;

kernel/kernel/boot.rad +5 -2

use super::registry;
use super::domains;
use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use std::arch::rv64::shared::catalog;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.

/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and user calls, and stop on unsupported traps.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    if trap::classify(frame.cause) == trap::Cause::Timer {
    let cause = trap::classify(frame.cause);
    if cause == trap::Cause::Timer or cause == trap::Cause::Software {
        dispatch::interrupt(frame, hart);
        return;
    }
    if trap::classify(frame.cause) == trap::Cause::UserCall and trap::fromUser(frame) {
    if cause == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);
        return;
    }
    print("kernel: unexpected trap\n");
    trap::halt();

        try! platform::protect(&mut PLATFORM, treeRange);
        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        domains::initialize(&mut domains::STORE);
        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        assert entryCount == 2;
        registry::initialize(&mut registry::STORE);
        let bootCatalog = @sliceOf(entries, entryCount);
        try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
        let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

kernel/kernel/budgets.rad +16 -0

/// Test half-open execution authority on the selected hart at the current tick.
export fn active(store: &Store, object: abi::Ref, hart: u32, now: u64) -> bool throws (abi::Error) {
    let window = try get(store, object);
    return window.context <> nil and window.hart == hart and window.start <= now and now < window.end;
}

/// Find all harts with retained authority for a receiver's live contexts.
export fn wakeTargets(windows: &Store, contexts: &domains::Store, receiver: abi::Ref, now: u64) -> u32 {
    let mut targets: u32 = 0;
    for i in 0..limits::BUDGETS {
        if windows.slots[i].state <> slots::State::Live { continue; }
        let window = windows.windows[i];
        if window.owner <> receiver or window.end <= now { continue; }
        let context = window.context else { continue; };
        if not slots::matches(&contexts.contextSlots[..], context, slots::State::Live) { continue; }
        if contexts.contexts[context.index].owner <> receiver
            or contexts.contexts[context.index].state == domains::ContextState::Stopped { continue; }
        set targets |= 1 << window.hart;
    }
    return targets;
}

kernel/kernel/calls.rad +69 -1

use super::transactions;
use super::loader;
use super::dispatch;
use super::timers;
use super::sync;
use super::remote;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
    match operation {
        case 11, 12, 70, 71, 73 => {},
        else => return nil,
    }
    let entry = try capability::get(table, abi::Handle(arguments[0]));
    if entry.kind <> abi::Kind::Budget { return nil; }
    return (try budgets::get(&budgets::STORE, entry.object)).hart;
}

/// Execute a queued budget change at its owning hart's dispatch boundary.
export unsafe fn remoteBudget(call: remote::Call, hart: u32, now: u64) -> remote::Reply {
    let domain = try domains::get(&domains::STORE, call.owner) catch error { return remote::Reply::Error(error); };
    if domain.state <> domains::Lifecycle::Active { return remote::Reply::Error(abi::Error::BadHandle); }
    let target = try budgetHart(&domain.memory.table, call.operation, &call.arguments[..]) catch error {
        return remote::Reply::Error(error);
    };
    if target <> hart { return remote::Reply::Error(abi::Error::BadHandle); }
    let result = try invoke(call.owner, call.operation, &call.arguments[..], now) catch error {
        return remote::Reply::Error(error);
    };
    return remote::Reply::Value(result);
}

/// Validate resident object generations for capabilities without page ownership.
unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },

    sync::leave(publicationGuard);
    return *result;
}

/// Execute a call with short metadata transactions and private bulk initialization.
/// The running context keeps its domain storage live until dispatch releases it.
/// Context ownership and pending calls retain storage across dispatch boundaries.
export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    if operation == 21 or operation == 50 { return try runtimeReady(owner, operation, arguments, now); }
    let guard = sync::enter();
    let mut domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
    if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
    let target = try budgetHart(&domain.memory.table, operation, arguments) catch error { sync::leave(guard); throw error; };
    if let destination = target {
        let source = dispatch::hart();
        if destination <> source {
            let call = remote::Call { owner, context: abi::reference(dispatch::current().context), operation,
                arguments: [arguments[0], arguments[1], arguments[2], arguments[3]] };
            let request = try dispatch::request(destination, remote::Action::Budget(call)) catch error {
                sync::leave(guard); throw error;
            };
            assert domains::STORE.contexts[call.context.index].pending == nil;
            set domains::STORE.contexts[call.context.index].pending = request;
            sync::leave(guard);
            return try dispatch::awaitRequest(source, request);
        }
    }
    let handle = abi::Handle(arguments[0]);
    match operation {
        case 20 => {
            let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
                &mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error { sync::leave(guard); throw error; };

            sync::leave(guard); return result;
        },
    }
}

/// Reserve remote dispatch capacity before activation or event publication.
unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) { throw abi::Error::InvalidArg; }
    let guard = sync::enter();
    let domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
    if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
    let handle = abi::Handle(arguments[0]);
    let rights = abi::EXECUTE if operation == 21 else abi::WAKE;
    let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
        sync::leave(guard); throw error;
    };
    let source = dispatch::hart();
    let context = abi::reference(dispatch::current().context);
    let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
    let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error { sync::leave(guard); throw error; };
    let result = try invoke(owner, operation, arguments, now) catch error {
        remote::cancelReservation(&mut remote::STORE, pending);
        sync::leave(guard); throw error;
    };
    let batch = remote::publish(&mut remote::STORE, pending, remote::Action::Ready(context));
    dispatch::startReady(context, &batch);
    sync::leave(guard);
    dispatch::finishReady(context, &batch);
    return 0;
}

/// Complete a user ecall through the supplied execution boundary.
export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
    execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
{
    assert trap::fromUser(frame) and frame.cause == 8;

kernel/kernel/dispatch.rad +147 -10

use super::trap;
use super::range;
use super::calls;
use super::timers;
use super::sync;
use super::remote;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

export record Timer: Copy {
    /// Shared monotonic mtime register.
    clock: u64,
    /// This hart's mtimecmp register.
    compare: u64,
    /// This hart's machine software-interrupt word.
    software: u64,
}

/// Timer and dispatch state owned by one physical hart.
record Runtime: Copy {
    /// Current context and retained idle frame.


/// Read the executing physical hart identifier.
export fn hart() -> u32;
/// Read the current machine stack pointer.
fn stackPointer() -> u64;
/// Trap anchor of the hart currently executing this continuation.
fn localAnchor() -> *mut trap::Hart;
/// Address of the M-mode idle loop.
fn idleAddress() -> u64;
/// Enable machine timer delivery while global interrupts remain disabled.
/// Enable timer and software interrupt delivery with global interrupts disabled.
fn enable();

/// Initialize this hart's idle frame and validated timer before installing dispatch.
export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
    let device = try timer(machine, id);

    assert id < limits::HARTS;
    let stack = HARTS[id].state.stack;
    let pointer = stackPointer();
    assert pointer >= stack.start and pointer < stack.end;
    let guard = sync::enter();
    writeSoftware(HARTS[id].timer.software, 0);
    let previous = HARTS[id].state.current;
    save(&mut HARTS[id].state, &mut domains::STORE, frame);
    let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
    let mut count: u32 = 0;
    for i in 0..limits::REMOTE_PER_HART {
        let delivery = try! remote::take(&mut remote::STORE, id) else { break; };
        match delivery.request.action {
            case remote::Action::InstructionSync => sync::syncInstructions(),
            case remote::Action::Reschedule, remote::Action::Ready(_) => {},
            case remote::Action::Budget(call) => {
                let reply = calls::remoteBudget(call, id, now(HARTS[id].timer));
                remote::respond(&mut remote::STORE, id, delivery.id, reply);
            },
        }
        set deliveries[count] = delivery;
        set count += 1;
    }
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout < deadline { set deadline = timeout; }
    if timeout.deadline < deadline { set deadline = timeout.deadline; }
    for target in 0..limits::HARTS {
        if target <> id and (timeout.harts & (1 << target)) <> 0 { signal(target); }
    }
    arm(HARTS[id].timer, deadline);
    if previous <> choice.context {
        if let context = previous { signalContext(context, clock); }
    }
    for delivery in &deliveries[..count] {
        assert remote::acknowledge(&mut remote::STORE, id, delivery.id);
        if delivery.request.source <> id { signal(delivery.request.source); }
        if let case remote::Action::Budget(call) = delivery.request.action { signalContext(call.context, clock); }
        if let case remote::Action::Ready(context) = delivery.request.action { signalContext(context, clock); }
    }
    sync::leave(guard);
}

/// Signal an initialized online hart while request metadata is serialized.
unsafe fn signal(target: u32) { writeSoftware(HARTS[target].timer.software, 1); }

/// Notify the current owner or the eligible hart after a context is released.
unsafe fn signalContext(context: abi::Ref, now: u64) {
    if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) { return; }
    if let owner = domains::STORE.contexts[context.index].hart { signal(owner); return; }
    for i in 0..limits::BUDGETS {
        if budgets::STORE.slots[i].state <> slots::State::Live { continue; }
        let window = budgets::STORE.windows[i];
        if window.context == context and window.start <= now and now < window.end { signal(window.hart); }
    }
}

/// Write one naturally aligned CLINT software-interrupt register.
fn writeSoftware(address: u64, value: u32);

/// Post a local action and interrupt its target under metadata serialization.
export unsafe fn request(target: u32, action: remote::Action) -> abi::Ref throws (abi::Error) {
    let id = try remote::send(&mut remote::STORE, hart(), target, action);
    signal(target);
    return id;
}

/// Retain the caller and signal every published readiness request under the lock.
export unsafe fn startReady(context: abi::Ref, batch: &remote::Batch) {
    assert domains::STORE.contexts[context.index].pending == nil;
    for target in 0..limits::HARTS {
        if (batch.targets & (1 << target)) == 0 { continue; }
        if domains::STORE.contexts[context.index].pending == nil {
            set domains::STORE.contexts[context.index].pending = batch.ids[target];
        }
        signal(target);
    }
}

/// Consume every readiness acknowledgement before releasing the caller's storage.
export unsafe fn finishReady(context: abi::Ref, batch: &remote::Batch) {
    for target in 0..limits::HARTS {
        if (batch.targets & (1 << target)) == 0 { continue; }
        waitRequest(batch.source, batch.ids[target]);
    }
    let guard = sync::enter();
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    for target in 0..limits::HARTS {
        if (batch.targets & (1 << target)) == 0 { continue; }
        assert try! remote::result(&remote::STORE, batch.source, batch.ids[target]) == 0;
        try! remote::release(&mut remote::STORE, batch.source, batch.ids[target]);
    }
    set domains::STORE.contexts[context.index].pending = nil;
    sync::leave(guard);
}

/// Wait for an interrupt with global interrupts masked, then enable delivery.
fn waitInterrupt();

/// Release a completed call's context retention and request generation.
unsafe fn releaseRequest(source: u32, request: abi::Ref) {
    if let case remote::Action::Budget(call) = remote::STORE.requests[request.index].action {
        assert slots::matches(&domains::STORE.contextSlots[..], call.context, slots::State::Live);
        assert domains::STORE.contexts[call.context.index].pending == request;
        set domains::STORE.contexts[call.context.index].pending = nil;
    }
    try! remote::release(&mut remote::STORE, source, request);
}

/// Await acknowledgement while retaining the record and allowing preemption.
unsafe fn waitRequest(source: u32, request: abi::Ref) {
    let interrupts = sync::maskInterrupts();
    loop {
        let guard = sync::enter();
        let complete = try! remote::completed(&remote::STORE, source, request);
        sync::leave(guard);
        if complete { break; }
        waitInterrupt();
        sync::maskInterrupts();
    }
    sync::restoreInterrupts(interrupts);
}

/// Await a retained reply through a preemptible kernel continuation.
export unsafe fn awaitRequest(source: u32, request: abi::Ref) -> u64 throws (abi::Error) {
    waitRequest(source, request);
    let guard = sync::enter();
    let result = try remote::result(&remote::STORE, source, request) catch error {
        releaseRequest(source, request);
        sync::leave(guard); throw error;
    };
    releaseRequest(source, request);
    sync::leave(guard); return result;
}

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else panic "no current context";

        sync::leave(guard);
    } else {
        sync::leave(guard);
        calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
    }
    interrupt(frame, anchor);
    interrupt(frame, &mut *localAnchor());
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();

        if device.kind <> platform::Kind::Clint { continue; }
        if result <> nil or device.memory.end < device.memory.start
            or device.memory.end - device.memory.start < 0xc000 or (device.memory.start & 7) <> 0 {
            throw abi::Error::InvalidArg;
        }
        set result = Timer { clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8 };
        set result = Timer {
            clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8,
            software: device.memory.start + hart as u64 * 4,
        };
    }
    let selected = result else { throw abi::Error::InvalidArg; };
    return selected;
}


/// Interrupts remain disabled and shared metadata stays serialized through exchange.
export fn exchange(state: &mut State, windows: &budgets::Store, contexts: &mut domains::Store,
    frame: &mut trap::Frame, anchor: &mut trap::Hart, now: u64) -> Choice throws (abi::Error)
{
    let choice = try select(windows, contexts, state.hart, now);
    if let current = state.current {
        assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
        set contexts.contexts[current.index].frame = *frame;
        set contexts.contexts[current.index].hart = nil;
    }
    save(state, contexts, frame);
    if let next = choice.context {
        let context = contexts.contexts[next.index];
        assert context.kernelStack.start < context.kernelStack.end;
        set *frame = context.frame;
        set anchor.stackBottom = context.kernelStack.start;

    }
    set state.current = choice.context;
    set state.budget = choice.budget;
    return choice;
}

/// Retain a context frame before releasing this hart's execution ownership.
fn save(state: &mut State, contexts: &mut domains::Store, frame: &trap::Frame) {
    if let current = state.current {
        assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
        set contexts.contexts[current.index].frame = *frame;
        set contexts.contexts[current.index].hart = nil;
    }
    set state.current = nil;
    set state.budget = nil;
}

kernel/kernel/dispatch.ras +22 -1

//! Naturally aligned RV64 CLINT access with device ordering.
.text;
.export @kernel::dispatch::read;
.export @kernel::dispatch::write;
.export @kernel::dispatch::writeSoftware;
.export @kernel::dispatch::hart;
.export @kernel::dispatch::stackPointer;
.export @kernel::dispatch::localAnchor;
.export @kernel::dispatch::idleAddress;
.export @kernel::dispatch::enable;
.export @kernel::dispatch::interrupt;
.export @kernel::dispatch::waitInterrupt;

// A pending enabled interrupt wakes wfi while global delivery is masked.
@kernel::dispatch::waitInterrupt
    wfi;
    csrsi mstatus 8;
    ret;

// Retain the frame outside the context stack before publishing its release.
@kernel::dispatch::interrupt
    mv %t0 %a0;
    addi %t1 %a1 64;

// Dispatch releases context stacks only from the hart's firmware stack.
@kernel::dispatch::stackPointer
    mv %a0 %sp;
    ret;

// A resumed kernel continuation uses its executing hart's trap storage.
@kernel::dispatch::localAnchor
    csrr %a0 mscratch;
    ret;

// Saved idle frames enter this loop with machine interrupts enabled by mret.
@kernel::dispatch::idleAddress
    la %a0 @dispatchIdle;
    ret;
@dispatchIdle

    j @dispatchIdle;

// Global interrupt enable is restored from the selected frame by mret.
@kernel::dispatch::enable
    csrr %t0 mie;
    ori %t0 %t0 128;
    ori %t0 %t0 136;
    csrw mie %t0;
    ret;

// The validated CLINT mapping contains a naturally aligned mtime register.
@kernel::dispatch::read

@kernel::dispatch::write
    fence iorw iorw;
    sd %a1 0(%a0);
    fence iorw iorw;
    ret;

// CLINT software-interrupt registers are naturally aligned 32-bit words.
@kernel::dispatch::writeSoftware
    fence iorw iorw;
    sw %a1 0(%a0);
    fence iorw iorw;
    ret;

kernel/kernel/domains.rad +5 -3

    owner: abi::Ref,
    /// Whether this context can resume.
    state: ContextState,
    /// Hart retaining this context as its current execution owner.
    hart: ?u32,
    /// Remote call whose retained continuation requires this context's storage.
    pending: ?abi::Ref,
    /// Physical frames retained until this context stops.
    kernelFrames: frames::Run,
    /// Exclusive stack bounds used by trap entry and suspended M-mode calls.
    kernelStack: range::Range,
    /// Exclusive user stack extent; empty until initial activation.

    authority: abi::Handle, object: abi::Ref) throws (abi::Error)
{
    let permit = try capability::authority(table, authority, abi::Rights(abi::DESTROY));
    let target = try context(store, permit.object, object);
    if store.records[permit.object.index].initial == object { throw abi::Error::Denied; }
    if target.hart <> nil { throw abi::Error::Busy; }
    if target.hart <> nil or target.pending <> nil { throw abi::Error::Busy; }
    try! frames::release(&mut memory.backings.pool, target.kernelFrames);
    try! slots::release(&mut store.contextSlots[..], object);
}

/// Derive a descending stack extent from one writable Page capability.

    set frame.registers[3] = domain.graph.table.ptr as u64;
    set frame.registers[10] = start.args;
    set frame.registers[11] = start.size;
    let object = slots::reference(&slot);
    set store.contexts[object.index] = Context {
        owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
        owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames, kernelStack, userStack, frame,
    };
    match slot {
        case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
            object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
        }),

            // MPIE enables interrupts after the first return into U-mode.
            let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
            set frame.registers[1] = returnAddress();
            set frame.registers[3] = graph.table.ptr as u64;
            set store.contexts[initial.index] = Context {
                owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,

kernel/kernel/limits.rad +4 -0

export constant CONTEXTS: u32 = 2048;
/// Maximum hardware threads.
export constant HARTS: u32 = 8;
/// One-shot timeouts retained by each hart, including pending delivery.
export constant TIMEOUTS_PER_HART: u32 = 64;
/// Outstanding cross-hart requests, including acknowledgements awaiting release.
export constant REMOTE_REQUESTS: u32 = 128;
/// Queued remote requests accepted by each hart.
export constant REMOTE_PER_HART: u32 = 16;
/// Maximum resident package records and gp table entries.
export constant PACKAGES: u32 = 256;
/// Generation values never wrap.
export constant LAST_GENERATION: u32 = 0xffffffff;

kernel/kernel/remote.rad added +269 -0

1	+	//! Fixed request queues and generation-checked completion records.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::slots;
5	+
6	+	/// A scalar operation retained until its owning hart reaches dispatch.
7	+	export record Call: Copy {
8	+	/// Domain whose capability table supplies the arguments.
9	+	owner: abi::Ref,
10	+	/// Context whose kernel continuation consumes the result.
11	+	context: abi::Ref,
12	+	/// Budget operation to execute with serialized metadata.
13	+	operation: u64,
14	+	/// Original scalar syscall arguments.
15	+	arguments: [u64; 4],
16	+	}
17	+
18	+	/// Result retained with a request's acknowledgement.
19	+	export union Reply: Copy {
20	+	/// Successful scalar return value.
21	+	Value(u64),
22	+	/// Operation failure with no partial commit.
23	+	Error(abi::Error),
24	+	}
25	+
26	+	/// Work performed by the receiving hart before acknowledgement.
27	+	export union Action: Copy {
28	+	/// Refresh dispatch eligibility and its timer boundary.
29	+	Reschedule,
30	+	/// Synchronize local instruction fetch before dispatch.
31	+	InstructionSync,
32	+	/// Execute a budget operation after releasing local context ownership.
33	+	Budget(Call),
34	+	/// Refresh domain eligibility and notify the waiting call's context.
35	+	Ready(abi::Ref),
36	+	}
37	+
38	+	/// Request delivery state retained until the source releases it.
39	+	export union State: Copy {
40	+	/// The request is queued for its target.
41	+	Pending,
42	+	/// The target has taken the request.
43	+	Delivered,
44	+	/// The target completed the requested work.
45	+	Acknowledged,
46	+	}
47	+
48	+	/// One request owned by a source hart.
49	+	export record Request: Copy {
50	+	/// Hart that must release the completion record.
51	+	source: u32,
52	+	/// Hart that performs and acknowledges the action.
53	+	target: u32,
54	+	/// Required local action.
55	+	action: Action,
56	+	/// Current delivery state.
57	+	state: State,
58	+	/// Scalar result available after acknowledgement.
59	+	reply: Reply,
60	+	}
61	+
62	+	/// Bounded FIFO whose indices remain within its fixed array.
63	+	export record Queue: Copy {
64	+	/// Generation-bearing request references in delivery order.
65	+	entries: [abi::Ref; limits::REMOTE_PER_HART],
66	+	/// Next entry to consume.
67	+	read: u32,
68	+	/// Next entry to produce.
69	+	write: u32,
70	+	/// Number of occupied entries, including cancelled references.
71	+	count: u32,
72	+	}
73	+
74	+	/// Remote request metadata serialized by the shared kernel lock.
75	+	export record Store: Copy {
76	+	/// Online physical hart mask fixed at boot.
77	+	online: u32,
78	+	/// Request generations retained through acknowledgement.
79	+	slots: [slots::Slot; limits::REMOTE_REQUESTS],
80	+	/// Live request payloads.
81	+	requests: [Request; limits::REMOTE_REQUESTS],
82	+	/// FIFO owned by each receiving hart.
83	+	queues: [Queue; limits::HARTS],
84	+	}
85	+
86	+	/// Global cross-hart request storage.
87	+	export unsafe static STORE: Store = undefined;
88	+
89	+	/// Delivery identity and immutable routing information.
90	+	export record Delivery: Copy {
91	+	/// Live request generation to acknowledge.
92	+	id: abi::Ref,
93	+	/// Routing, action, and delivered state.
94	+	request: Request,
95	+	}
96	+
97	+	/// One completion reference for each hart selected by a bounded request batch.
98	+	export record Batch: Copy {
99	+	/// Hart that owns and releases all completion records.
100	+	source: u32,
101	+	/// Target hart mask; only these reference array elements are initialized.
102	+	targets: u32,
103	+	/// Request references indexed by target hart.
104	+	ids: [abi::Ref; limits::HARTS],
105	+	}
106	+
107	+	/// Reserved records whose queue capacity stays protected by the metadata lock.
108	+	export union Reservation: Once {
109	+	/// Batch to publish or cancel before releasing metadata serialization.
110	+	Held(Batch),
111	+	}
112	+
113	+	/// Reserve every target before any request or associated state becomes visible.
114	+	export unsafe fn reserve(store: &mut Store, source: u32, targets: u32) -> Reservation throws (abi::Error) {
115	+	try hart(store, source);
116	+	if (targets & ~store.online) <> 0 { throw abi::Error::InvalidArg; }
117	+	let mut count: u32 = 0;
118	+	for target in 0..limits::HARTS {
119	+	if (targets & (1 << target)) == 0 { continue; }
120	+	if store.queues[target].count == limits::REMOTE_PER_HART { throw abi::Error::Busy; }
121	+	set count += 1;
122	+	}
123	+	let mut free: u32 = 0;
124	+	for slot in &store.slots[..] { if slot.state == slots::State::Free { set free += 1; } }
125	+	if free < count { throw abi::Error::Busy; }
126	+	let mut batch = Batch { source, targets, ids: undefined };
127	+	for target in 0..limits::HARTS {
128	+	if (targets & (1 << target)) == 0 { continue; }
129	+	let pending = try! slots::reserve(&mut store.slots[..]);
130	+	match pending { case slots::Reservation::Held(id) => { set batch.ids[target] = id; }, }
131	+	}
132	+	return Reservation::Held(batch);
133	+	}
134	+
135	+	/// Return all unpublished records while the queue-capacity lock remains held.
136	+	export fn cancelReservation(store: &mut Store, reservation: Reservation) {
137	+	match reservation {
138	+	case Reservation::Held(batch) => {
139	+	for target in 0..limits::HARTS {
140	+	if (batch.targets & (1 << target)) == 0 { continue; }
141	+	try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(batch.ids[target]));
142	+	}
143	+	},
144	+	}
145	+	}
146	+
147	+	/// Publish the reserved batch before releasing metadata serialization.
148	+	export fn publish(store: &mut Store, reservation: Reservation, action: Action) -> Batch {
149	+	match reservation {
150	+	case Reservation::Held(batch) => {
151	+	for target in 0..limits::HARTS {
152	+	if (batch.targets & (1 << target)) == 0 { continue; }
153	+	let id = batch.ids[target];
154	+	set store.requests[id.index] = Request {
155	+	source: batch.source, target, action, state: State::Pending, reply: Reply::Value(0),
156	+	};
157	+	enqueue(store, target, id);
158	+	let live = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(id));
159	+	}
160	+	return batch;
161	+	},
162	+	}
163	+	}
164	+
165	+	/// Append a generation reference to a queue with reserved capacity.
166	+	fn enqueue(store: &mut Store, target: u32, id: abi::Ref) {
167	+	assert store.queues[target].count < limits::REMOTE_PER_HART;
168	+	let index = store.queues[target].write;
169	+	set store.queues[target].entries[index] = id;
170	+	set store.queues[target].write = (index + 1) % limits::REMOTE_PER_HART;
171	+	set store.queues[target].count += 1;
172	+	}
173	+
174	+	/// Initialize all request capacity before secondary harts start.
175	+	export fn initialize(store: &mut Store, online: u32) {
176	+	assert online <> 0 and (online >> limits::HARTS) == 0;
177	+	set store.online = online;
178	+	slots::initialize(&mut store.slots[..]);
179	+	for hart in 0..limits::HARTS {
180	+	set store.queues[hart].read = 0;
181	+	set store.queues[hart].write = 0;
182	+	set store.queues[hart].count = 0;
183	+	}
184	+	}
185	+
186	+	/// Check an online hart before indexing its mailbox.
187	+	fn hart(store: &Store, id: u32) throws (abi::Error) {
188	+	if id >= limits::HARTS or (store.online & (1 << id)) == 0 { throw abi::Error::InvalidArg; }
189	+	}
190	+
191	+	/// Enqueue a request atomically under metadata serialization.
192	+	export fn send(store: &mut Store, source: u32, target: u32, action: Action) -> abi::Ref throws (abi::Error) {
193	+	try hart(store, source); try hart(store, target);
194	+	if store.queues[target].count == limits::REMOTE_PER_HART { throw abi::Error::Busy; }
195	+	let pending = try slots::reserve(&mut store.slots[..]) catch { throw abi::Error::Busy; };
196	+	let id = slots::reference(&pending);
197	+	set store.requests[id.index] = Request { source, target, action, state: State::Pending, reply: Reply::Value(0) };
198	+	enqueue(store, target, id);
199	+	return try! slots::commit(&mut store.slots[..], pending);
200	+	}
201	+
202	+	/// Take the next live request, discarding stale or completed queue references.
203	+	export fn take(store: &mut Store, target: u32) -> ?Delivery throws (abi::Error) {
204	+	try hart(store, target);
205	+	for i in 0..limits::REMOTE_PER_HART {
206	+	if store.queues[target].count == 0 { return nil; }
207	+	let index = store.queues[target].read;
208	+	let id = store.queues[target].entries[index];
209	+	set store.queues[target].read = (index + 1) % limits::REMOTE_PER_HART;
210	+	set store.queues[target].count -= 1;
211	+	if not slots::matches(&store.slots[..], id, slots::State::Live) { continue; }
212	+	let request = store.requests[id.index];
213	+	if request.target <> target or request.state <> State::Pending { continue; }
214	+	set store.requests[id.index].state = State::Delivered;
215	+	return Delivery { id, request: store.requests[id.index] };
216	+	}
217	+	return nil;
218	+	}
219	+
220	+	/// Acknowledge one delivered generation exactly once under metadata serialization.
221	+	export fn acknowledge(store: &mut Store, target: u32, id: abi::Ref) -> bool {
222	+	if not slots::matches(&store.slots[..], id, slots::State::Live) { return false; }
223	+	let request = store.requests[id.index];
224	+	if request.target <> target or request.state <> State::Delivered { return false; }
225	+	set store.requests[id.index].state = State::Acknowledged;
226	+	return true;
227	+	}
228	+
229	+	/// Resolve a live request through its issuing hart.
230	+	fn owned(store: &Store, source: u32, id: abi::Ref) -> Request throws (abi::Error) {
231	+	if not slots::matches(&store.slots[..], id, slots::State::Live) { throw abi::Error::BadHandle; }
232	+	let request = store.requests[id.index];
233	+	if request.source <> source { throw abi::Error::BadHandle; }
234	+	return request;
235	+	}
236	+
237	+	/// Test completion while retaining the acknowledgement record.
238	+	export fn completed(store: &Store, source: u32, id: abi::Ref) -> bool throws (abi::Error) {
239	+	return (try owned(store, source, id)).state == State::Acknowledged;
240	+	}
241	+
242	+	/// Store a result before the receiving hart acknowledges its delivery.
243	+	export fn respond(store: &mut Store, target: u32, id: abi::Ref, reply: Reply) {
244	+	assert slots::matches(&store.slots[..], id, slots::State::Live);
245	+	assert store.requests[id.index].target == target and store.requests[id.index].state == State::Delivered;
246	+	set store.requests[id.index].reply = reply;
247	+	}
248	+
249	+	/// Read a completed operation's result without releasing its record.
250	+	export fn result(store: &Store, source: u32, id: abi::Ref) -> u64 throws (abi::Error) {
251	+	let request = try owned(store, source, id);
252	+	if request.state <> State::Acknowledged { throw abi::Error::Busy; }
253	+	match request.reply {
254	+	case Reply::Value(value) => return value,
255	+	case Reply::Error(error) => throw error,
256	+	}
257	+	}
258	+
259	+	/// Release an acknowledged request and advance its generation.
260	+	export fn release(store: &mut Store, source: u32, id: abi::Ref) throws (abi::Error) {
261	+	if (try owned(store, source, id)).state <> State::Acknowledged { throw abi::Error::Busy; }
262	+	try! slots::release(&mut store.slots[..], id);
263	+	}
264	+
265	+	/// Cancel queued work or discard a completed acknowledgement.
266	+	export fn cancel(store: &mut Store, source: u32, id: abi::Ref) throws (abi::Error) {
267	+	if (try owned(store, source, id)).state == State::Delivered { throw abi::Error::Busy; }
268	+	try! slots::release(&mut store.slots[..], id);
269	+	}

kernel/kernel/tests.rad +1 -0

export mod domains;
export mod budgets;
export mod dispatch;
export mod calls;
export mod timers;
export mod remote;

kernel/kernel/tests/budgets.rad +1 -1

    let child = slots::reference(&childSlot);
    capability::initialize(&mut MEMORY.table, child);
    let contextSlot = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
    let context = slots::reference(&contextSlot);
    set DOMAINS.contexts[context.index] = domains::Context {
        owner: child, state: domains::ContextState::Ready, hart: nil,
        owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil,
        kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },
        userStack: range::Range { start: 0, end: 0 },
        frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
    };
    set DOMAINS.records[child.index] = domains::Domain {

kernel/kernel/tests/calls.rad +36 -0

use kernel::slots;
use kernel::registry;
use kernel::limits;
use kernel::trap;
use kernel::budgets;
use kernel::remote;

/// Deferred budget operations recheck authority and expiration at execution time.
@test unsafe fn deferredBudget() throws (testing::TestError) {
    let owner = initialize();
    let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);
    let entry = try! capability::get(&MEMORY.table, handle);
    let call = remote::Call { owner, context: owner, operation: 70, arguments: [*handle, 50, 0, 0] };
    let wrongHart = calls::remoteBudget(call, 0, 20);
    assert wrongHart == remote::Reply::Error(abi::Error::BadHandle);
    remote::initialize(&mut remote::STORE, 3);
    let request = try! remote::send(&mut remote::STORE, 0, 1, remote::Action::Budget(call));
    let delivery = try! remote::take(&mut remote::STORE, 1) else panic "missing expired call";
    assert delivery.id == request;
    let case remote::Action::Budget(queued) = delivery.request.action else panic "queued budget call";
    let expired = calls::remoteBudget(queued, 1, 100);
    assert expired == remote::Reply::Error(abi::Error::Busy);
    remote::respond(&mut remote::STORE, 1, request, expired);
    assert remote::acknowledge(&mut remote::STORE, 1, request);
    assert try! remote::completed(&remote::STORE, 0, request);
    try! remote::release(&mut remote::STORE, 0, request);
    let unchanged = try! budgets::get(&budgets::STORE, entry.object);
    assert unchanged.start == 10 and unchanged.end == 100;
    let reply = calls::remoteBudget(call, 1, 20);
    let case remote::Reply::Value(value) = reply else panic "deferred split";
    let left = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
    let right = try! budgets::query(&budgets::STORE, &MEMORY.table, abi::Handle(value));
    assert left.start == 20 and left.end == 50 and right.start == 50 and right.end == 100;
    let duplicate = calls::remoteBudget(call, 1, 25);
    assert duplicate == remote::Reply::Error(abi::Error::InvalidArg);
    let retained = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
    assert retained.start == 20 and retained.end == 50;
    let dropped = try! calls::invoke(owner, 12, &[*handle, 0, 0, 0], 25);
    assert dropped == 0;
    assert calls::remoteBudget(call, 1, 25) == remote::Reply::Error(abi::Error::BadHandle);
}

/// Mapped physical storage for four pages plus alignment padding.
static RAM: [u64; 2560] = [0; 2560];
/// Kernel-only metadata for the calling domain.
unsafe static MEMORY: domains::Memory = undefined;

kernel/kernel/tests/dispatch.rad +15 -0

    let b = try! capability::get(&TABLE, last);
    set BUDGETS.windows[a.object.index].context = owner;
    set BUDGETS.windows[b.object.index].context = owner;
}

/// Wakeup reaches future windows and excludes expired or stopped contexts.
@test unsafe fn wakeTargets() throws (testing::TestError) {
    initialize();
    set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 1;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 25) == 1;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 40) == 0;
    set DOMAINS.contexts[0].state = domains::ContextState::Stopped;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
    set DOMAINS.contexts[0].state = domains::ContextState::Ready;
    set DOMAINS.contextSlots[0].generation += 1;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
}

/// Dispatch selects only the half-open interval and wakes at the next runnable start.
@test unsafe fn boundaries() throws (testing::TestError) {
    initialize();
    let before = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
    try testing::expect(before.context == nil and before.deadline == 10);

    set PLATFORM.devices[0] = platform::Device {
        kind: platform::Kind::Clint, memory: range::Range { start: 0x2000000, end: 0x2010000 },
    };
    let first = try! dispatch::timer(&PLATFORM, 0);
    let last = try! dispatch::timer(&PLATFORM, 7);
    assert first.software == 0x2000000 and last.software == first.software + 28;
    try testing::expect(first.clock == 0x200bff8 and first.compare == 0x2004000);
    try testing::expect(last.clock == first.clock and last.compare == first.compare + 56);
    let mut failures: u32 = 0;
    try dispatch::timer(&PLATFORM, 1) catch err {
        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;

kernel/kernel/tests/domains.rad +10 -0

    };
    try! domains::returned(&mut DOMAINS, authority.object, context);
    try testing::expect(DOMAINS.contexts[context.index].state == domains::ContextState::Stopped);
    try testing::expect(DOMAINS.records[authority.object.index].state == domains::Lifecycle::Pending);
    set DOMAINS.contexts[context.index].hart = nil;
    set DOMAINS.contexts[context.index].pending = abi::Ref { index: 0, generation: 1 };
    let mut retained = false;
    try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context) catch error {
        assert error == abi::Error::Busy; set retained = true;
    };
    assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        assert not PAGES.backings.pool.free[i];
    }
    set DOMAINS.contexts[context.index].pending = nil;
    try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context);
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        try testing::expect(PAGES.backings.pool.free[i]);
    }
    try domains::context(&DOMAINS, authority.object, context) catch error {

kernel/kernel/tests/remote.rad added +202 -0

1	+	//! Bounded request queues, acknowledgement identity, and capacity recovery.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::remote;
5	+	use kernel::limits;
6	+	use kernel::slots;
7	+
8	+	/// Serialized request state shared by two test harts.
9	+	unsafe static STORE: remote::Store = undefined;
10	+
11	+	/// Requests retain source ownership until an explicit acknowledgement is released.
12	+	@test unsafe fn acknowledgement() throws (testing::TestError) {
13	+	remote::initialize(&mut STORE, 3);
14	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
15	+	assert not try! remote::completed(&STORE, 0, id);
16	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing request";
17	+	assert delivery.id == id and delivery.request.source == 0;
18	+	assert not remote::acknowledge(&mut STORE, 0, id);
19	+	assert remote::acknowledge(&mut STORE, 1, id);
20	+	assert not remote::acknowledge(&mut STORE, 1, id);
21	+	assert try! remote::completed(&STORE, 0, id);
22	+	let mut denied = false;
23	+	try remote::release(&mut STORE, 1, id) catch error {
24	+	assert error == abi::Error::BadHandle; set denied = true;
25	+	};
26	+	assert denied;
27	+	try! remote::release(&mut STORE, 0, id);
28	+	assert not remote::acknowledge(&mut STORE, 1, id);
29	+	}
30	+
31	+	/// Cancellation leaves a harmless stale queue entry until the receiver drains it.
32	+	@test unsafe fn saturation() throws (testing::TestError) {
33	+	remote::initialize(&mut STORE, 3);
34	+	let mut ids: [abi::Ref; 16] = undefined;
35	+	for i in 0..16 { set ids[i] = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule); }
36	+	let mut full = false;
37	+	try remote::send(&mut STORE, 0, 1, remote::Action::Reschedule) catch error {
38	+	assert error == abi::Error::Busy; set full = true;
39	+	};
40	+	assert full;
41	+	try! remote::cancel(&mut STORE, 0, ids[0]);
42	+	for i in 1..16 {
43	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing queued request";
44	+	assert delivery.id == ids[i];
45	+	assert remote::acknowledge(&mut STORE, 1, delivery.id);
46	+	try! remote::release(&mut STORE, 0, delivery.id);
47	+	}
48	+	assert try! remote::take(&mut STORE, 1) == nil;
49	+	for i in 0..40 {
50	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
51	+	let delivery = try! remote::take(&mut STORE, 1) else panic "request reuse";
52	+	assert delivery.id == id and id <> ids[0];
53	+	assert not remote::acknowledge(&mut STORE, 1, ids[0]);
54	+	assert remote::acknowledge(&mut STORE, 1, id);
55	+	try! remote::release(&mut STORE, 0, id);
56	+	}
57	+	}
58	+
59	+	/// Completed records retain capacity until their source releases them.
60	+	@test unsafe fn retainedCapacity() throws (testing::TestError) {
61	+	remote::initialize(&mut STORE, 3);
62	+	let mut ids: [abi::Ref; limits::REMOTE_REQUESTS] = undefined;
63	+	for i in 0..limits::REMOTE_REQUESTS {
64	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
65	+	set ids[i] = id;
66	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing request";
67	+	assert delivery.id == id;
68	+	assert remote::acknowledge(&mut STORE, 1, id);
69	+	}
70	+	let mut full = false;
71	+	try remote::send(&mut STORE, 0, 1, remote::Action::Reschedule) catch error {
72	+	assert error == abi::Error::Busy; set full = true;
73	+	};
74	+	assert full;
75	+	for id in &ids[..] { try! remote::release(&mut STORE, 0, id); }
76	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
77	+	try! remote::cancel(&mut STORE, 0, id);
78	+	}
79	+
80	+	/// A cancelled queue reference cannot deliver a reused slot generation.
81	+	@test unsafe fn cancelledGeneration() throws (testing::TestError) {
82	+	remote::initialize(&mut STORE, 3);
83	+	let stale = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
84	+	try! remote::cancel(&mut STORE, 0, stale);
85	+	let live = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
86	+	assert live.index == stale.index and live <> stale;
87	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing reused request";
88	+	assert delivery.id == live and delivery.request.action == remote::Action::InstructionSync;
89	+	assert try! remote::take(&mut STORE, 1) == nil;
90	+	assert not remote::acknowledge(&mut STORE, 1, stale);
91	+	let mut cancelDenied = false;
92	+	try remote::cancel(&mut STORE, 0, live) catch error {
93	+	assert error == abi::Error::Busy; set cancelDenied = true;
94	+	};
95	+	assert cancelDenied;
96	+	let mut releaseDenied = false;
97	+	try remote::release(&mut STORE, 0, live) catch error {
98	+	assert error == abi::Error::Busy; set releaseDenied = true;
99	+	};
100	+	assert releaseDenied;
101	+	assert remote::acknowledge(&mut STORE, 1, live);
102	+	try! remote::release(&mut STORE, 0, live);
103	+	}
104	+
105	+	/// Invalid routes and premature release preserve all queued work.
106	+	@test unsafe fn invalidRoute() throws (testing::TestError) {
107	+	remote::initialize(&mut STORE, 1);
108	+	for target in &[1, limits::HARTS] {
109	+	let mut denied = false;
110	+	try remote::send(&mut STORE, 0, target, remote::Action::Reschedule) catch error {
111	+	assert error == abi::Error::InvalidArg; set denied = true;
112	+	};
113	+	assert denied;
114	+	}
115	+	let mut sourceDenied = false;
116	+	try remote::send(&mut STORE, 1, 0, remote::Action::Reschedule) catch error {
117	+	assert error == abi::Error::InvalidArg; set sourceDenied = true;
118	+	};
119	+	assert sourceDenied;
120	+	let id = try! remote::send(&mut STORE, 0, 0, remote::Action::Reschedule);
121	+	let mut releaseDenied = false;
122	+	try remote::release(&mut STORE, 0, id) catch error {
123	+	assert error == abi::Error::Busy; set releaseDenied = true;
124	+	};
125	+	assert releaseDenied;
126	+	assert not remote::acknowledge(&mut STORE, 0, id);
127	+	let delivery = try! remote::take(&mut STORE, 0) else panic "missing local request";
128	+	assert delivery.id == id;
129	+	assert remote::acknowledge(&mut STORE, 0, id);
130	+	try! remote::release(&mut STORE, 0, id);
131	+	assert try! remote::take(&mut STORE, 0) == nil;
132	+	}
133	+
134	+	/// Replies become observable only after the target acknowledges completion.
135	+	@test unsafe fn replies() throws (testing::TestError) {
136	+	remote::initialize(&mut STORE, 3);
137	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
138	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing reply request";
139	+	assert delivery.id == id;
140	+	remote::respond(&mut STORE, 1, id, remote::Reply::Value(42));
141	+	let mut pending = false;
142	+	try remote::result(&STORE, 0, id) catch error {
143	+	assert error == abi::Error::Busy; set pending = true;
144	+	};
145	+	assert pending;
146	+	assert remote::acknowledge(&mut STORE, 1, id);
147	+	assert try! remote::result(&STORE, 0, id) == 42;
148	+	try! remote::release(&mut STORE, 0, id);
149	+	let failed = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
150	+	let taken = try! remote::take(&mut STORE, 1) else panic "missing failure request";
151	+	assert taken.id == failed;
152	+	remote::respond(&mut STORE, 1, failed, remote::Reply::Error(abi::Error::BadHandle));
153	+	assert remote::acknowledge(&mut STORE, 1, failed);
154	+	let mut rejected = false;
155	+	try remote::result(&STORE, 0, failed) catch error {
156	+	assert error == abi::Error::BadHandle; set rejected = true;
157	+	};
158	+	assert rejected;
159	+	try! remote::release(&mut STORE, 0, failed);
160	+	}
161	+
162	+	/// Batch failure and cancellation leave every mailbox unchanged.
163	+	@test unsafe fn batchReservation() throws (testing::TestError) {
164	+	remote::initialize(&mut STORE, 7);
165	+	for i in 0..limits::REMOTE_PER_HART {
166	+	let id = try! remote::send(&mut STORE, 0, 2, remote::Action::Reschedule);
167	+	}
168	+	assert batchFull() and STORE.queues[1].count == 0;
169	+	let pending = try! remote::reserve(&mut STORE, 0, 2);
170	+	assert STORE.queues[1].count == 0;
171	+	remote::cancelReservation(&mut STORE, pending);
172	+	assert STORE.queues[1].count == 0;
173	+	let mut live: u32 = 0;
174	+	for slot in &STORE.slots[..] {
175	+	assert slot.state <> slots::State::Reserved;
176	+	if slot.state == slots::State::Live { set live += 1; }
177	+	}
178	+	assert live == limits::REMOTE_PER_HART;
179	+	for i in 0..limits::REMOTE_PER_HART {
180	+	let delivery = try! remote::take(&mut STORE, 2) else panic "full mailbox lost request";
181	+	assert remote::acknowledge(&mut STORE, 2, delivery.id);
182	+	try! remote::release(&mut STORE, 0, delivery.id);
183	+	}
184	+	let reserved = try! remote::reserve(&mut STORE, 0, 6);
185	+	let batch = remote::publish(&mut STORE, reserved, remote::Action::Reschedule);
186	+	assert batch.source == 0 and batch.targets == 6;
187	+	for target in 1..3 {
188	+	let delivery = try! remote::take(&mut STORE, target) else panic "missing batch request";
189	+	assert delivery.id == batch.ids[target];
190	+	assert remote::acknowledge(&mut STORE, target, delivery.id);
191	+	try! remote::release(&mut STORE, 0, delivery.id);
192	+	}
193	+	}
194	+
195	+	/// Cancel an unexpected reservation so every linear result is consumed.
196	+	unsafe fn batchFull() -> bool {
197	+	let pending = try remote::reserve(&mut STORE, 0, 6) catch error {
198	+	assert error == abi::Error::Busy; return true;
199	+	};
200	+	remote::cancelReservation(&mut STORE, pending);
201	+	return false;
202	+	}

kernel/kernel/tests/timers.rad +32 -9

use kernel::slots;
use kernel::domains;
use kernel::events;
use kernel::timers;
use kernel::limits;
use kernel::budgets;

/// Timer records under test.
unsafe static TIMERS: timers::Store = undefined;
/// Domain metadata and trusted notification state.
unsafe static DOMAINS: domains::Store = undefined;
/// CPU authority used to identify remote notification targets.
unsafe static BUDGETS: budgets::Store = undefined;
/// Receiver ring and private metadata.
unsafe static MEMORY: domains::Memory = undefined;

/// Open an active domain queue without assigning CPU authority.
unsafe fn initialize() -> abi::Ref {
    timers::initialize(&mut TIMERS);
    budgets::initialize(&mut BUDGETS);
    domains::initialize(&mut DOMAINS);
    let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
    let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
    set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
    set DOMAINS.records[owner.index].memory = &mut MEMORY;
    try! events::open(&mut DOMAINS.events, owner, &mut MEMORY.ring);
    return owner;
}

/// A timeout reports the hart of a receiver whose context moved before delivery.
@test unsafe fn remoteReceiver() throws (testing::TestError) {
    let owner = initialize();
    set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };
    let context = abi::Ref { index: 0, generation: 1 };
    set DOMAINS.contexts[0].owner = owner;
    set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
    set BUDGETS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
    set BUDGETS.windows[0] = budgets::Window {
        hart: 1, start: 10, end: 100, owner, handle: abi::Handle(0), context,
    };
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 15, 17);
    let result = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15);
    assert result.harts == 2 and result.deadline == 0xffffffffffffffff;
    assert DOMAINS.contexts[0].state == domains::ContextState::Ready;
    let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 16);
    assert again.harts == 0;
}

/// A timeout is delivered once, at its deadline, on its registered hart.
@test unsafe fn boundaries() throws (testing::TestError) {
    let owner = initialize();
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 1, 10, 5, 17);
    assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 15) == 0xffffffffffffffff);
    assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 14) == 15);
    assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15)).deadline == 0xffffffffffffffff);
    assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 14)).deadline == 15);
    assert (try! events::pop(&mut MEMORY.ring) == nil);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 15);
    assert (next == 0xffffffffffffffff);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 15);
    assert (next.deadline == 0xffffffffffffffff);
    let event = try! events::pop(&mut MEMORY.ring) else panic "timeout";
    assert (event.kind == events::TIMEOUT and event.code == 17 and event.value == 0);
    let again = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 16);
    let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 16);
    assert (try! events::pop(&mut MEMORY.ring) == nil);
}

/// Full queues retain due timeouts without requesting another immediate interrupt.
@test unsafe fn saturation() throws (testing::TestError) {

    let mut full = false;
    try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 1, 99) catch error {
        assert (error == abi::Error::Busy); set full = true;
    };
    assert (full);
    assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1) == 0xffffffffffffffff);
    assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1)).deadline == 0xffffffffffffffff);
    for i in 0..events::ORDINARY { let event = try! events::pop(&mut MEMORY.ring) else panic "wake"; }
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 2);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 2);
    for i in 0..limits::TIMEOUTS_PER_HART {
        let event = try! events::pop(&mut MEMORY.ring) else panic "retained timeout";
        assert (event.kind == events::TIMEOUT and event.code == i);
    }
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 2, 1, 100);

    try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0xffffffffffffffff, 1, 1) catch error {
        assert (error == abi::Error::InvalidArg); set rejected += 1;
    };
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 5, 1);
    set DOMAINS.records[owner.index].state = domains::Lifecycle::Dead;
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1);
    assert (next == 0xffffffffffffffff and rejected == 2);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1);
    assert (next.deadline == 0xffffffffffffffff and rejected == 2);
    assert (TIMERS.entries[0][0].owner == nil);
}

kernel/kernel/timers.rad +15 -5

use super::abi;
use super::limits;
use super::slots;
use super::domains;
use super::events;
use super::budgets;

/// One armed timeout or retained due notification.
export record Entry: Copy {
    /// Receiver generation, or nil for free capacity.
    owner: ?abi::Ref,

        return;
    }
    throw abi::Error::Busy;
}

/// Deliver due entries and return the next future timeout boundary.
/// Next timer boundary and dispatch notifications from delivered events.
export record Result: Copy {
    /// Earliest future timeout, or the maximum tick when none is armed.
    deadline: u64,
    /// Harts with retained receiver authority that must refresh dispatch.
    harts: u32,
}

/// Deliver due entries and report the next boundary and affected dispatch harts.
/// Full queues retain due entries for the next kernel entry without a timer spin.
export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, hart: u32, now: u64) -> u64 throws (abi::Error) {
export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, windows: &budgets::Store, hart: u32, now: u64) -> Result throws (abi::Error) {
    if hart >= limits::HARTS { throw abi::Error::InvalidArg; }
    let mut next: u64 = 0xffffffffffffffff;
    let mut result = Result { deadline: 0xffffffffffffffff, harts: 0 };
    for i in 0..limits::TIMEOUTS_PER_HART {
        let entry = store.entries[hart][i];
        let owner = entry.owner else { continue; };
        if not slots::matches(&receivers.slots[..], owner, slots::State::Live)
            or receivers.records[owner.index].state == domains::Lifecycle::Dead {
            set store.entries[hart][i].owner = nil;
            continue;
        }
        if entry.deadline > now {
            if entry.deadline < next { set next = entry.deadline; }
            if entry.deadline < result.deadline { set result.deadline = entry.deadline; }
            continue;
        }
        if try domains::notify(receivers, owner, events::Event { kind: events::TIMEOUT, reserved: 0, code: entry.token, value: 0 }) {
            set store.entries[hart][i].owner = nil;
            set result.harts |= budgets::wakeTargets(windows, receivers, owner, now);
        }
    }
    return next;
    return result;
}

test/smp/kernel/dispatchcheck.rad +159 -6

use kernel::domains;
use kernel::budgets;
use kernel::dispatch;
use kernel::boot;
use kernel::sync;
use kernel::remote;
use kernel::calls;
use kernel::events;
use kernel::timers;
use kernel::dispatchinput;

/// Bootstrap resource authority.
unsafe static TABLE: capability::Table = undefined;
/// Release/acquire barrier after fixture publication.

unsafe static USER: abi::Ref = undefined;
/// Per-hart user execution contexts.
unsafe static CONTEXTS: [abi::Ref; 8] = undefined;
/// Mapped progress counters in the user's private state.
static COUNTERS: u64 = 0;
/// Per-context migration observations in the user's private state.
static MIGRATIONS: u64 = 0;
/// Shared handshake for the independently activated context on hart one.
static WAKE_STATE: u64 = 0;
/// Bootstrap management capability for the pending idle-wakeup domain.
unsafe static IDLE_HANDLE: abi::Handle = undefined;
/// Context whose Wait call must release the remote hart.
unsafe static IDLE_CONTEXT: abi::Ref = undefined;
/// Validated initial stack pointer for the idle-wakeup domain.
static IDLE_STACK: u64 = 0;
/// Generation of the idle-wakeup domain.
unsafe static IDLE_DOMAIN: abi::Ref = undefined;
/// Completion checker's live domain generation.
unsafe static CHECKER: abi::Ref = undefined;
/// Checker's authority to activate and notify the idle-wakeup domain.
unsafe static RECEIVER: abi::Handle = undefined;
/// Unbound authority reserved for the idle-wakeup check on hart one.
unsafe static IDLE_WINDOW: abi::Handle = undefined;
/// Validated physical memory mapping.
fn memory(address: u64) -> *mut u8;
/// Current kernel package-state table.
fn kernelGp() -> u64;
/// M-mode entry of the completion checker.

    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let cap = try! capability::get(&TABLE, handle);
    set USER = cap.object;
    let mut user = try! domains::get(&domains::STORE, USER);
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE),
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),
    });
    let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);
    let case shared::Target::Function(entry) = target else panic "worker entry";
    let counterTarget = try! registry::exported(&registry::STORE, user.image, &"spin::count"[..]);
    let case shared::Target::Data(data) = counterTarget else panic "counter state";
    set COUNTERS = user.graph.table[data.slot] + data.offset as u64;
    set COUNTERS = dataAddress(&user, &"spin::count"[..]);
    set MIGRATIONS = dataAddress(&user, &"spin::migrations"[..]);
    let notification = @sliceOf(memory(dataAddress(&user, &"spin::notification"[..])), 8);
    for i in 0..8 { set notification[i] = ((*self) >> (i as u64 * 8)) as u8; }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
        let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ | abi::WRITE) as u64);
        let permission = try! capability::get(&TABLE, stack);

        }
    }
    let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let checkerCap = try! capability::get(&TABLE, checkerHandle);
    let checker = try! domains::get(&domains::STORE, checkerCap.object);
    set CHECKER = checkerCap.object;
    set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;
    set domains::STORE.contexts[checker.initial.index].frame.pc = completion();
    set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;
    set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;
    set domains::STORE.contexts[checker.initial.index].frame.registers[3] = kernelGp();
    idleSetup(authority, image);
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let start = clock + 1000000;
    let mut harts: u64 = 0;
    for hart in 0..8 { if (boot::PLATFORM.harts & (1 << hart)) <> 0 { set harts += 1; } }
    let end = start + 10000000 * harts * harts;
    let migrationEnd = end + 10000000 * harts * harts;
    let windowBase = dataAddress(&user, &"spin::windows"[..]);
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        let budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);
        let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);
        let checking = try! budgets::split(&mut budgets::STORE, &mut TABLE, final, migrationEnd, clock);
        let future = try! budgets::split(&mut budgets::STORE, &mut TABLE, checking, migrationEnd + 10000000000, clock);
        let transferred = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut user.memory.table,
            future, budgets::DEFAULT_RIGHTS as u64, clock);
        let caller = (hart as u64 + harts - 1) % harts;
        let output = @sliceOf(memory(windowBase + caller * 8), 8);
        for i in 0..8 { set output[i] = ((*transferred) >> (i as u64 * 8)) as u8; }
        let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
            budgets::Binding { budget, domain: handle, context: CONTEXTS[hart] }, clock);
        let rotated = ((hart as u64 + 1) % harts) as u32;
        let moved = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
            budgets::Binding { budget: final, domain: handle, context: CONTEXTS[rotated] }, clock);
        if hart == 0 {
            let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
                budgets::Binding { budget: final, domain: checkerHandle, context: checker.initial }, clock);
                budgets::Binding { budget: checking, domain: checkerHandle, context: checker.initial }, clock);
        }
        if hart == 1 { set IDLE_WINDOW = checking; }
    }
}

/// Resolve one exported private-data address in the fixture's user instance.
unsafe fn dataAddress(user: &domains::Domain, name: &[u8]) -> u64 {
    let target = try! registry::exported(&registry::STORE, user.image, name);
    let case shared::Target::Data(data) = target else panic "fixture data export";
    return user.graph.table[data.slot] + data.offset as u64;
}

/// Prepare a pending domain and its stack for activation on an idle hart.
unsafe fn idleSetup(authority: abi::Handle, image: abi::Handle) {
    set IDLE_HANDLE = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
        &registry::STORE, &mut TABLE, authority, image);
    let permit = try! capability::get(&TABLE, IDLE_HANDLE);
    set IDLE_DOMAIN = permit.object;
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    set IDLE_CONTEXT = domain.initial;
    set WAKE_STATE = dataAddress(&domain, &"spin::wakeState"[..]);
    sync::storeRelease(memory(dataAddress(&domain, &"spin::idle"[..])) as *mut u64, 1);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut domain.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let permission = try! capability::get(&TABLE, stack);
    let page = try! pages::get(&pages::STORE, permission.object);
    set IDLE_STACK = page.base + 4096;
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE),
    });
}

/// Check shared progress and context isolation after the user windows finish.
export unsafe fn verify() {
    let guard = sync::enter();
    let mut counts: [u64; 8] = [0; 8];
    let mut notifications: u32 = 0;
    for hart in 0..8 { if (boot::PLATFORM.harts & (1 << hart)) <> 0 { set notifications += 1; } }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);
        set counts[hart] = value;
        let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);
        assert context.owner == USER and (context.frame.status & 0x1800) == 0;
        assert context.hart == nil or context.hart == hart;
        let destination = (hart + notifications - 1) % notifications;
        assert context.hart == nil or context.hart == destination;
        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;
        assert context.pending == nil;
    }
    assert domains::STORE.events.queues[USER.index].tail == notifications;
    sync::leave(guard);
    for byte in "smp completed transactions:" { put(byte); }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        put(32); number(counts[hart]);
    }
    put(10);
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 { assert counts[hart] == 16; }
    }
    for target in 0..8 {
        if (boot::PLATFORM.harts & (1 << target)) == 0 { continue; }
        for action in &[remote::Action::Reschedule, remote::Action::InstructionSync] {
            let posting = sync::enter();
            let id = try! dispatch::request(target, action);
            sync::leave(posting);
            let mut acknowledged = false;
            for attempt in 0..100000 {
                let guard = sync::enter();
                if try! remote::completed(&remote::STORE, 0, id) {
                    try! remote::release(&mut remote::STORE, 0, id);
                    set acknowledged = true;
                }
                sync::leave(guard);
                if acknowledged { break; }
            }
            assert acknowledged;
        }
    }
    for byte in "smp request acknowledgements passed\n" { put(byte); }
    if (boot::PLATFORM.harts & 2) <> 0 { idleWakeup(); }
    let maximum = sync::maximum();
    assert maximum > 0 and maximum < 1000000;
    for byte in "smp metadata instructions: 0x" { put(byte); }
    number(maximum);
    put(10);
    finish();
}

/// Resume an actual U-mode Wait call through an IPI to an idle remote hart.
unsafe fn idleWakeup() {
    let guard = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let handle = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
        budgets::Binding { budget: IDLE_WINDOW, domain: IDLE_HANDLE, context: IDLE_CONTEXT }, clock);
    let request = try! dispatch::request(1, remote::Action::Reschedule);
    sync::leave(guard);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let activated = try! calls::synchronized(CHECKER, 21, &[*RECEIVER, IDLE_STACK, 0, 0], clock);
    assert activated == 0 and waiting();
    let now = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let result = try! calls::synchronized(CHECKER, 50, &[*RECEIVER, 0x55, 0, 0], now);
    assert result == 0;
    let mut resumed = false;
    for attempt in 0..100000 {
        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 3 { set resumed = true; break; }
    }
    assert resumed;
    for byte in "smp idle-hart activation and wakeup passed\n" { put(byte); }
    remoteTimeout();
}

/// Deliver a timeout from hart zero to a waiting user context on idle hart one.
unsafe fn remoteTimeout() {
    let guard = sync::enter();
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let event = try! events::pop(&mut domain.memory.ring) else panic "missing wakeup";
    assert event.kind == events::WAKEUP and event.code == 0x55;
    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 4);
    sync::leave(guard);
    assert waiting();
    let posting = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    try! timers::request(&mut timers::STORE, &domains::STORE, IDLE_DOMAIN, 0, clock, 100000, 0x66);
    let request = try! dispatch::request(0, remote::Action::Reschedule);
    sync::leave(posting);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let mut resumed = false;
    for attempt in 0..100000 {
        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 6 { set resumed = true; break; }
    }
    assert resumed;
    for byte in "smp remote timeout passed\n" { put(byte); }
}

/// Confirm the waiting context has released hart ownership before its wakeup.
unsafe fn waiting() -> bool {
    for attempt in 0..100000 {
        let guard = sync::enter();
        let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);
        let stopped = context.state == domains::ContextState::Waiting and context.hart == nil;
        sync::leave(guard);
        if stopped { return true; }
    }
    return false;
}

/// Print one full-width hexadecimal diagnostic value.
fn number(value: u64) {
    let digits = "0123456789abcdef";
    for i in 0..16 { put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]); }
}

test/smp/spin.rad +46 -2

export mod sys;

/// Completed allocation transactions, indexed by the executing hart.
export static count: [u64; 8] = [0; 8];

/// Future budget handles assigned to callers on the preceding online hart.
export static windows: [u64; 8] = [0; 8];

/// Capability used to notify the shared domain from each online hart.
export static notification: u64 = 0;

/// Observed destination hart plus one after each context changes budget.
export static migrations: [u64; 8] = [0; 8];

/// Wait handshake: 2/3 bracket wakeup; 4 requests a second wait, bracketed by 5/6.
export static wakeState: u64 = 0;

/// Nonzero selects the independently activated idle-wakeup context.
export static idle: u64 = 0;

/// Check hart identity and shared-table allocation through repeated direct calls.
fn run(hart: u64) {
    let original = sys::currentContext();
    assert original.hart == hart;
    let budget = abi::Handle(windows[hart as u32]);
    let before = try! sys::queryBudget(budget);
    let right = try! sys::budgetSplit(budget, before.start + 1000);
    let leftInfo = try! sys::queryBudget(budget);
    let rightInfo = try! sys::queryBudget(right);
    assert leftInfo.start == before.start and leftInfo.end == rightInfo.start;
    assert rightInfo.end == before.end and rightInfo.hart == before.hart;
    let merged = try! sys::budgetMerge(budget, right);
    assert merged == budget;
    let after = try! sys::queryBudget(merged);
    assert after.start == before.start and after.end == before.end;
    try! sys::capabilityDrop(merged);
    try! sys::wakeup(abi::Handle(notification), hart as u32);
    for i in 0..16 {
        let page = try! sys::pageAllocate(abi::Handle(0), 1);
        let info = sys::queryPage(page);
        assert info.count == 1 and info.base <> 0;
        try! sys::capabilityDrop(page);
        let current = sys::currentContext();
        assert current.context == original.context and current.hart == hart;
        set count[hart as u32] += 1;
    }
    while true {}
    while true {
        let current = sys::currentContext();
        assert current.context == original.context;
        if current.budget <> original.budget { set migrations[hart as u32] = current.hart + 1; }
    }
}

/// Start an additional context with its physical hart index in validated memory.
export fn worker(args: *u64, size: u64) {
    assert size == 8;
    run(*args);
}

/// The initial context owns hart zero's window.
@default fn main(env: *opaque) { run(0); }
@default fn main(env: *opaque) {
    if idle <> 0 {
        set wakeState = 2;
        sys::wait();
        set wakeState = 3;
        while wakeState <> 4 {}
        set wakeState = 5;
        sys::wait();
        set wakeState = 6;
        while true {}
    }
    run(0);
}

274	300		sync::leave(publicationGuard);
275	301		return *result;
276	302		}
277	303
278	304		/// Execute a call with short metadata transactions and private bulk initialization.
279		-	/// The running context keeps its domain storage live until dispatch releases it.
	305	+	/// Context ownership and pending calls retain storage across dispatch boundaries.
280	306		export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
281	307		assert arguments.len == 4;
	308	+	if operation == 21 or operation == 50 { return try runtimeReady(owner, operation, arguments, now); }
282	309		let guard = sync::enter();
283	310		let mut domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
284	311		if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
	312	+	let target = try budgetHart(&domain.memory.table, operation, arguments) catch error { sync::leave(guard); throw error; };
	313	+	if let destination = target {
	314	+	let source = dispatch::hart();
	315	+	if destination <> source {
	316	+	let call = remote::Call { owner, context: abi::reference(dispatch::current().context), operation,
	317	+	arguments: [arguments[0], arguments[1], arguments[2], arguments[3]] };
	318	+	let request = try dispatch::request(destination, remote::Action::Budget(call)) catch error {
	319	+	sync::leave(guard); throw error;
	320	+	};
	321	+	assert domains::STORE.contexts[call.context.index].pending == nil;
	322	+	set domains::STORE.contexts[call.context.index].pending = request;
	323	+	sync::leave(guard);
	324	+	return try dispatch::awaitRequest(source, request);
	325	+	}
	326	+	}
285	327		let handle = abi::Handle(arguments[0]);
286	328		match operation {
287	329		case 20 => {
288	330		let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
289	331		&mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error { sync::leave(guard); throw error; };

344	386		sync::leave(guard); return result;
345	387		},
346	388		}
347	389		}
348	390
	391	+	/// Reserve remote dispatch capacity before activation or event publication.
	392	+	unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
	393	+	if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) { throw abi::Error::InvalidArg; }
	394	+	let guard = sync::enter();
	395	+	let domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
	396	+	if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
	397	+	let handle = abi::Handle(arguments[0]);
	398	+	let rights = abi::EXECUTE if operation == 21 else abi::WAKE;
	399	+	let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
	400	+	sync::leave(guard); throw error;
	401	+	};
	402	+	let source = dispatch::hart();
	403	+	let context = abi::reference(dispatch::current().context);
	404	+	let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
	405	+	let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error { sync::leave(guard); throw error; };
	406	+	let result = try invoke(owner, operation, arguments, now) catch error {
	407	+	remote::cancelReservation(&mut remote::STORE, pending);
	408	+	sync::leave(guard); throw error;
	409	+	};
	410	+	let batch = remote::publish(&mut remote::STORE, pending, remote::Action::Ready(context));
	411	+	dispatch::startReady(context, &batch);
	412	+	sync::leave(guard);
	413	+	dispatch::finishReady(context, &batch);
	414	+	return 0;
	415	+	}
	416	+
349	417		/// Complete a user ecall through the supplied execution boundary.
350	418		export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
351	419		execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
352	420		{
353	421		assert trap::fromUser(frame) and frame.cause == 8;

306	308		set frame.registers[3] = domain.graph.table.ptr as u64;
307	309		set frame.registers[10] = start.args;
308	310		set frame.registers[11] = start.size;
309	311		let object = slots::reference(&slot);
310	312		set store.contexts[object.index] = Context {
311		-	owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
	313	+	owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames, kernelStack, userStack, frame,
312	314		};
313	315		match slot {
314	316		case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
315	317		object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
316	318		}),

530	532		// MPIE enables interrupts after the first return into U-mode.
531	533		let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
532	534		set frame.registers[1] = returnAddress();
533	535		set frame.registers[3] = graph.table.ptr as u64;
534	536		set store.contexts[initial.index] = Context {
535		-	owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
	537	+	owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
536	538		userStack: range::Range { start: 0, end: 0 }, frame,
537	539		};
538	540		set store.records[object.index] = Domain {
539	541		state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
540	542		allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,

21	21		export mod instances;
22	22		export mod domains;
23	23		export mod budgets;
24	24		export mod dispatch;
25	25		export mod timers;
	26	+	export mod remote;
26	27		export mod calls;
27	28		export mod sys;
28	29		export mod boot;
29	30		@test export mod tests;

10	10		use super::registry;
11	11		use super::domains;
12	12		use super::budgets;
13	13		use super::dispatch;
14	14		use super::timers;
	15	+	use super::remote;
15	16		use std::arch::rv64::shared::catalog;
16	17
17	18		/// Platform data published by hart zero before secondary initialization.
18	19		export unsafe static PLATFORM: platform::Platform = undefined;
19	20		/// Release/acquire publication flag for PLATFORM.

23	24		/// Trap anchors owned by their physical harts.
24	25		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
25	26
26	27		/// Handle timer boundaries and user calls, and stop on unsupported traps.
27	28		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
28		-	if trap::classify(frame.cause) == trap::Cause::Timer {
	29	+	let cause = trap::classify(frame.cause);
	30	+	if cause == trap::Cause::Timer or cause == trap::Cause::Software {
29	31		dispatch::interrupt(frame, hart);
30	32		return;
31	33		}
32		-	if trap::classify(frame.cause) == trap::Cause::UserCall and trap::fromUser(frame) {
	34	+	if cause == trap::Cause::UserCall and trap::fromUser(frame) {
33	35		dispatch::call(frame, hart);
34	36		return;
35	37		}
36	38		print("kernel: unexpected trap\n");
37	39		trap::halt();

77	79		try! platform::protect(&mut PLATFORM, treeRange);
78	80		try! pages::initialize(&mut pages::STORE, &PLATFORM);
79	81		domains::initialize(&mut domains::STORE);
80	82		budgets::initialize(&mut budgets::STORE);
81	83		timers::initialize(&mut timers::STORE);
	84	+	remote::initialize(&mut remote::STORE, PLATFORM.harts);
82	85		assert entryCount == 2;
83	86		registry::initialize(&mut registry::STORE);
84	87		let bootCatalog = @sliceOf(entries, entryCount);
85	88		try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
86	89		let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

249	249		/// Test half-open execution authority on the selected hart at the current tick.
250	250		export fn active(store: &Store, object: abi::Ref, hart: u32, now: u64) -> bool throws (abi::Error) {
251	251		let window = try get(store, object);
252	252		return window.context <> nil and window.hart == hart and window.start <= now and now < window.end;
253	253		}
	254	+
	255	+	/// Find all harts with retained authority for a receiver's live contexts.
	256	+	export fn wakeTargets(windows: &Store, contexts: &domains::Store, receiver: abi::Ref, now: u64) -> u32 {
	257	+	let mut targets: u32 = 0;
	258	+	for i in 0..limits::BUDGETS {
	259	+	if windows.slots[i].state <> slots::State::Live { continue; }
	260	+	let window = windows.windows[i];
	261	+	if window.owner <> receiver or window.end <= now { continue; }
	262	+	let context = window.context else { continue; };
	263	+	if not slots::matches(&contexts.contextSlots[..], context, slots::State::Live) { continue; }
	264	+	if contexts.contexts[context.index].owner <> receiver
	265	+	or contexts.contexts[context.index].state == domains::ContextState::Stopped { continue; }
	266	+	set targets \|= 1 << window.hart;
	267	+	}
	268	+	return targets;
	269	+	}

13	13		use super::transactions;
14	14		use super::loader;
15	15		use super::dispatch;
16	16		use super::timers;
17	17		use super::sync;
	18	+	use super::remote;
	19	+
	20	+	/// Resolve the owning hart for an operation that can change CPU authority.
	21	+	unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
	22	+	match operation {
	23	+	case 11, 12, 70, 71, 73 => {},
	24	+	else => return nil,
	25	+	}
	26	+	let entry = try capability::get(table, abi::Handle(arguments[0]));
	27	+	if entry.kind <> abi::Kind::Budget { return nil; }
	28	+	return (try budgets::get(&budgets::STORE, entry.object)).hart;
	29	+	}
	30	+
	31	+	/// Execute a queued budget change at its owning hart's dispatch boundary.
	32	+	export unsafe fn remoteBudget(call: remote::Call, hart: u32, now: u64) -> remote::Reply {
	33	+	let domain = try domains::get(&domains::STORE, call.owner) catch error { return remote::Reply::Error(error); };
	34	+	if domain.state <> domains::Lifecycle::Active { return remote::Reply::Error(abi::Error::BadHandle); }
	35	+	let target = try budgetHart(&domain.memory.table, call.operation, &call.arguments[..]) catch error {
	36	+	return remote::Reply::Error(error);
	37	+	};
	38	+	if target <> hart { return remote::Reply::Error(abi::Error::BadHandle); }
	39	+	let result = try invoke(call.owner, call.operation, &call.arguments[..], now) catch error {
	40	+	return remote::Reply::Error(error);
	41	+	};
	42	+	return remote::Reply::Value(result);
	43	+	}
18	44
19	45		/// Validate resident object generations for capabilities without page ownership.
20	46		unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
21	47		match entry.kind {
22	48		case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },

9	9		use super::trap;
10	10		use super::range;
11	11		use super::calls;
12	12		use super::timers;
13	13		use super::sync;
	14	+	use super::remote;
14	15
15	16		/// Dispatch ownership and idle state retained by one physical hart.
16	17		export record State: Copy {
17	18		/// Physical hart whose windows this state selects.
18	19		hart: u32,

30	31		export record Timer: Copy {
31	32		/// Shared monotonic mtime register.
32	33		clock: u64,
33	34		/// This hart's mtimecmp register.
34	35		compare: u64,
	36	+	/// This hart's machine software-interrupt word.
	37	+	software: u64,
35	38		}
36	39
37	40		/// Timer and dispatch state owned by one physical hart.
38	41		record Runtime: Copy {
39	42		/// Current context and retained idle frame.

47	50
48	51		/// Read the executing physical hart identifier.
49	52		export fn hart() -> u32;
50	53		/// Read the current machine stack pointer.
51	54		fn stackPointer() -> u64;
	55	+	/// Trap anchor of the hart currently executing this continuation.
	56	+	fn localAnchor() -> *mut trap::Hart;
52	57		/// Address of the M-mode idle loop.
53	58		fn idleAddress() -> u64;
54		-	/// Enable machine timer delivery while global interrupts remain disabled.
	59	+	/// Enable timer and software interrupt delivery with global interrupts disabled.
55	60		fn enable();
56	61
57	62		/// Initialize this hart's idle frame and validated timer before installing dispatch.
58	63		export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
59	64		let device = try timer(machine, id);

76	81		assert id < limits::HARTS;
77	82		let stack = HARTS[id].state.stack;
78	83		let pointer = stackPointer();
79	84		assert pointer >= stack.start and pointer < stack.end;
80	85		let guard = sync::enter();
	86	+	writeSoftware(HARTS[id].timer.software, 0);
	87	+	let previous = HARTS[id].state.current;
	88	+	save(&mut HARTS[id].state, &mut domains::STORE, frame);
	89	+	let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
	90	+	let mut count: u32 = 0;
	91	+	for i in 0..limits::REMOTE_PER_HART {
	92	+	let delivery = try! remote::take(&mut remote::STORE, id) else { break; };
	93	+	match delivery.request.action {
	94	+	case remote::Action::InstructionSync => sync::syncInstructions(),
	95	+	case remote::Action::Reschedule, remote::Action::Ready(_) => {},
	96	+	case remote::Action::Budget(call) => {
	97	+	let reply = calls::remoteBudget(call, id, now(HARTS[id].timer));
	98	+	remote::respond(&mut remote::STORE, id, delivery.id, reply);
	99	+	},
	100	+	}
	101	+	set deliveries[count] = delivery;
	102	+	set count += 1;
	103	+	}
81	104		let clock = now(HARTS[id].timer);
82		-	let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
	105	+	let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
83	106		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
84	107		let mut deadline = choice.deadline;
85		-	if timeout < deadline { set deadline = timeout; }
	108	+	if timeout.deadline < deadline { set deadline = timeout.deadline; }
	109	+	for target in 0..limits::HARTS {
	110	+	if target <> id and (timeout.harts & (1 << target)) <> 0 { signal(target); }
	111	+	}
86	112		arm(HARTS[id].timer, deadline);
	113	+	if previous <> choice.context {
	114	+	if let context = previous { signalContext(context, clock); }
	115	+	}
	116	+	for delivery in &deliveries[..count] {
	117	+	assert remote::acknowledge(&mut remote::STORE, id, delivery.id);
	118	+	if delivery.request.source <> id { signal(delivery.request.source); }
	119	+	if let case remote::Action::Budget(call) = delivery.request.action { signalContext(call.context, clock); }
	120	+	if let case remote::Action::Ready(context) = delivery.request.action { signalContext(context, clock); }
	121	+	}
	122	+	sync::leave(guard);
	123	+	}
	124	+
	125	+	/// Signal an initialized online hart while request metadata is serialized.
	126	+	unsafe fn signal(target: u32) { writeSoftware(HARTS[target].timer.software, 1); }
	127	+
	128	+	/// Notify the current owner or the eligible hart after a context is released.
	129	+	unsafe fn signalContext(context: abi::Ref, now: u64) {
	130	+	if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) { return; }
	131	+	if let owner = domains::STORE.contexts[context.index].hart { signal(owner); return; }
	132	+	for i in 0..limits::BUDGETS {
	133	+	if budgets::STORE.slots[i].state <> slots::State::Live { continue; }
	134	+	let window = budgets::STORE.windows[i];
	135	+	if window.context == context and window.start <= now and now < window.end { signal(window.hart); }
	136	+	}
	137	+	}
	138	+
	139	+	/// Write one naturally aligned CLINT software-interrupt register.
	140	+	fn writeSoftware(address: u64, value: u32);
	141	+
	142	+	/// Post a local action and interrupt its target under metadata serialization.
	143	+	export unsafe fn request(target: u32, action: remote::Action) -> abi::Ref throws (abi::Error) {
	144	+	let id = try remote::send(&mut remote::STORE, hart(), target, action);
	145	+	signal(target);
	146	+	return id;
	147	+	}
	148	+
	149	+	/// Retain the caller and signal every published readiness request under the lock.
	150	+	export unsafe fn startReady(context: abi::Ref, batch: &remote::Batch) {
	151	+	assert domains::STORE.contexts[context.index].pending == nil;
	152	+	for target in 0..limits::HARTS {
	153	+	if (batch.targets & (1 << target)) == 0 { continue; }
	154	+	if domains::STORE.contexts[context.index].pending == nil {
	155	+	set domains::STORE.contexts[context.index].pending = batch.ids[target];
	156	+	}
	157	+	signal(target);
	158	+	}
	159	+	}
	160	+
	161	+	/// Consume every readiness acknowledgement before releasing the caller's storage.
	162	+	export unsafe fn finishReady(context: abi::Ref, batch: &remote::Batch) {
	163	+	for target in 0..limits::HARTS {
	164	+	if (batch.targets & (1 << target)) == 0 { continue; }
	165	+	waitRequest(batch.source, batch.ids[target]);
	166	+	}
	167	+	let guard = sync::enter();
	168	+	assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
	169	+	for target in 0..limits::HARTS {
	170	+	if (batch.targets & (1 << target)) == 0 { continue; }
	171	+	assert try! remote::result(&remote::STORE, batch.source, batch.ids[target]) == 0;
	172	+	try! remote::release(&mut remote::STORE, batch.source, batch.ids[target]);
	173	+	}
	174	+	set domains::STORE.contexts[context.index].pending = nil;
87	175		sync::leave(guard);
88	176		}
89	177
	178	+	/// Wait for an interrupt with global interrupts masked, then enable delivery.
	179	+	fn waitInterrupt();
	180	+
	181	+	/// Release a completed call's context retention and request generation.
	182	+	unsafe fn releaseRequest(source: u32, request: abi::Ref) {
	183	+	if let case remote::Action::Budget(call) = remote::STORE.requests[request.index].action {
	184	+	assert slots::matches(&domains::STORE.contextSlots[..], call.context, slots::State::Live);
	185	+	assert domains::STORE.contexts[call.context.index].pending == request;
	186	+	set domains::STORE.contexts[call.context.index].pending = nil;
	187	+	}
	188	+	try! remote::release(&mut remote::STORE, source, request);
	189	+	}
	190	+
	191	+	/// Await acknowledgement while retaining the record and allowing preemption.
	192	+	unsafe fn waitRequest(source: u32, request: abi::Ref) {
	193	+	let interrupts = sync::maskInterrupts();
	194	+	loop {
	195	+	let guard = sync::enter();
	196	+	let complete = try! remote::completed(&remote::STORE, source, request);
	197	+	sync::leave(guard);
	198	+	if complete { break; }
	199	+	waitInterrupt();
	200	+	sync::maskInterrupts();
	201	+	}
	202	+	sync::restoreInterrupts(interrupts);
	203	+	}
	204	+
	205	+	/// Await a retained reply through a preemptible kernel continuation.
	206	+	export unsafe fn awaitRequest(source: u32, request: abi::Ref) -> u64 throws (abi::Error) {
	207	+	waitRequest(source, request);
	208	+	let guard = sync::enter();
	209	+	let result = try remote::result(&remote::STORE, source, request) catch error {
	210	+	releaseRequest(source, request);
	211	+	sync::leave(guard); throw error;
	212	+	};
	213	+	releaseRequest(source, request);
	214	+	sync::leave(guard); return result;
	215	+	}
	216	+
90	217		/// Read the running context's identity and domain-relative CPU authority.
91	218		export unsafe fn current() -> abi::CurrentContextInfo {
92	219		let id = hart();
93	220		assert id < limits::HARTS;
94	221		let context = HARTS[id].state.current else panic "no current context";

111	238		sync::leave(guard);
112	239		} else {
113	240		sync::leave(guard);
114	241		calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
115	242		}
116		-	interrupt(frame, anchor);
	243	+	interrupt(frame, &mut *localAnchor());
117	244		}
118	245
119	246		/// Enter the first authorized context or the hart's retained idle frame.
120	247		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
121	248		let id = hart();

139	266		if device.kind <> platform::Kind::Clint { continue; }
140	267		if result <> nil or device.memory.end < device.memory.start
141	268		or device.memory.end - device.memory.start < 0xc000 or (device.memory.start & 7) <> 0 {
142	269		throw abi::Error::InvalidArg;
143	270		}
144		-	set result = Timer { clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8 };
	271	+	set result = Timer {
	272	+	clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8,
	273	+	software: device.memory.start + hart as u64 * 4,
	274	+	};
145	275		}
146	276		let selected = result else { throw abi::Error::InvalidArg; };
147	277		return selected;
148	278		}
149	279

206	336		/// Interrupts remain disabled and shared metadata stays serialized through exchange.
207	337		export fn exchange(state: &mut State, windows: &budgets::Store, contexts: &mut domains::Store,
208	338		frame: &mut trap::Frame, anchor: &mut trap::Hart, now: u64) -> Choice throws (abi::Error)
209	339		{
210	340		let choice = try select(windows, contexts, state.hart, now);
211		-	if let current = state.current {
212		-	assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
213		-	set contexts.contexts[current.index].frame = *frame;
214		-	set contexts.contexts[current.index].hart = nil;
215		-	}
	341	+	save(state, contexts, frame);
216	342		if let next = choice.context {
217	343		let context = contexts.contexts[next.index];
218	344		assert context.kernelStack.start < context.kernelStack.end;
219	345		set *frame = context.frame;
220	346		set anchor.stackBottom = context.kernelStack.start;

227	353		}
228	354		set state.current = choice.context;
229	355		set state.budget = choice.budget;
230	356		return choice;
231	357		}
	358	+
	359	+	/// Retain a context frame before releasing this hart's execution ownership.
	360	+	fn save(state: &mut State, contexts: &mut domains::Store, frame: &trap::Frame) {
	361	+	if let current = state.current {
	362	+	assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
	363	+	set contexts.contexts[current.index].frame = *frame;
	364	+	set contexts.contexts[current.index].hart = nil;
	365	+	}
	366	+	set state.current = nil;
	367	+	set state.budget = nil;
	368	+	}

1	1		//! Naturally aligned RV64 CLINT access with device ordering.
2	2		.text;
3	3		.export @kernel::dispatch::read;
4	4		.export @kernel::dispatch::write;
	5	+	.export @kernel::dispatch::writeSoftware;
5	6		.export @kernel::dispatch::hart;
6	7		.export @kernel::dispatch::stackPointer;
	8	+	.export @kernel::dispatch::localAnchor;
7	9		.export @kernel::dispatch::idleAddress;
8	10		.export @kernel::dispatch::enable;
9	11		.export @kernel::dispatch::interrupt;
	12	+	.export @kernel::dispatch::waitInterrupt;
	13	+
	14	+	// A pending enabled interrupt wakes wfi while global delivery is masked.
	15	+	@kernel::dispatch::waitInterrupt
	16	+	wfi;
	17	+	csrsi mstatus 8;
	18	+	ret;
10	19
11	20		// Retain the frame outside the context stack before publishing its release.
12	21		@kernel::dispatch::interrupt
13	22		mv %t0 %a0;
14	23		addi %t1 %a1 64;

35	44		// Dispatch releases context stacks only from the hart's firmware stack.
36	45		@kernel::dispatch::stackPointer
37	46		mv %a0 %sp;
38	47		ret;
39	48
	49	+	// A resumed kernel continuation uses its executing hart's trap storage.
	50	+	@kernel::dispatch::localAnchor
	51	+	csrr %a0 mscratch;
	52	+	ret;
	53	+
40	54		// Saved idle frames enter this loop with machine interrupts enabled by mret.
41	55		@kernel::dispatch::idleAddress
42	56		la %a0 @dispatchIdle;
43	57		ret;
44	58		@dispatchIdle

46	60		j @dispatchIdle;
47	61
48	62		// Global interrupt enable is restored from the selected frame by mret.
49	63		@kernel::dispatch::enable
50	64		csrr %t0 mie;
51		-	ori %t0 %t0 128;
	65	+	ori %t0 %t0 136;
52	66		csrw mie %t0;
53	67		ret;
54	68
55	69		// The validated CLINT mapping contains a naturally aligned mtime register.
56	70		@kernel::dispatch::read

63	77		@kernel::dispatch::write
64	78		fence iorw iorw;
65	79		sd %a1 0(%a0);
66	80		fence iorw iorw;
67	81		ret;
	82	+
	83	+	// CLINT software-interrupt registers are naturally aligned 32-bit words.
	84	+	@kernel::dispatch::writeSoftware
	85	+	fence iorw iorw;
	86	+	sw %a1 0(%a0);
	87	+	fence iorw iorw;
	88	+	ret;

56	56		owner: abi::Ref,
57	57		/// Whether this context can resume.
58	58		state: ContextState,
59	59		/// Hart retaining this context as its current execution owner.
60	60		hart: ?u32,
	61	+	/// Remote call whose retained continuation requires this context's storage.
	62	+	pending: ?abi::Ref,
61	63		/// Physical frames retained until this context stops.
62	64		kernelFrames: frames::Run,
63	65		/// Exclusive stack bounds used by trap entry and suspended M-mode calls.
64	66		kernelStack: range::Range,
65	67		/// Exclusive user stack extent; empty until initial activation.

190	192		authority: abi::Handle, object: abi::Ref) throws (abi::Error)
191	193		{
192	194		let permit = try capability::authority(table, authority, abi::Rights(abi::DESTROY));
193	195		let target = try context(store, permit.object, object);
194	196		if store.records[permit.object.index].initial == object { throw abi::Error::Denied; }
195		-	if target.hart <> nil { throw abi::Error::Busy; }
	197	+	if target.hart <> nil or target.pending <> nil { throw abi::Error::Busy; }
196	198		try! frames::release(&mut memory.backings.pool, target.kernelFrames);
197	199		try! slots::release(&mut store.contextSlots[..], object);
198	200		}
199	201
200	202		/// Derive a descending stack extent from one writable Page capability.

22	22		export constant CONTEXTS: u32 = 2048;
23	23		/// Maximum hardware threads.
24	24		export constant HARTS: u32 = 8;
25	25		/// One-shot timeouts retained by each hart, including pending delivery.
26	26		export constant TIMEOUTS_PER_HART: u32 = 64;
	27	+	/// Outstanding cross-hart requests, including acknowledgements awaiting release.
	28	+	export constant REMOTE_REQUESTS: u32 = 128;
	29	+	/// Queued remote requests accepted by each hart.
	30	+	export constant REMOTE_PER_HART: u32 = 16;
27	31		/// Maximum resident package records and gp table entries.
28	32		export constant PACKAGES: u32 = 256;
29	33		/// Generation values never wrap.
30	34		export constant LAST_GENERATION: u32 = 0xffffffff;

185	185		let child = slots::reference(&childSlot);
186	186		capability::initialize(&mut MEMORY.table, child);
187	187		let contextSlot = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
188	188		let context = slots::reference(&contextSlot);
189	189		set DOMAINS.contexts[context.index] = domains::Context {
190		-	owner: child, state: domains::ContextState::Ready, hart: nil,
	190	+	owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil,
191	191		kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },
192	192		userStack: range::Range { start: 0, end: 0 },
193	193		frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
194	194		};
195	195		set DOMAINS.records[child.index] = domains::Domain {

10	10		use kernel::slots;
11	11		use kernel::registry;
12	12		use kernel::limits;
13	13		use kernel::trap;
14	14		use kernel::budgets;
	15	+	use kernel::remote;
	16	+
	17	+	/// Deferred budget operations recheck authority and expiration at execution time.
	18	+	@test unsafe fn deferredBudget() throws (testing::TestError) {
	19	+	let owner = initialize();
	20	+	let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);
	21	+	let entry = try! capability::get(&MEMORY.table, handle);
	22	+	let call = remote::Call { owner, context: owner, operation: 70, arguments: [*handle, 50, 0, 0] };
	23	+	let wrongHart = calls::remoteBudget(call, 0, 20);
	24	+	assert wrongHart == remote::Reply::Error(abi::Error::BadHandle);
	25	+	remote::initialize(&mut remote::STORE, 3);
	26	+	let request = try! remote::send(&mut remote::STORE, 0, 1, remote::Action::Budget(call));
	27	+	let delivery = try! remote::take(&mut remote::STORE, 1) else panic "missing expired call";
	28	+	assert delivery.id == request;
	29	+	let case remote::Action::Budget(queued) = delivery.request.action else panic "queued budget call";
	30	+	let expired = calls::remoteBudget(queued, 1, 100);
	31	+	assert expired == remote::Reply::Error(abi::Error::Busy);
	32	+	remote::respond(&mut remote::STORE, 1, request, expired);
	33	+	assert remote::acknowledge(&mut remote::STORE, 1, request);
	34	+	assert try! remote::completed(&remote::STORE, 0, request);
	35	+	try! remote::release(&mut remote::STORE, 0, request);
	36	+	let unchanged = try! budgets::get(&budgets::STORE, entry.object);
	37	+	assert unchanged.start == 10 and unchanged.end == 100;
	38	+	let reply = calls::remoteBudget(call, 1, 20);
	39	+	let case remote::Reply::Value(value) = reply else panic "deferred split";
	40	+	let left = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
	41	+	let right = try! budgets::query(&budgets::STORE, &MEMORY.table, abi::Handle(value));
	42	+	assert left.start == 20 and left.end == 50 and right.start == 50 and right.end == 100;
	43	+	let duplicate = calls::remoteBudget(call, 1, 25);
	44	+	assert duplicate == remote::Reply::Error(abi::Error::InvalidArg);
	45	+	let retained = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
	46	+	assert retained.start == 20 and retained.end == 50;
	47	+	let dropped = try! calls::invoke(owner, 12, &[*handle, 0, 0, 0], 25);
	48	+	assert dropped == 0;
	49	+	assert calls::remoteBudget(call, 1, 25) == remote::Reply::Error(abi::Error::BadHandle);
	50	+	}
15	51
16	52		/// Mapped physical storage for four pages plus alignment padding.
17	53		static RAM: [u64; 2560] = [0; 2560];
18	54		/// Kernel-only metadata for the calling domain.
19	55		unsafe static MEMORY: domains::Memory = undefined;

40	40		let b = try! capability::get(&TABLE, last);
41	41		set BUDGETS.windows[a.object.index].context = owner;
42	42		set BUDGETS.windows[b.object.index].context = owner;
43	43		}
44	44
	45	+	/// Wakeup reaches future windows and excludes expired or stopped contexts.
	46	+	@test unsafe fn wakeTargets() throws (testing::TestError) {
	47	+	initialize();
	48	+	set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
	49	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 1;
	50	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 25) == 1;
	51	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 40) == 0;
	52	+	set DOMAINS.contexts[0].state = domains::ContextState::Stopped;
	53	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
	54	+	set DOMAINS.contexts[0].state = domains::ContextState::Ready;
	55	+	set DOMAINS.contextSlots[0].generation += 1;
	56	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
	57	+	}
	58	+
45	59		/// Dispatch selects only the half-open interval and wakes at the next runnable start.
46	60		@test unsafe fn boundaries() throws (testing::TestError) {
47	61		initialize();
48	62		let before = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
49	63		try testing::expect(before.context == nil and before.deadline == 10);

98	112		set PLATFORM.devices[0] = platform::Device {
99	113		kind: platform::Kind::Clint, memory: range::Range { start: 0x2000000, end: 0x2010000 },
100	114		};
101	115		let first = try! dispatch::timer(&PLATFORM, 0);
102	116		let last = try! dispatch::timer(&PLATFORM, 7);
	117	+	assert first.software == 0x2000000 and last.software == first.software + 28;
103	118		try testing::expect(first.clock == 0x200bff8 and first.compare == 0x2004000);
104	119		try testing::expect(last.clock == first.clock and last.compare == first.compare + 56);
105	120		let mut failures: u32 = 0;
106	121		try dispatch::timer(&PLATFORM, 1) catch err {
107	122		try testing::expect(err == abi::Error::InvalidArg); set failures += 1;

211	211		};
212	212		try! domains::returned(&mut DOMAINS, authority.object, context);
213	213		try testing::expect(DOMAINS.contexts[context.index].state == domains::ContextState::Stopped);
214	214		try testing::expect(DOMAINS.records[authority.object.index].state == domains::Lifecycle::Pending);
215	215		set DOMAINS.contexts[context.index].hart = nil;
	216	+	set DOMAINS.contexts[context.index].pending = abi::Ref { index: 0, generation: 1 };
	217	+	let mut retained = false;
	218	+	try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context) catch error {
	219	+	assert error == abi::Error::Busy; set retained = true;
	220	+	};
	221	+	assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
	222	+	for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
	223	+	assert not PAGES.backings.pool.free[i];
	224	+	}
	225	+	set DOMAINS.contexts[context.index].pending = nil;
216	226		try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context);
217	227		for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
218	228		try testing::expect(PAGES.backings.pool.free[i]);
219	229		}
220	230		try domains::context(&DOMAINS, authority.object, context) catch error {

4	4		use kernel::slots;
5	5		use kernel::domains;
6	6		use kernel::events;
7	7		use kernel::timers;
8	8		use kernel::limits;
	9	+	use kernel::budgets;
9	10
10	11		/// Timer records under test.
11	12		unsafe static TIMERS: timers::Store = undefined;
12	13		/// Domain metadata and trusted notification state.
13	14		unsafe static DOMAINS: domains::Store = undefined;
	15	+	/// CPU authority used to identify remote notification targets.
	16	+	unsafe static BUDGETS: budgets::Store = undefined;
14	17		/// Receiver ring and private metadata.
15	18		unsafe static MEMORY: domains::Memory = undefined;
16	19
17	20		/// Open an active domain queue without assigning CPU authority.
18	21		unsafe fn initialize() -> abi::Ref {
19	22		timers::initialize(&mut TIMERS);
	23	+	budgets::initialize(&mut BUDGETS);
20	24		domains::initialize(&mut DOMAINS);
21	25		let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
22	26		let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
23	27		set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
24	28		set DOMAINS.records[owner.index].memory = &mut MEMORY;
25	29		try! events::open(&mut DOMAINS.events, owner, &mut MEMORY.ring);
26	30		return owner;
27	31		}
28	32
	33	+	/// A timeout reports the hart of a receiver whose context moved before delivery.
	34	+	@test unsafe fn remoteReceiver() throws (testing::TestError) {
	35	+	let owner = initialize();
	36	+	set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };
	37	+	let context = abi::Ref { index: 0, generation: 1 };
	38	+	set DOMAINS.contexts[0].owner = owner;
	39	+	set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
	40	+	set BUDGETS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
	41	+	set BUDGETS.windows[0] = budgets::Window {
	42	+	hart: 1, start: 10, end: 100, owner, handle: abi::Handle(0), context,
	43	+	};
	44	+	try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 15, 17);
	45	+	let result = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15);
	46	+	assert result.harts == 2 and result.deadline == 0xffffffffffffffff;
	47	+	assert DOMAINS.contexts[0].state == domains::ContextState::Ready;
	48	+	let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 16);
	49	+	assert again.harts == 0;
	50	+	}
	51	+
29	52		/// A timeout is delivered once, at its deadline, on its registered hart.
30	53		@test unsafe fn boundaries() throws (testing::TestError) {
31	54		let owner = initialize();
32	55		try! timers::request(&mut TIMERS, &DOMAINS, owner, 1, 10, 5, 17);
33		-	assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 15) == 0xffffffffffffffff);
34		-	assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 14) == 15);
	56	+	assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15)).deadline == 0xffffffffffffffff);
	57	+	assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 14)).deadline == 15);
35	58		assert (try! events::pop(&mut MEMORY.ring) == nil);
36		-	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 15);
37		-	assert (next == 0xffffffffffffffff);
	59	+	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 15);
	60	+	assert (next.deadline == 0xffffffffffffffff);
38	61		let event = try! events::pop(&mut MEMORY.ring) else panic "timeout";
39	62		assert (event.kind == events::TIMEOUT and event.code == 17 and event.value == 0);
40		-	let again = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 16);
	63	+	let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 16);
41	64		assert (try! events::pop(&mut MEMORY.ring) == nil);
42	65		}
43	66
44	67		/// Full queues retain due timeouts without requesting another immediate interrupt.
45	68		@test unsafe fn saturation() throws (testing::TestError) {

52	75		let mut full = false;
53	76		try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 1, 99) catch error {
54	77		assert (error == abi::Error::Busy); set full = true;
55	78		};
56	79		assert (full);
57		-	assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1) == 0xffffffffffffffff);
	80	+	assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1)).deadline == 0xffffffffffffffff);
58	81		for i in 0..events::ORDINARY { let event = try! events::pop(&mut MEMORY.ring) else panic "wake"; }
59		-	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 2);
	82	+	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 2);
60	83		for i in 0..limits::TIMEOUTS_PER_HART {
61	84		let event = try! events::pop(&mut MEMORY.ring) else panic "retained timeout";
62	85		assert (event.kind == events::TIMEOUT and event.code == i);
63	86		}
64	87		try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 2, 1, 100);

74	97		try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0xffffffffffffffff, 1, 1) catch error {
75	98		assert (error == abi::Error::InvalidArg); set rejected += 1;
76	99		};
77	100		try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 5, 1);
78	101		set DOMAINS.records[owner.index].state = domains::Lifecycle::Dead;
79		-	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1);
80		-	assert (next == 0xffffffffffffffff and rejected == 2);
	102	+	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1);
	103	+	assert (next.deadline == 0xffffffffffffffff and rejected == 2);
81	104		assert (TIMERS.entries[0][0].owner == nil);
82	105		}

44	45		return;
45	46		}
46	47		throw abi::Error::Busy;
47	48		}
48	49
49		-	/// Deliver due entries and return the next future timeout boundary.
	50	+	/// Next timer boundary and dispatch notifications from delivered events.
	51	+	export record Result: Copy {
	52	+	/// Earliest future timeout, or the maximum tick when none is armed.
	53	+	deadline: u64,
	54	+	/// Harts with retained receiver authority that must refresh dispatch.
	55	+	harts: u32,
	56	+	}
	57	+
	58	+	/// Deliver due entries and report the next boundary and affected dispatch harts.
50	59		/// Full queues retain due entries for the next kernel entry without a timer spin.
51		-	export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, hart: u32, now: u64) -> u64 throws (abi::Error) {
	60	+	export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, windows: &budgets::Store, hart: u32, now: u64) -> Result throws (abi::Error) {
52	61		if hart >= limits::HARTS { throw abi::Error::InvalidArg; }
53		-	let mut next: u64 = 0xffffffffffffffff;
	62	+	let mut result = Result { deadline: 0xffffffffffffffff, harts: 0 };
54	63		for i in 0..limits::TIMEOUTS_PER_HART {
55	64		let entry = store.entries[hart][i];
56	65		let owner = entry.owner else { continue; };
57	66		if not slots::matches(&receivers.slots[..], owner, slots::State::Live)
58	67		or receivers.records[owner.index].state == domains::Lifecycle::Dead {
59	68		set store.entries[hart][i].owner = nil;
60	69		continue;
61	70		}
62	71		if entry.deadline > now {
63		-	if entry.deadline < next { set next = entry.deadline; }
	72	+	if entry.deadline < result.deadline { set result.deadline = entry.deadline; }
64	73		continue;
65	74		}
66	75		if try domains::notify(receivers, owner, events::Event { kind: events::TIMEOUT, reserved: 0, code: entry.token, value: 0 }) {
67	76		set store.entries[hart][i].owner = nil;
	77	+	set result.harts \|= budgets::wakeTargets(windows, receivers, owner, now);
68	78		}
69	79		}
70		-	return next;
	80	+	return result;
71	81		}