radiance · commit

kernel: Complete cross-hart transactions

04c663fcabe11a081cb2038ad5147863668277122093799071366cec0d8b328d

Execute remote budget changes at the owning hart after saving its
context. Reserve bounded readiness batches before activation or
wakeup, retain preemptible callers through acknowledgement, and notify
migrated timeout receivers.

Validate queue capacity, stale replies, queued expiration, context
retention, and atomic rejection. Native 1/2/8-hart checks cover
concurrent calls, context migration, idle-hart activation and wakeup,
and remote timeout delivery. Full make test passes with the sibling
kernel-emulator.

Alexis Sellier committed 13 days ago 1 parent ba7c3317

kernel/kernel.rad +1 -0

export mod instances;
export mod domains;
export mod budgets;
export mod dispatch;
export mod timers;
export mod remote;
export mod calls;
export mod sys;
export mod boot;
@test export mod tests;

kernel/kernel/boot.rad +5 -2

use super::registry;
use super::domains;
use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use std::arch::rv64::shared::catalog;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.

/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and user calls, and stop on unsupported traps.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    if trap::classify(frame.cause) == trap::Cause::Timer {
    let cause = trap::classify(frame.cause);
    if cause == trap::Cause::Timer or cause == trap::Cause::Software {
        dispatch::interrupt(frame, hart);
        return;
    }
    if trap::classify(frame.cause) == trap::Cause::UserCall and trap::fromUser(frame) {
    if cause == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);
        return;
    }
    print("kernel: unexpected trap\n");
    trap::halt();

        try! platform::protect(&mut PLATFORM, treeRange);
        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        domains::initialize(&mut domains::STORE);
        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        assert entryCount == 2;
        registry::initialize(&mut registry::STORE);
        let bootCatalog = @sliceOf(entries, entryCount);
        try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
        let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

kernel/kernel/budgets.rad +26 -0

/// Test half-open execution authority on the selected hart at the current tick.
export fn active(store: &Store, object: abi::Ref, hart: u32, now: u64) -> bool throws (abi::Error) {
    let window = try get(store, object);
    return window.context <> nil and window.hart == hart and window.start <= now and now < window.end;
}

/// Find all harts with retained authority for a receiver's live contexts.
export fn wakeTargets(windows: &Store, contexts: &domains::Store, receiver: abi::Ref, now: u64) -> u32 {
    let mut targets: u32 = 0;
    for i in 0..limits::BUDGETS {
        if windows.slots[i].state <> slots::State::Live {
            continue;
        }
        let window = windows.windows[i];
        if window.owner <> receiver or window.end <= now {
            continue;
        }
        let context = window.context else {
            continue;
        };
        if not slots::matches(&contexts.contextSlots[..], context, slots::State::Live) {
            continue;
        }
        if contexts.contexts[context.index].owner <> receiver
            or contexts.contexts[context.index].state == domains::ContextState::Stopped {
                continue;
            }
        set targets |= 1 << window.hart;
    }
    return targets;
}

kernel/kernel/calls.rad +94 -1

use super::transactions;
use super::loader;
use super::dispatch;
use super::timers;
use super::sync;
use super::remote;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
    match operation {
        case 11, 12, 70, 71, 73 => {
        },
        else => return nil,
    }
    let entry = try capability::get(table, abi::Handle(arguments[0]));
    if entry.kind <> abi::Kind::Budget {
        return nil;
    }
    return (try budgets::get(&budgets::STORE, entry.object)).hart;
}

/// Execute a queued budget change at its owning hart's dispatch boundary.
export unsafe fn remoteBudget(call: remote::Call, hart: u32, now: u64) -> remote::Reply {
    let domain = try domains::get(&domains::STORE, call.owner) catch error {
        return remote::Reply::Error(error);
    };
    if domain.state <> domains::Lifecycle::Active {
        return remote::Reply::Error(abi::Error::BadHandle);
    }
    let target = try budgetHart(&domain.memory.table, call.operation, &call.arguments[..]) catch error {
        return remote::Reply::Error(error);
    };
    if target <> hart {
        return remote::Reply::Error(abi::Error::BadHandle);
    }
    let result = try invoke(call.owner, call.operation, &call.arguments[..], now) catch error {
        return remote::Reply::Error(error);
    };
    return remote::Reply::Value(result);
}

/// Validate resident object generations for capabilities without page ownership.
unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => {

    sync::leave(publicationGuard);
    return *result;
}

/// Execute a call with short metadata transactions and private bulk initialization.
/// The running context keeps its domain storage live until dispatch releases it.
/// Context ownership and pending calls retain storage across dispatch boundaries.
export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    if operation == 21 or operation == 50 {
        return try runtimeReady(owner, operation, arguments, now);
    }
    let guard = sync::enter();
    let mut domain = try domains::get(&domains::STORE, owner) catch error {
        sync::leave(guard);
        throw error;
    };
    if domain.state <> domains::Lifecycle::Active {
        sync::leave(guard);
        throw abi::Error::BadHandle;
    }
    let target = try budgetHart(&domain.memory.table, operation, arguments) catch error {
        sync::leave(guard);
        throw error;
    };
    if let destination = target {
        let source = dispatch::hart();
        if destination <> source {
            let call = remote::Call { owner, context: abi::reference(dispatch::current().context), operation,
                arguments: [arguments[0], arguments[1], arguments[2], arguments[3]] };
            let request = try dispatch::request(destination, remote::Action::Budget(call)) catch error {
                sync::leave(guard); throw error;
            };
            assert domains::STORE.contexts[call.context.index].pending == nil;
            set domains::STORE.contexts[call.context.index].pending = request;
            sync::leave(guard);
            return try dispatch::awaitRequest(source, request);
        }
    }
    let handle = abi::Handle(arguments[0]);
    match operation {
        case 20 => {
            let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
                &mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error {

            sync::leave(guard); return result;
        },
    }
}

/// Reserve remote dispatch capacity before activation or event publication.
unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) {
        throw abi::Error::InvalidArg;
    }
    let guard = sync::enter();
    let domain = try domains::get(&domains::STORE, owner) catch error {
        sync::leave(guard);
        throw error;
    };
    if domain.state <> domains::Lifecycle::Active {
        sync::leave(guard);
        throw abi::Error::BadHandle;
    }
    let handle = abi::Handle(arguments[0]);
    let rights = abi::EXECUTE if operation == 21 else abi::WAKE;
    let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
        sync::leave(guard); throw error;
    };
    let source = dispatch::hart();
    let context = abi::reference(dispatch::current().context);
    let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
    let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error {
        sync::leave(guard);
        throw error;
    };
    let result = try invoke(owner, operation, arguments, now) catch error {
        remote::cancelReservation(&mut remote::STORE, pending);
        sync::leave(guard); throw error;
    };
    let batch = remote::publish(&mut remote::STORE, pending, remote::Action::Ready(context));
    dispatch::startReady(context, &batch);
    sync::leave(guard);
    dispatch::finishReady(context, &batch);
    return 0;
}

/// Complete a user ecall through the supplied execution boundary.
export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
    execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
{
    assert trap::fromUser(frame) and frame.cause == 8;

kernel/kernel/dispatch.rad +181 -14

use super::trap;
use super::range;
use super::calls;
use super::timers;
use super::sync;
use super::remote;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

export record Timer: Copy {
    /// Shared monotonic mtime register.
    clock: u64,
    /// This hart's mtimecmp register.
    compare: u64,
    /// This hart's machine software-interrupt word.
    software: u64,
}

/// Timer and dispatch state owned by one physical hart.
record Runtime: Copy {
    /// Current context and retained idle frame.


/// Read the executing physical hart identifier.
export fn hart() -> u32;
/// Read the current machine stack pointer.
fn stackPointer() -> u64;
/// Trap anchor of the hart currently executing this continuation.
fn localAnchor() -> *mut trap::Hart;
/// Address of the M-mode idle loop.
fn idleAddress() -> u64;
/// Enable machine timer delivery while global interrupts remain disabled.
/// Enable timer and software interrupt delivery with global interrupts disabled.
fn enable();

/// Initialize this hart's idle frame and validated timer before installing dispatch.
export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
    let device = try timer(machine, id);

    assert id < limits::HARTS;
    let stack = HARTS[id].state.stack;
    let pointer = stackPointer();
    assert pointer >= stack.start and pointer < stack.end;
    let guard = sync::enter();
    writeSoftware(HARTS[id].timer.software, 0);
    let previous = HARTS[id].state.current;
    save(&mut HARTS[id].state, &mut domains::STORE, frame);
    let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
    let mut count: u32 = 0;
    for i in 0..limits::REMOTE_PER_HART {
        let delivery = try! remote::take(&mut remote::STORE, id) else {
            break;
        };
        match delivery.request.action {
            case remote::Action::InstructionSync => sync::syncInstructions(),
            case remote::Action::Reschedule, remote::Action::Ready(_) => {
            },
            case remote::Action::Budget(call) => {
                let reply = calls::remoteBudget(call, id, now(HARTS[id].timer));
                remote::respond(&mut remote::STORE, id, delivery.id, reply);
            },
        }
        set deliveries[count] = delivery;
        set count += 1;
    }
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout < deadline {
        set deadline = timeout;
    if timeout.deadline < deadline {
        set deadline = timeout.deadline;
    }
    for target in 0..limits::HARTS {
        if target <> id and (timeout.harts & (1 << target)) <> 0 {
            signal(target);
        }
    }
    arm(HARTS[id].timer, deadline);
    if previous <> choice.context {
        if let context = previous {
            signalContext(context, clock);
        }
    }
    for delivery in &deliveries[..count] {
        assert remote::acknowledge(&mut remote::STORE, id, delivery.id);
        if delivery.request.source <> id {
            signal(delivery.request.source);
        }
        if let case remote::Action::Budget(call) = delivery.request.action {
            signalContext(call.context, clock);
        }
        if let case remote::Action::Ready(context) = delivery.request.action {
            signalContext(context, clock);
        }
    }
    sync::leave(guard);
}

/// Signal an initialized online hart while request metadata is serialized.
unsafe fn signal(target: u32) {
    writeSoftware(HARTS[target].timer.software, 1);
}

/// Notify the current owner or the eligible hart after a context is released.
unsafe fn signalContext(context: abi::Ref, now: u64) {
    if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) {
        return;
    }
    if let owner = domains::STORE.contexts[context.index].hart {
        signal(owner);
        return;
    }
    for i in 0..limits::BUDGETS {
        if budgets::STORE.slots[i].state <> slots::State::Live {
            continue;
        }
        let window = budgets::STORE.windows[i];
        if window.context == context and window.start <= now and now < window.end {
            signal(window.hart);
        }
    }
}

/// Write one naturally aligned CLINT software-interrupt register.
fn writeSoftware(address: u64, value: u32);

/// Post a local action and interrupt its target under metadata serialization.
export unsafe fn request(target: u32, action: remote::Action) -> abi::Ref throws (abi::Error) {
    let id = try remote::send(&mut remote::STORE, hart(), target, action);
    signal(target);
    return id;
}

/// Retain the caller and signal every published readiness request under the lock.
export unsafe fn startReady(context: abi::Ref, batch: &remote::Batch) {
    assert domains::STORE.contexts[context.index].pending == nil;
    for target in 0..limits::HARTS {
        if (batch.targets & (1 << target)) == 0 {
            continue;
        }
        if domains::STORE.contexts[context.index].pending == nil {
            set domains::STORE.contexts[context.index].pending = batch.ids[target];
        }
        signal(target);
    }
}

/// Consume every readiness acknowledgement before releasing the caller's storage.
export unsafe fn finishReady(context: abi::Ref, batch: &remote::Batch) {
    for target in 0..limits::HARTS {
        if (batch.targets & (1 << target)) == 0 {
            continue;
        }
        waitRequest(batch.source, batch.ids[target]);
    }
    let guard = sync::enter();
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    for target in 0..limits::HARTS {
        if (batch.targets & (1 << target)) == 0 {
            continue;
        }
        assert try! remote::result(&remote::STORE, batch.source, batch.ids[target]) == 0;
        try! remote::release(&mut remote::STORE, batch.source, batch.ids[target]);
    }
    set domains::STORE.contexts[context.index].pending = nil;
    sync::leave(guard);
}

/// Wait for an interrupt with global interrupts masked, then enable delivery.
fn waitInterrupt();

/// Release a completed call's context retention and request generation.
unsafe fn releaseRequest(source: u32, request: abi::Ref) {
    if let case remote::Action::Budget(call) = remote::STORE.requests[request.index].action {
        assert slots::matches(&domains::STORE.contextSlots[..], call.context, slots::State::Live);
        assert domains::STORE.contexts[call.context.index].pending == request;
        set domains::STORE.contexts[call.context.index].pending = nil;
    }
    try! remote::release(&mut remote::STORE, source, request);
}

/// Await acknowledgement while retaining the record and allowing preemption.
unsafe fn waitRequest(source: u32, request: abi::Ref) {
    let interrupts = sync::maskInterrupts();
    loop {
        let guard = sync::enter();
        let complete = try! remote::completed(&remote::STORE, source, request);
        sync::leave(guard);
        if complete {
            break;
        }
        waitInterrupt();
        sync::maskInterrupts();
    }
    sync::restoreInterrupts(interrupts);
}

/// Await a retained reply through a preemptible kernel continuation.
export unsafe fn awaitRequest(source: u32, request: abi::Ref) -> u64 throws (abi::Error) {
    waitRequest(source, request);
    let guard = sync::enter();
    let result = try remote::result(&remote::STORE, source, request) catch error {
        releaseRequest(source, request);
        sync::leave(guard); throw error;
    };
    releaseRequest(source, request);
    sync::leave(guard); return result;
}

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else panic "no current context";

        sync::leave(guard);
    } else {
        sync::leave(guard);
        calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
    }
    interrupt(frame, anchor);
    interrupt(frame, &mut *localAnchor());
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();

        }
        if result <> nil or device.memory.end < device.memory.start
            or device.memory.end - device.memory.start < 0xc000 or (device.memory.start & 7) <> 0 {
            throw abi::Error::InvalidArg;
        }
        set result = Timer { clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8 };
        set result = Timer {
            clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8,
            software: device.memory.start + hart as u64 * 4,
        };
    }
    let selected = result else {
        throw abi::Error::InvalidArg;
    };
    return selected;
}

/// Read the platform's monotonic clock in mtime ticks.
export fn now(timer: Timer) -> u64 {
    return read(timer.clock);
}
export fn now(timer: Timer) -> u64 { return read(timer.clock); }

/// Program the next boundary while this hart's timer handling is serialized.
export fn arm(timer: Timer, deadline: u64) {
    write(timer.compare, deadline);
}

/// Interrupts remain disabled and shared metadata stays serialized through exchange.
export fn exchange(state: &mut State, windows: &budgets::Store, contexts: &mut domains::Store,
    frame: &mut trap::Frame, anchor: &mut trap::Hart, now: u64) -> Choice throws (abi::Error)
{
    let choice = try select(windows, contexts, state.hart, now);
    if let current = state.current {
        assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
        set contexts.contexts[current.index].frame = *frame;
        set contexts.contexts[current.index].hart = nil;
    }
    save(state, contexts, frame);
    if let next = choice.context {
        let context = contexts.contexts[next.index];
        assert context.kernelStack.start < context.kernelStack.end;
        set *frame = context.frame;
        set anchor.stackBottom = context.kernelStack.start;

    }
    set state.current = choice.context;
    set state.budget = choice.budget;
    return choice;
}

/// Retain a context frame before releasing this hart's execution ownership.
fn save(state: &mut State, contexts: &mut domains::Store, frame: &trap::Frame) {
    if let current = state.current {
        assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
        set contexts.contexts[current.index].frame = *frame;
        set contexts.contexts[current.index].hart = nil;
    }
    set state.current = nil;
    set state.budget = nil;
}

kernel/kernel/dispatch.ras +22 -1

//! Naturally aligned RV64 CLINT access with device ordering.
.text;
.export @kernel::dispatch::read;
.export @kernel::dispatch::write;
.export @kernel::dispatch::writeSoftware;
.export @kernel::dispatch::hart;
.export @kernel::dispatch::stackPointer;
.export @kernel::dispatch::localAnchor;
.export @kernel::dispatch::idleAddress;
.export @kernel::dispatch::enable;
.export @kernel::dispatch::interrupt;
.export @kernel::dispatch::waitInterrupt;

// A pending enabled interrupt wakes wfi while global delivery is masked.
@kernel::dispatch::waitInterrupt
    wfi;
    csrsi mstatus 8;
    ret;

// Retain the frame outside the context stack before publishing its release.
@kernel::dispatch::interrupt
    mv %t0 %a0;
    addi %t1 %a1 64;

// Dispatch releases context stacks only from the hart's firmware stack.
@kernel::dispatch::stackPointer
    mv %a0 %sp;
    ret;

// A resumed kernel continuation uses its executing hart's trap storage.
@kernel::dispatch::localAnchor
    csrr %a0 mscratch;
    ret;

// Saved idle frames enter this loop with machine interrupts enabled by mret.
@kernel::dispatch::idleAddress
    la %a0 @dispatchIdle;
    ret;
@dispatchIdle

    j @dispatchIdle;

// Global interrupt enable is restored from the selected frame by mret.
@kernel::dispatch::enable
    csrr %t0 mie;
    ori %t0 %t0 128;
    ori %t0 %t0 136;
    csrw mie %t0;
    ret;

// The validated CLINT mapping contains a naturally aligned mtime register.
@kernel::dispatch::read

@kernel::dispatch::write
    fence iorw iorw;
    sd %a1 0(%a0);
    fence iorw iorw;
    ret;

// CLINT software-interrupt registers are naturally aligned 32-bit words.
@kernel::dispatch::writeSoftware
    fence iorw iorw;
    sw %a1 0(%a0);
    fence iorw iorw;
    ret;

kernel/kernel/domains.rad +5 -3

    owner: abi::Ref,
    /// Whether this context can resume.
    state: ContextState,
    /// Hart retaining this context as its current execution owner.
    hart: ?u32,
    /// Remote call whose retained continuation requires this context's storage.
    pending: ?abi::Ref,
    /// Physical frames retained until this context stops.
    kernelFrames: frames::Run,
    /// Exclusive stack bounds used by trap entry and suspended M-mode calls.
    kernelStack: range::Range,
    /// Exclusive user stack extent; empty until initial activation.

    let permit = try capability::authority(table, authority, abi::Rights(abi::DESTROY));
    let target = try context(store, permit.object, object);
    if store.records[permit.object.index].initial == object {
        throw abi::Error::Denied;
    }
    if target.hart <> nil {
    if target.hart <> nil or target.pending <> nil {
        throw abi::Error::Busy;
    }
    try! frames::release(&mut memory.backings.pool, target.kernelFrames);
    try! slots::release(&mut store.contextSlots[..], object);
}

    set frame.registers[3] = domain.graph.table.ptr as u64;
    set frame.registers[10] = start.args;
    set frame.registers[11] = start.size;
    let object = slots::reference(&slot);
    set store.contexts[object.index] = Context {
        owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
        owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames, kernelStack, userStack, frame,
    };
    match slot {
        case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
            object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
        }),

            // MPIE enables interrupts after the first return into U-mode.
            let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
            set frame.registers[1] = returnAddress();
            set frame.registers[3] = graph.table.ptr as u64;
            set store.contexts[initial.index] = Context {
                owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,

kernel/kernel/limits.rad +4 -0

export constant CONTEXTS: u32 = 2048;
/// Maximum hardware threads.
export constant HARTS: u32 = 8;
/// One-shot timeouts retained by each hart, including pending delivery.
export constant TIMEOUTS_PER_HART: u32 = 64;
/// Outstanding cross-hart requests, including acknowledgements awaiting release.
export constant REMOTE_REQUESTS: u32 = 128;
/// Queued remote requests accepted by each hart.
export constant REMOTE_PER_HART: u32 = 16;
/// Maximum resident package records and gp table entries.
export constant PACKAGES: u32 = 256;
/// Generation values never wrap.
export constant LAST_GENERATION: u32 = 0xffffffff;

kernel/kernel/remote.rad added +317 -0

1	+	//! Fixed request queues and generation-checked completion records.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::slots;
5	+
6	+	/// A scalar operation retained until its owning hart reaches dispatch.
7	+	export record Call: Copy {
8	+	/// Domain whose capability table supplies the arguments.
9	+	owner: abi::Ref,
10	+	/// Context whose kernel continuation consumes the result.
11	+	context: abi::Ref,
12	+	/// Budget operation to execute with serialized metadata.
13	+	operation: u64,
14	+	/// Original scalar syscall arguments.
15	+	arguments: [u64; 4],
16	+	}
17	+
18	+	/// Result retained with a request's acknowledgement.
19	+	export union Reply: Copy {
20	+	/// Successful scalar return value.
21	+	Value(u64),
22	+	/// Operation failure with no partial commit.
23	+	Error(abi::Error),
24	+	}
25	+
26	+	/// Work performed by the receiving hart before acknowledgement.
27	+	export union Action: Copy {
28	+	/// Refresh dispatch eligibility and its timer boundary.
29	+	Reschedule,
30	+	/// Synchronize local instruction fetch before dispatch.
31	+	InstructionSync,
32	+	/// Execute a budget operation after releasing local context ownership.
33	+	Budget(Call),
34	+	/// Refresh domain eligibility and notify the waiting call's context.
35	+	Ready(abi::Ref),
36	+	}
37	+
38	+	/// Request delivery state retained until the source releases it.
39	+	export union State: Copy {
40	+	/// The request is queued for its target.
41	+	Pending,
42	+	/// The target has taken the request.
43	+	Delivered,
44	+	/// The target completed the requested work.
45	+	Acknowledged,
46	+	}
47	+
48	+	/// One request owned by a source hart.
49	+	export record Request: Copy {
50	+	/// Hart that must release the completion record.
51	+	source: u32,
52	+	/// Hart that performs and acknowledges the action.
53	+	target: u32,
54	+	/// Required local action.
55	+	action: Action,
56	+	/// Current delivery state.
57	+	state: State,
58	+	/// Scalar result available after acknowledgement.
59	+	reply: Reply,
60	+	}
61	+
62	+	/// Bounded FIFO whose indices remain within its fixed array.
63	+	export record Queue: Copy {
64	+	/// Generation-bearing request references in delivery order.
65	+	entries: [abi::Ref; limits::REMOTE_PER_HART],
66	+	/// Next entry to consume.
67	+	read: u32,
68	+	/// Next entry to produce.
69	+	write: u32,
70	+	/// Number of occupied entries, including cancelled references.
71	+	count: u32,
72	+	}
73	+
74	+	/// Remote request metadata serialized by the shared kernel lock.
75	+	export record Store: Copy {
76	+	/// Online physical hart mask fixed at boot.
77	+	online: u32,
78	+	/// Request generations retained through acknowledgement.
79	+	slots: [slots::Slot; limits::REMOTE_REQUESTS],
80	+	/// Live request payloads.
81	+	requests: [Request; limits::REMOTE_REQUESTS],
82	+	/// FIFO owned by each receiving hart.
83	+	queues: [Queue; limits::HARTS],
84	+	}
85	+
86	+	/// Global cross-hart request storage.
87	+	export unsafe static STORE: Store = undefined;
88	+
89	+	/// Delivery identity and immutable routing information.
90	+	export record Delivery: Copy {
91	+	/// Live request generation to acknowledge.
92	+	id: abi::Ref,
93	+	/// Routing, action, and delivered state.
94	+	request: Request,
95	+	}
96	+
97	+	/// One completion reference for each hart selected by a bounded request batch.
98	+	export record Batch: Copy {
99	+	/// Hart that owns and releases all completion records.
100	+	source: u32,
101	+	/// Target hart mask; only these reference array elements are initialized.
102	+	targets: u32,
103	+	/// Request references indexed by target hart.
104	+	ids: [abi::Ref; limits::HARTS],
105	+	}
106	+
107	+	/// Reserved records whose queue capacity stays protected by the metadata lock.
108	+	export union Reservation: Once {
109	+	/// Batch to publish or cancel before releasing metadata serialization.
110	+	Held(Batch),
111	+	}
112	+
113	+	/// Reserve every target before any request or associated state becomes visible.
114	+	export unsafe fn reserve(store: &mut Store, source: u32, targets: u32) -> Reservation throws (abi::Error) {
115	+	try hart(store, source);
116	+	if (targets & ~store.online) <> 0 {
117	+	throw abi::Error::InvalidArg;
118	+	}
119	+	let mut count: u32 = 0;
120	+	for target in 0..limits::HARTS {
121	+	if (targets & (1 << target)) == 0 {
122	+	continue;
123	+	}
124	+	if store.queues[target].count == limits::REMOTE_PER_HART {
125	+	throw abi::Error::Busy;
126	+	}
127	+	set count += 1;
128	+	}
129	+	let mut free: u32 = 0;
130	+	for slot in &store.slots[..] {
131	+	if slot.state == slots::State::Free {
132	+	set free += 1;
133	+	}
134	+	}
135	+	if free < count {
136	+	throw abi::Error::Busy;
137	+	}
138	+	let mut batch = Batch { source, targets, ids: undefined };
139	+	for target in 0..limits::HARTS {
140	+	if (targets & (1 << target)) == 0 {
141	+	continue;
142	+	}
143	+	let pending = try! slots::reserve(&mut store.slots[..]);
144	+	match pending {
145	+	case slots::Reservation::Held(id) => {
146	+	set batch.ids[target] = id;
147	+	},
148	+	}
149	+	}
150	+	return Reservation::Held(batch);
151	+	}
152	+
153	+	/// Return all unpublished records while the queue-capacity lock remains held.
154	+	export fn cancelReservation(store: &mut Store, reservation: Reservation) {
155	+	match reservation {
156	+	case Reservation::Held(batch) => {
157	+	for target in 0..limits::HARTS {
158	+	if (batch.targets & (1 << target)) == 0 {
159	+	continue;
160	+	}
161	+	try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(batch.ids[target]));
162	+	}
163	+	},
164	+	}
165	+	}
166	+
167	+	/// Publish the reserved batch before releasing metadata serialization.
168	+	export fn publish(store: &mut Store, reservation: Reservation, action: Action) -> Batch {
169	+	match reservation {
170	+	case Reservation::Held(batch) => {
171	+	for target in 0..limits::HARTS {
172	+	if (batch.targets & (1 << target)) == 0 {
173	+	continue;
174	+	}
175	+	let id = batch.ids[target];
176	+	set store.requests[id.index] = Request {
177	+	source: batch.source, target, action, state: State::Pending, reply: Reply::Value(0),
178	+	};
179	+	enqueue(store, target, id);
180	+	let live = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(id));
181	+	}
182	+	return batch;
183	+	},
184	+	}
185	+	}
186	+
187	+	/// Append a generation reference to a queue with reserved capacity.
188	+	fn enqueue(store: &mut Store, target: u32, id: abi::Ref) {
189	+	assert store.queues[target].count < limits::REMOTE_PER_HART;
190	+	let index = store.queues[target].write;
191	+	set store.queues[target].entries[index] = id;
192	+	set store.queues[target].write = (index + 1) % limits::REMOTE_PER_HART;
193	+	set store.queues[target].count += 1;
194	+	}
195	+
196	+	/// Initialize all request capacity before secondary harts start.
197	+	export fn initialize(store: &mut Store, online: u32) {
198	+	assert online <> 0 and (online >> limits::HARTS) == 0;
199	+	set store.online = online;
200	+	slots::initialize(&mut store.slots[..]);
201	+	for hart in 0..limits::HARTS {
202	+	set store.queues[hart].read = 0;
203	+	set store.queues[hart].write = 0;
204	+	set store.queues[hart].count = 0;
205	+	}
206	+	}
207	+
208	+	/// Check an online hart before indexing its mailbox.
209	+	fn hart(store: &Store, id: u32) throws (abi::Error) {
210	+	if id >= limits::HARTS or (store.online & (1 << id)) == 0 {
211	+	throw abi::Error::InvalidArg;
212	+	}
213	+	}
214	+
215	+	/// Enqueue a request atomically under metadata serialization.
216	+	export fn send(store: &mut Store, source: u32, target: u32, action: Action) -> abi::Ref throws (abi::Error) {
217	+	try hart(store, source); try hart(store, target);
218	+	if store.queues[target].count == limits::REMOTE_PER_HART {
219	+	throw abi::Error::Busy;
220	+	}
221	+	let pending = try slots::reserve(&mut store.slots[..]) catch {
222	+	throw abi::Error::Busy;
223	+	};
224	+	let id = slots::reference(&pending);
225	+	set store.requests[id.index] = Request { source, target, action, state: State::Pending, reply: Reply::Value(0) };
226	+	enqueue(store, target, id);
227	+	return try! slots::commit(&mut store.slots[..], pending);
228	+	}
229	+
230	+	/// Take the next live request, discarding stale or completed queue references.
231	+	export fn take(store: &mut Store, target: u32) -> ?Delivery throws (abi::Error) {
232	+	try hart(store, target);
233	+	for i in 0..limits::REMOTE_PER_HART {
234	+	if store.queues[target].count == 0 {
235	+	return nil;
236	+	}
237	+	let index = store.queues[target].read;
238	+	let id = store.queues[target].entries[index];
239	+	set store.queues[target].read = (index + 1) % limits::REMOTE_PER_HART;
240	+	set store.queues[target].count -= 1;
241	+	if not slots::matches(&store.slots[..], id, slots::State::Live) {
242	+	continue;
243	+	}
244	+	let request = store.requests[id.index];
245	+	if request.target <> target or request.state <> State::Pending {
246	+	continue;
247	+	}
248	+	set store.requests[id.index].state = State::Delivered;
249	+	return Delivery { id, request: store.requests[id.index] };
250	+	}
251	+	return nil;
252	+	}
253	+
254	+	/// Acknowledge one delivered generation exactly once under metadata serialization.
255	+	export fn acknowledge(store: &mut Store, target: u32, id: abi::Ref) -> bool {
256	+	if not slots::matches(&store.slots[..], id, slots::State::Live) {
257	+	return false;
258	+	}
259	+	let request = store.requests[id.index];
260	+	if request.target <> target or request.state <> State::Delivered {
261	+	return false;
262	+	}
263	+	set store.requests[id.index].state = State::Acknowledged;
264	+	return true;
265	+	}
266	+
267	+	/// Resolve a live request through its issuing hart.
268	+	fn owned(store: &Store, source: u32, id: abi::Ref) -> Request throws (abi::Error) {
269	+	if not slots::matches(&store.slots[..], id, slots::State::Live) {
270	+	throw abi::Error::BadHandle;
271	+	}
272	+	let request = store.requests[id.index];
273	+	if request.source <> source {
274	+	throw abi::Error::BadHandle;
275	+	}
276	+	return request;
277	+	}
278	+
279	+	/// Test completion while retaining the acknowledgement record.
280	+	export fn completed(store: &Store, source: u32, id: abi::Ref) -> bool throws (abi::Error) {
281	+	return (try owned(store, source, id)).state == State::Acknowledged;
282	+	}
283	+
284	+	/// Store a result before the receiving hart acknowledges its delivery.
285	+	export fn respond(store: &mut Store, target: u32, id: abi::Ref, reply: Reply) {
286	+	assert slots::matches(&store.slots[..], id, slots::State::Live);
287	+	assert store.requests[id.index].target == target and store.requests[id.index].state == State::Delivered;
288	+	set store.requests[id.index].reply = reply;
289	+	}
290	+
291	+	/// Read a completed operation's result without releasing its record.
292	+	export fn result(store: &Store, source: u32, id: abi::Ref) -> u64 throws (abi::Error) {
293	+	let request = try owned(store, source, id);
294	+	if request.state <> State::Acknowledged {
295	+	throw abi::Error::Busy;
296	+	}
297	+	match request.reply {
298	+	case Reply::Value(value) => return value,
299	+	case Reply::Error(error) => throw error,
300	+	}
301	+	}
302	+
303	+	/// Release an acknowledged request and advance its generation.
304	+	export fn release(store: &mut Store, source: u32, id: abi::Ref) throws (abi::Error) {
305	+	if (try owned(store, source, id)).state <> State::Acknowledged {
306	+	throw abi::Error::Busy;
307	+	}
308	+	try! slots::release(&mut store.slots[..], id);
309	+	}
310	+
311	+	/// Cancel queued work or discard a completed acknowledgement.
312	+	export fn cancel(store: &mut Store, source: u32, id: abi::Ref) throws (abi::Error) {
313	+	if (try owned(store, source, id)).state == State::Delivered {
314	+	throw abi::Error::Busy;
315	+	}
316	+	try! slots::release(&mut store.slots[..], id);
317	+	}

kernel/kernel/tests.rad +1 -0

export mod domains;
export mod budgets;
export mod dispatch;
export mod calls;
export mod timers;
export mod remote;

kernel/kernel/tests/budgets.rad +1 -1

    let child = slots::reference(&childSlot);
    capability::initialize(&mut MEMORY.table, child);
    let contextSlot = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
    let context = slots::reference(&contextSlot);
    set DOMAINS.contexts[context.index] = domains::Context {
        owner: child, state: domains::ContextState::Ready, hart: nil,
        owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil,
        kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },
        userStack: range::Range { start: 0, end: 0 },
        frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
    };
    set DOMAINS.records[child.index] = domains::Domain {

kernel/kernel/tests/calls.rad +36 -0

use kernel::slots;
use kernel::registry;
use kernel::limits;
use kernel::trap;
use kernel::budgets;
use kernel::remote;

/// Deferred budget operations recheck authority and expiration at execution time.
@test unsafe fn deferredBudget() throws (testing::TestError) {
    let owner = initialize();
    let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);
    let entry = try! capability::get(&MEMORY.table, handle);
    let call = remote::Call { owner, context: owner, operation: 70, arguments: [*handle, 50, 0, 0] };
    let wrongHart = calls::remoteBudget(call, 0, 20);
    assert wrongHart == remote::Reply::Error(abi::Error::BadHandle);
    remote::initialize(&mut remote::STORE, 3);
    let request = try! remote::send(&mut remote::STORE, 0, 1, remote::Action::Budget(call));
    let delivery = try! remote::take(&mut remote::STORE, 1) else panic "missing expired call";
    assert delivery.id == request;
    let case remote::Action::Budget(queued) = delivery.request.action else panic "queued budget call";
    let expired = calls::remoteBudget(queued, 1, 100);
    assert expired == remote::Reply::Error(abi::Error::Busy);
    remote::respond(&mut remote::STORE, 1, request, expired);
    assert remote::acknowledge(&mut remote::STORE, 1, request);
    assert try! remote::completed(&remote::STORE, 0, request);
    try! remote::release(&mut remote::STORE, 0, request);
    let unchanged = try! budgets::get(&budgets::STORE, entry.object);
    assert unchanged.start == 10 and unchanged.end == 100;
    let reply = calls::remoteBudget(call, 1, 20);
    let case remote::Reply::Value(value) = reply else panic "deferred split";
    let left = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
    let right = try! budgets::query(&budgets::STORE, &MEMORY.table, abi::Handle(value));
    assert left.start == 20 and left.end == 50 and right.start == 50 and right.end == 100;
    let duplicate = calls::remoteBudget(call, 1, 25);
    assert duplicate == remote::Reply::Error(abi::Error::InvalidArg);
    let retained = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
    assert retained.start == 20 and retained.end == 50;
    let dropped = try! calls::invoke(owner, 12, &[*handle, 0, 0, 0], 25);
    assert dropped == 0;
    assert calls::remoteBudget(call, 1, 25) == remote::Reply::Error(abi::Error::BadHandle);
}

/// Mapped physical storage for four pages plus alignment padding.
static RAM: [u64; 2560] = [0; 2560];
/// Kernel-only metadata for the calling domain.
unsafe static MEMORY: domains::Memory = undefined;

kernel/kernel/tests/dispatch.rad +15 -0

    let b = try! capability::get(&TABLE, last);
    set BUDGETS.windows[a.object.index].context = owner;
    set BUDGETS.windows[b.object.index].context = owner;
}

/// Wakeup reaches future windows and excludes expired or stopped contexts.
@test unsafe fn wakeTargets() throws (testing::TestError) {
    initialize();
    set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 1;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 25) == 1;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 40) == 0;
    set DOMAINS.contexts[0].state = domains::ContextState::Stopped;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
    set DOMAINS.contexts[0].state = domains::ContextState::Ready;
    set DOMAINS.contextSlots[0].generation += 1;
    assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
}

/// Dispatch selects only the half-open interval and wakes at the next runnable start.
@test unsafe fn boundaries() throws (testing::TestError) {
    initialize();
    let before = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
    try testing::expect(before.context == nil and before.deadline == 10);

    set PLATFORM.devices[0] = platform::Device {
        kind: platform::Kind::Clint, memory: range::Range { start: 0x2000000, end: 0x2010000 },
    };
    let first = try! dispatch::timer(&PLATFORM, 0);
    let last = try! dispatch::timer(&PLATFORM, 7);
    assert first.software == 0x2000000 and last.software == first.software + 28;
    try testing::expect(first.clock == 0x200bff8 and first.compare == 0x2004000);
    try testing::expect(last.clock == first.clock and last.compare == first.compare + 56);
    let mut failures: u32 = 0;
    try dispatch::timer(&PLATFORM, 1) catch err {
        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;

kernel/kernel/tests/domains.rad +10 -0

    };
    try! domains::returned(&mut DOMAINS, authority.object, context);
    try testing::expect(DOMAINS.contexts[context.index].state == domains::ContextState::Stopped);
    try testing::expect(DOMAINS.records[authority.object.index].state == domains::Lifecycle::Pending);
    set DOMAINS.contexts[context.index].hart = nil;
    set DOMAINS.contexts[context.index].pending = abi::Ref { index: 0, generation: 1 };
    let mut retained = false;
    try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context) catch error {
        assert error == abi::Error::Busy; set retained = true;
    };
    assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        assert not PAGES.backings.pool.free[i];
    }
    set DOMAINS.contexts[context.index].pending = nil;
    try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context);
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        try testing::expect(PAGES.backings.pool.free[i]);
    }
    try domains::context(&DOMAINS, authority.object, context) catch error {

kernel/kernel/tests/remote.rad added +208 -0

1	+	//! Bounded request queues, acknowledgement identity, and capacity recovery.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::remote;
5	+	use kernel::limits;
6	+	use kernel::slots;
7	+
8	+	/// Serialized request state shared by two test harts.
9	+	unsafe static STORE: remote::Store = undefined;
10	+
11	+	/// Requests retain source ownership until an explicit acknowledgement is released.
12	+	@test unsafe fn acknowledgement() throws (testing::TestError) {
13	+	remote::initialize(&mut STORE, 3);
14	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
15	+	assert not try! remote::completed(&STORE, 0, id);
16	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing request";
17	+	assert delivery.id == id and delivery.request.source == 0;
18	+	assert not remote::acknowledge(&mut STORE, 0, id);
19	+	assert remote::acknowledge(&mut STORE, 1, id);
20	+	assert not remote::acknowledge(&mut STORE, 1, id);
21	+	assert try! remote::completed(&STORE, 0, id);
22	+	let mut denied = false;
23	+	try remote::release(&mut STORE, 1, id) catch error {
24	+	assert error == abi::Error::BadHandle; set denied = true;
25	+	};
26	+	assert denied;
27	+	try! remote::release(&mut STORE, 0, id);
28	+	assert not remote::acknowledge(&mut STORE, 1, id);
29	+	}
30	+
31	+	/// Cancellation leaves a harmless stale queue entry until the receiver drains it.
32	+	@test unsafe fn saturation() throws (testing::TestError) {
33	+	remote::initialize(&mut STORE, 3);
34	+	let mut ids: [abi::Ref; 16] = undefined;
35	+	for i in 0..16 {
36	+	set ids[i] = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
37	+	}
38	+	let mut full = false;
39	+	try remote::send(&mut STORE, 0, 1, remote::Action::Reschedule) catch error {
40	+	assert error == abi::Error::Busy; set full = true;
41	+	};
42	+	assert full;
43	+	try! remote::cancel(&mut STORE, 0, ids[0]);
44	+	for i in 1..16 {
45	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing queued request";
46	+	assert delivery.id == ids[i];
47	+	assert remote::acknowledge(&mut STORE, 1, delivery.id);
48	+	try! remote::release(&mut STORE, 0, delivery.id);
49	+	}
50	+	assert try! remote::take(&mut STORE, 1) == nil;
51	+	for i in 0..40 {
52	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
53	+	let delivery = try! remote::take(&mut STORE, 1) else panic "request reuse";
54	+	assert delivery.id == id and id <> ids[0];
55	+	assert not remote::acknowledge(&mut STORE, 1, ids[0]);
56	+	assert remote::acknowledge(&mut STORE, 1, id);
57	+	try! remote::release(&mut STORE, 0, id);
58	+	}
59	+	}
60	+
61	+	/// Completed records retain capacity until their source releases them.
62	+	@test unsafe fn retainedCapacity() throws (testing::TestError) {
63	+	remote::initialize(&mut STORE, 3);
64	+	let mut ids: [abi::Ref; limits::REMOTE_REQUESTS] = undefined;
65	+	for i in 0..limits::REMOTE_REQUESTS {
66	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
67	+	set ids[i] = id;
68	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing request";
69	+	assert delivery.id == id;
70	+	assert remote::acknowledge(&mut STORE, 1, id);
71	+	}
72	+	let mut full = false;
73	+	try remote::send(&mut STORE, 0, 1, remote::Action::Reschedule) catch error {
74	+	assert error == abi::Error::Busy; set full = true;
75	+	};
76	+	assert full;
77	+	for id in &ids[..] {
78	+	try! remote::release(&mut STORE, 0, id);
79	+	}
80	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
81	+	try! remote::cancel(&mut STORE, 0, id);
82	+	}
83	+
84	+	/// A cancelled queue reference cannot deliver a reused slot generation.
85	+	@test unsafe fn cancelledGeneration() throws (testing::TestError) {
86	+	remote::initialize(&mut STORE, 3);
87	+	let stale = try! remote::send(&mut STORE, 0, 1, remote::Action::Reschedule);
88	+	try! remote::cancel(&mut STORE, 0, stale);
89	+	let live = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
90	+	assert live.index == stale.index and live <> stale;
91	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing reused request";
92	+	assert delivery.id == live and delivery.request.action == remote::Action::InstructionSync;
93	+	assert try! remote::take(&mut STORE, 1) == nil;
94	+	assert not remote::acknowledge(&mut STORE, 1, stale);
95	+	let mut cancelDenied = false;
96	+	try remote::cancel(&mut STORE, 0, live) catch error {
97	+	assert error == abi::Error::Busy; set cancelDenied = true;
98	+	};
99	+	assert cancelDenied;
100	+	let mut releaseDenied = false;
101	+	try remote::release(&mut STORE, 0, live) catch error {
102	+	assert error == abi::Error::Busy; set releaseDenied = true;
103	+	};
104	+	assert releaseDenied;
105	+	assert remote::acknowledge(&mut STORE, 1, live);
106	+	try! remote::release(&mut STORE, 0, live);
107	+	}
108	+
109	+	/// Invalid routes and premature release preserve all queued work.
110	+	@test unsafe fn invalidRoute() throws (testing::TestError) {
111	+	remote::initialize(&mut STORE, 1);
112	+	for target in &[1, limits::HARTS] {
113	+	let mut denied = false;
114	+	try remote::send(&mut STORE, 0, target, remote::Action::Reschedule) catch error {
115	+	assert error == abi::Error::InvalidArg; set denied = true;
116	+	};
117	+	assert denied;
118	+	}
119	+	let mut sourceDenied = false;
120	+	try remote::send(&mut STORE, 1, 0, remote::Action::Reschedule) catch error {
121	+	assert error == abi::Error::InvalidArg; set sourceDenied = true;
122	+	};
123	+	assert sourceDenied;
124	+	let id = try! remote::send(&mut STORE, 0, 0, remote::Action::Reschedule);
125	+	let mut releaseDenied = false;
126	+	try remote::release(&mut STORE, 0, id) catch error {
127	+	assert error == abi::Error::Busy; set releaseDenied = true;
128	+	};
129	+	assert releaseDenied;
130	+	assert not remote::acknowledge(&mut STORE, 0, id);
131	+	let delivery = try! remote::take(&mut STORE, 0) else panic "missing local request";
132	+	assert delivery.id == id;
133	+	assert remote::acknowledge(&mut STORE, 0, id);
134	+	try! remote::release(&mut STORE, 0, id);
135	+	assert try! remote::take(&mut STORE, 0) == nil;
136	+	}
137	+
138	+	/// Replies become observable only after the target acknowledges completion.
139	+	@test unsafe fn replies() throws (testing::TestError) {
140	+	remote::initialize(&mut STORE, 3);
141	+	let id = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
142	+	let delivery = try! remote::take(&mut STORE, 1) else panic "missing reply request";
143	+	assert delivery.id == id;
144	+	remote::respond(&mut STORE, 1, id, remote::Reply::Value(42));
145	+	let mut pending = false;
146	+	try remote::result(&STORE, 0, id) catch error {
147	+	assert error == abi::Error::Busy; set pending = true;
148	+	};
149	+	assert pending;
150	+	assert remote::acknowledge(&mut STORE, 1, id);
151	+	assert try! remote::result(&STORE, 0, id) == 42;
152	+	try! remote::release(&mut STORE, 0, id);
153	+	let failed = try! remote::send(&mut STORE, 0, 1, remote::Action::InstructionSync);
154	+	let taken = try! remote::take(&mut STORE, 1) else panic "missing failure request";
155	+	assert taken.id == failed;
156	+	remote::respond(&mut STORE, 1, failed, remote::Reply::Error(abi::Error::BadHandle));
157	+	assert remote::acknowledge(&mut STORE, 1, failed);
158	+	let mut rejected = false;
159	+	try remote::result(&STORE, 0, failed) catch error {
160	+	assert error == abi::Error::BadHandle; set rejected = true;
161	+	};
162	+	assert rejected;
163	+	try! remote::release(&mut STORE, 0, failed);
164	+	}
165	+
166	+	/// Batch failure and cancellation leave every mailbox unchanged.
167	+	@test unsafe fn batchReservation() throws (testing::TestError) {
168	+	remote::initialize(&mut STORE, 7);
169	+	for i in 0..limits::REMOTE_PER_HART {
170	+	let id = try! remote::send(&mut STORE, 0, 2, remote::Action::Reschedule);
171	+	}
172	+	assert batchFull() and STORE.queues[1].count == 0;
173	+	let pending = try! remote::reserve(&mut STORE, 0, 2);
174	+	assert STORE.queues[1].count == 0;
175	+	remote::cancelReservation(&mut STORE, pending);
176	+	assert STORE.queues[1].count == 0;
177	+	let mut live: u32 = 0;
178	+	for slot in &STORE.slots[..] {
179	+	assert slot.state <> slots::State::Reserved;
180	+	if slot.state == slots::State::Live {
181	+	set live += 1;
182	+	}
183	+	}
184	+	assert live == limits::REMOTE_PER_HART;
185	+	for i in 0..limits::REMOTE_PER_HART {
186	+	let delivery = try! remote::take(&mut STORE, 2) else panic "full mailbox lost request";
187	+	assert remote::acknowledge(&mut STORE, 2, delivery.id);
188	+	try! remote::release(&mut STORE, 0, delivery.id);
189	+	}
190	+	let reserved = try! remote::reserve(&mut STORE, 0, 6);
191	+	let batch = remote::publish(&mut STORE, reserved, remote::Action::Reschedule);
192	+	assert batch.source == 0 and batch.targets == 6;
193	+	for target in 1..3 {
194	+	let delivery = try! remote::take(&mut STORE, target) else panic "missing batch request";
195	+	assert delivery.id == batch.ids[target];
196	+	assert remote::acknowledge(&mut STORE, target, delivery.id);
197	+	try! remote::release(&mut STORE, 0, delivery.id);
198	+	}
199	+	}
200	+
201	+	/// Cancel an unexpected reservation so every linear result is consumed.
202	+	unsafe fn batchFull() -> bool {
203	+	let pending = try remote::reserve(&mut STORE, 0, 6) catch error {
204	+	assert error == abi::Error::Busy; return true;
205	+	};
206	+	remote::cancelReservation(&mut STORE, pending);
207	+	return false;
208	+	}

kernel/kernel/tests/timers.rad +32 -9

use kernel::slots;
use kernel::domains;
use kernel::events;
use kernel::timers;
use kernel::limits;
use kernel::budgets;

/// Timer records under test.
unsafe static TIMERS: timers::Store = undefined;
/// Domain metadata and trusted notification state.
unsafe static DOMAINS: domains::Store = undefined;
/// CPU authority used to identify remote notification targets.
unsafe static BUDGETS: budgets::Store = undefined;
/// Receiver ring and private metadata.
unsafe static MEMORY: domains::Memory = undefined;

/// Open an active domain queue without assigning CPU authority.
unsafe fn initialize() -> abi::Ref {
    timers::initialize(&mut TIMERS);
    budgets::initialize(&mut BUDGETS);
    domains::initialize(&mut DOMAINS);
    let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
    let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
    set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
    set DOMAINS.records[owner.index].memory = &mut MEMORY;
    try! events::open(&mut DOMAINS.events, owner, &mut MEMORY.ring);
    return owner;
}

/// A timeout reports the hart of a receiver whose context moved before delivery.
@test unsafe fn remoteReceiver() throws (testing::TestError) {
    let owner = initialize();
    set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };
    let context = abi::Ref { index: 0, generation: 1 };
    set DOMAINS.contexts[0].owner = owner;
    set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
    set BUDGETS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
    set BUDGETS.windows[0] = budgets::Window {
        hart: 1, start: 10, end: 100, owner, handle: abi::Handle(0), context,
    };
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 15, 17);
    let result = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15);
    assert result.harts == 2 and result.deadline == 0xffffffffffffffff;
    assert DOMAINS.contexts[0].state == domains::ContextState::Ready;
    let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 16);
    assert again.harts == 0;
}

/// A timeout is delivered once, at its deadline, on its registered hart.
@test unsafe fn boundaries() throws (testing::TestError) {
    let owner = initialize();
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 1, 10, 5, 17);
    assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 15) == 0xffffffffffffffff);
    assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 14) == 15);
    assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15)).deadline == 0xffffffffffffffff);
    assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 14)).deadline == 15);
    assert (try! events::pop(&mut MEMORY.ring) == nil);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 15);
    assert (next == 0xffffffffffffffff);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 15);
    assert (next.deadline == 0xffffffffffffffff);
    let event = try! events::pop(&mut MEMORY.ring) else panic "timeout";
    assert (event.kind == events::TIMEOUT and event.code == 17 and event.value == 0);
    let again = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 16);
    let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 16);
    assert (try! events::pop(&mut MEMORY.ring) == nil);
}

/// Full queues retain due timeouts without requesting another immediate interrupt.
@test unsafe fn saturation() throws (testing::TestError) {

    let mut full = false;
    try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 1, 99) catch error {
        assert (error == abi::Error::Busy); set full = true;
    };
    assert (full);
    assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1) == 0xffffffffffffffff);
    assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1)).deadline == 0xffffffffffffffff);
    for i in 0..events::ORDINARY {
        let event = try! events::pop(&mut MEMORY.ring) else panic "wake";
    }
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 2);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 2);
    for i in 0..limits::TIMEOUTS_PER_HART {
        let event = try! events::pop(&mut MEMORY.ring) else panic "retained timeout";
        assert (event.kind == events::TIMEOUT and event.code == i);
    }
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 2, 1, 100);

    try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0xffffffffffffffff, 1, 1) catch error {
        assert (error == abi::Error::InvalidArg); set rejected += 1;
    };
    try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 5, 1);
    set DOMAINS.records[owner.index].state = domains::Lifecycle::Dead;
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1);
    assert (next == 0xffffffffffffffff and rejected == 2);
    let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1);
    assert (next.deadline == 0xffffffffffffffff and rejected == 2);
    assert (TIMERS.entries[0][0].owner == nil);
}

kernel/kernel/timers.rad +16 -6

use super::abi;
use super::limits;
use super::slots;
use super::domains;
use super::events;
use super::budgets;

/// One armed timeout or retained due notification.
export record Entry: Copy {
    /// Receiver generation, or nil for free capacity.
    owner: ?abi::Ref,

        return;
    }
    throw abi::Error::Busy;
}

/// Deliver due entries and return the next future timeout boundary.
/// Next timer boundary and dispatch notifications from delivered events.
export record Result: Copy {
    /// Earliest future timeout, or the maximum tick when none is armed.
    deadline: u64,
    /// Harts with retained receiver authority that must refresh dispatch.
    harts: u32,
}

/// Deliver due entries and report the next boundary and affected dispatch harts.
/// Full queues retain due entries for the next kernel entry without a timer spin.
export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, hart: u32, now: u64) -> u64 throws (abi::Error) {
export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, windows: &budgets::Store, hart: u32, now: u64) -> Result throws (abi::Error) {
    if hart >= limits::HARTS {
        throw abi::Error::InvalidArg;
    }
    let mut next: u64 = 0xffffffffffffffff;
    let mut result = Result { deadline: 0xffffffffffffffff, harts: 0 };
    for i in 0..limits::TIMEOUTS_PER_HART {
        let entry = store.entries[hart][i];
        let owner = entry.owner else {
            continue;
        };

            or receivers.records[owner.index].state == domains::Lifecycle::Dead {
            set store.entries[hart][i].owner = nil;
            continue;
        }
        if entry.deadline > now {
            if entry.deadline < next {
                set next = entry.deadline;
            if entry.deadline < result.deadline {
                set result.deadline = entry.deadline;
            }
            continue;
        }
        if try domains::notify(receivers, owner, events::Event { kind: events::TIMEOUT, reserved: 0, code: entry.token, value: 0 }) {
            set store.entries[hart][i].owner = nil;
            set result.harts |= budgets::wakeTargets(windows, receivers, owner, now);
        }
    }
    return next;
    return result;
}

test/smp/kernel/dispatchcheck.rad +189 -6

use kernel::domains;
use kernel::budgets;
use kernel::dispatch;
use kernel::boot;
use kernel::sync;
use kernel::remote;
use kernel::calls;
use kernel::events;
use kernel::timers;
use kernel::dispatchinput;

/// Bootstrap resource authority.
unsafe static TABLE: capability::Table = undefined;
/// Release/acquire barrier after fixture publication.

unsafe static USER: abi::Ref = undefined;
/// Per-hart user execution contexts.
unsafe static CONTEXTS: [abi::Ref; 8] = undefined;
/// Mapped progress counters in the user's private state.
static COUNTERS: u64 = 0;
/// Per-context migration observations in the user's private state.
static MIGRATIONS: u64 = 0;
/// Shared handshake for the independently activated context on hart one.
static WAKE_STATE: u64 = 0;
/// Bootstrap management capability for the pending idle-wakeup domain.
unsafe static IDLE_HANDLE: abi::Handle = undefined;
/// Context whose Wait call must release the remote hart.
unsafe static IDLE_CONTEXT: abi::Ref = undefined;
/// Validated initial stack pointer for the idle-wakeup domain.
static IDLE_STACK: u64 = 0;
/// Generation of the idle-wakeup domain.
unsafe static IDLE_DOMAIN: abi::Ref = undefined;
/// Completion checker's live domain generation.
unsafe static CHECKER: abi::Ref = undefined;
/// Checker's authority to activate and notify the idle-wakeup domain.
unsafe static RECEIVER: abi::Handle = undefined;
/// Unbound authority reserved for the idle-wakeup check on hart one.
unsafe static IDLE_WINDOW: abi::Handle = undefined;
/// Validated physical memory mapping.
fn memory(address: u64) -> *mut u8;
/// Current kernel package-state table.
fn kernelGp() -> u64;
/// M-mode entry of the completion checker.

    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let cap = try! capability::get(&TABLE, handle);
    set USER = cap.object;
    let mut user = try! domains::get(&domains::STORE, USER);
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE),
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),
    });
    let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);
    let case shared::Target::Function(entry) = target else panic "worker entry";
    let counterTarget = try! registry::exported(&registry::STORE, user.image, &"spin::count"[..]);
    let case shared::Target::Data(data) = counterTarget else panic "counter state";
    set COUNTERS = user.graph.table[data.slot] + data.offset as u64;
    set COUNTERS = dataAddress(&user, &"spin::count"[..]);
    set MIGRATIONS = dataAddress(&user, &"spin::migrations"[..]);
    let notification = @sliceOf(memory(dataAddress(&user, &"spin::notification"[..])), 8);
    for i in 0..8 {
        set notification[i] = ((*self) >> (i as u64 * 8)) as u8;
    }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
            continue;
        }
        let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);

        }
    }
    let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let checkerCap = try! capability::get(&TABLE, checkerHandle);
    let checker = try! domains::get(&domains::STORE, checkerCap.object);
    set CHECKER = checkerCap.object;
    set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;
    set domains::STORE.contexts[checker.initial.index].frame.pc = completion();
    set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;
    set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;
    set domains::STORE.contexts[checker.initial.index].frame.registers[3] = kernelGp();
    idleSetup(authority, image);
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let start = clock + 1000000;
    let mut harts: u64 = 0;
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
            set harts += 1;
        }
    }
    let end = start + 10000000 * harts * harts;
    let migrationEnd = end + 10000000 * harts * harts;
    let windowBase = dataAddress(&user, &"spin::windows"[..]);
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
            continue;
        }
        let budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);
        let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);
        let checking = try! budgets::split(&mut budgets::STORE, &mut TABLE, final, migrationEnd, clock);
        let future = try! budgets::split(&mut budgets::STORE, &mut TABLE, checking, migrationEnd + 10000000000, clock);
        let transferred = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut user.memory.table,
            future, budgets::DEFAULT_RIGHTS as u64, clock);
        let caller = (hart as u64 + harts - 1) % harts;
        let output = @sliceOf(memory(windowBase + caller * 8), 8);
        for i in 0..8 {
            set output[i] = ((*transferred) >> (i as u64 * 8)) as u8;
        }
        let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
            budgets::Binding { budget, domain: handle, context: CONTEXTS[hart] }, clock);
        let rotated = ((hart as u64 + 1) % harts) as u32;
        let moved = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
            budgets::Binding { budget: final, domain: handle, context: CONTEXTS[rotated] }, clock);
        if hart == 0 {
            let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
                budgets::Binding { budget: final, domain: checkerHandle, context: checker.initial }, clock);
                budgets::Binding { budget: checking, domain: checkerHandle, context: checker.initial }, clock);
        }
        if hart == 1 {
            set IDLE_WINDOW = checking;
        }
    }
}

/// Resolve one exported private-data address in the fixture's user instance.
unsafe fn dataAddress(user: &domains::Domain, name: &[u8]) -> u64 {
    let target = try! registry::exported(&registry::STORE, user.image, name);
    let case shared::Target::Data(data) = target else panic "fixture data export";
    return user.graph.table[data.slot] + data.offset as u64;
}

/// Prepare a pending domain and its stack for activation on an idle hart.
unsafe fn idleSetup(authority: abi::Handle, image: abi::Handle) {
    set IDLE_HANDLE = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
        &registry::STORE, &mut TABLE, authority, image);
    let permit = try! capability::get(&TABLE, IDLE_HANDLE);
    set IDLE_DOMAIN = permit.object;
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    set IDLE_CONTEXT = domain.initial;
    set WAKE_STATE = dataAddress(&domain, &"spin::wakeState"[..]);
    sync::storeRelease(memory(dataAddress(&domain, &"spin::idle"[..])) as *mut u64, 1);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut domain.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let permission = try! capability::get(&TABLE, stack);
    let page = try! pages::get(&pages::STORE, permission.object);
    set IDLE_STACK = page.base + 4096;
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE),
    });
}

/// Check shared progress and context isolation after the user windows finish.
export unsafe fn verify() {
    let guard = sync::enter();
    let mut counts: [u64; 8] = [0; 8];
    let mut notifications: u32 = 0;
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
            set notifications += 1;
        }
    }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
            continue;
        }
        let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);
        set counts[hart] = value;
        let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);
        assert context.owner == USER and (context.frame.status & 0x1800) == 0;
        assert context.hart == nil or context.hart == hart;
        let destination = (hart + notifications - 1) % notifications;
        assert context.hart == nil or context.hart == destination;
        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;
        assert context.pending == nil;
    }
    assert domains::STORE.events.queues[USER.index].tail == notifications;
    sync::leave(guard);
    for byte in "smp completed transactions:" {
        put(byte);
    }
    for hart in 0..8 {

    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
            assert counts[hart] == 16;
        }
    }
    for target in 0..8 {
        if (boot::PLATFORM.harts & (1 << target)) == 0 {
            continue;
        }
        for action in &[remote::Action::Reschedule, remote::Action::InstructionSync] {
            let posting = sync::enter();
            let id = try! dispatch::request(target, action);
            sync::leave(posting);
            let mut acknowledged = false;
            for attempt in 0..100000 {
                let guard = sync::enter();
                if try! remote::completed(&remote::STORE, 0, id) {
                    try! remote::release(&mut remote::STORE, 0, id);
                    set acknowledged = true;
                }
                sync::leave(guard);
                if acknowledged {
                    break;
                }
            }
            assert acknowledged;
        }
    }
    for byte in "smp request acknowledgements passed\n" {
        put(byte);
    }
    if (boot::PLATFORM.harts & 2) <> 0 {
        idleWakeup();
    }
    let maximum = sync::maximum();
    assert maximum > 0 and maximum < 1000000;
    for byte in "smp metadata instructions: 0x" {
        put(byte);
    }
    number(maximum);
    put(10);
    finish();
}

/// Resume an actual U-mode Wait call through an IPI to an idle remote hart.
unsafe fn idleWakeup() {
    let guard = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let handle = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
        budgets::Binding { budget: IDLE_WINDOW, domain: IDLE_HANDLE, context: IDLE_CONTEXT }, clock);
    let request = try! dispatch::request(1, remote::Action::Reschedule);
    sync::leave(guard);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let activated = try! calls::synchronized(CHECKER, 21, &[*RECEIVER, IDLE_STACK, 0, 0], clock);
    assert activated == 0 and waiting();
    let now = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    let result = try! calls::synchronized(CHECKER, 50, &[*RECEIVER, 0x55, 0, 0], now);
    assert result == 0;
    let mut resumed = false;
    for attempt in 0..100000 {
        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 3 {
            set resumed = true;
            break;
        }
    }
    assert resumed;
    for byte in "smp idle-hart activation and wakeup passed\n" {
        put(byte);
    }
    remoteTimeout();
}

/// Deliver a timeout from hart zero to a waiting user context on idle hart one.
unsafe fn remoteTimeout() {
    let guard = sync::enter();
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let event = try! events::pop(&mut domain.memory.ring) else panic "missing wakeup";
    assert event.kind == events::WAKEUP and event.code == 0x55;
    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 4);
    sync::leave(guard);
    assert waiting();
    let posting = sync::enter();
    let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
    try! timers::request(&mut timers::STORE, &domains::STORE, IDLE_DOMAIN, 0, clock, 100000, 0x66);
    let request = try! dispatch::request(0, remote::Action::Reschedule);
    sync::leave(posting);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let mut resumed = false;
    for attempt in 0..100000 {
        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 6 {
            set resumed = true;
            break;
        }
    }
    assert resumed;
    for byte in "smp remote timeout passed\n" {
        put(byte);
    }
}

/// Confirm the waiting context has released hart ownership before its wakeup.
unsafe fn waiting() -> bool {
    for attempt in 0..100000 {
        let guard = sync::enter();
        let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);
        let stopped = context.state == domains::ContextState::Waiting and context.hart == nil;
        sync::leave(guard);
        if stopped {
            return true;
        }
    }
    return false;
}

/// Print one full-width hexadecimal diagnostic value.
fn number(value: u64) {
    let digits = "0123456789abcdef";
    for i in 0..16 {
        put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]);

test/smp/spin.rad +45 -0

export mod sys;

/// Completed allocation transactions, indexed by the executing hart.
export static count: [u64; 8] = [0; 8];

/// Future budget handles assigned to callers on the preceding online hart.
export static windows: [u64; 8] = [0; 8];

/// Capability used to notify the shared domain from each online hart.
export static notification: u64 = 0;

/// Observed destination hart plus one after each context changes budget.
export static migrations: [u64; 8] = [0; 8];

/// Wait handshake: 2/3 bracket wakeup; 4 requests a second wait, bracketed by 5/6.
export static wakeState: u64 = 0;

/// Nonzero selects the independently activated idle-wakeup context.
export static idle: u64 = 0;

/// Check hart identity and shared-table allocation through repeated direct calls.
fn run(hart: u64) {
    let original = sys::currentContext();
    assert original.hart == hart;
    let budget = abi::Handle(windows[hart as u32]);
    let before = try! sys::queryBudget(budget);
    let right = try! sys::budgetSplit(budget, before.start + 1000);
    let leftInfo = try! sys::queryBudget(budget);
    let rightInfo = try! sys::queryBudget(right);
    assert leftInfo.start == before.start and leftInfo.end == rightInfo.start;
    assert rightInfo.end == before.end and rightInfo.hart == before.hart;
    let merged = try! sys::budgetMerge(budget, right);
    assert merged == budget;
    let after = try! sys::queryBudget(merged);
    assert after.start == before.start and after.end == before.end;
    try! sys::capabilityDrop(merged);
    try! sys::wakeup(abi::Handle(notification), hart as u32);
    for i in 0..16 {
        let page = try! sys::pageAllocate(abi::Handle(0), 1);
        let info = sys::queryPage(page);
        assert info.count == 1 and info.base <> 0;
        try! sys::capabilityDrop(page);
        let current = sys::currentContext();
        assert current.context == original.context and current.hart == hart;
        set count[hart as u32] += 1;
    }
    while true {
        let current = sys::currentContext();
        assert current.context == original.context;
        if current.budget <> original.budget {
            set migrations[hart as u32] = current.hart + 1;
        }
    }
}

/// Start an additional context with its physical hart index in validated memory.
export fn worker(args: *u64, size: u64) {

    run(*args);
}

/// The initial context owns hart zero's window.
@default fn main(env: *opaque) {
    if idle <> 0 {
        set wakeState = 2;
        sys::wait();
        set wakeState = 3;
        while wakeState <> 4 {
        }
        set wakeState = 5;
        sys::wait();
        set wakeState = 6;
        while true {
        }
    }
    run(0);
}

336	371		sync::leave(publicationGuard);
337	372		return *result;
338	373		}
339	374
340	375		/// Execute a call with short metadata transactions and private bulk initialization.
341		-	/// The running context keeps its domain storage live until dispatch releases it.
	376	+	/// Context ownership and pending calls retain storage across dispatch boundaries.
342	377		export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
343	378		assert arguments.len == 4;
	379	+	if operation == 21 or operation == 50 {
	380	+	return try runtimeReady(owner, operation, arguments, now);
	381	+	}
344	382		let guard = sync::enter();
345	383		let mut domain = try domains::get(&domains::STORE, owner) catch error {
346	384		sync::leave(guard);
347	385		throw error;
348	386		};
349	387		if domain.state <> domains::Lifecycle::Active {
350	388		sync::leave(guard);
351	389		throw abi::Error::BadHandle;
352	390		}
	391	+	let target = try budgetHart(&domain.memory.table, operation, arguments) catch error {
	392	+	sync::leave(guard);
	393	+	throw error;
	394	+	};
	395	+	if let destination = target {
	396	+	let source = dispatch::hart();
	397	+	if destination <> source {
	398	+	let call = remote::Call { owner, context: abi::reference(dispatch::current().context), operation,
	399	+	arguments: [arguments[0], arguments[1], arguments[2], arguments[3]] };
	400	+	let request = try dispatch::request(destination, remote::Action::Budget(call)) catch error {
	401	+	sync::leave(guard); throw error;
	402	+	};
	403	+	assert domains::STORE.contexts[call.context.index].pending == nil;
	404	+	set domains::STORE.contexts[call.context.index].pending = request;
	405	+	sync::leave(guard);
	406	+	return try dispatch::awaitRequest(source, request);
	407	+	}
	408	+	}
353	409		let handle = abi::Handle(arguments[0]);
354	410		match operation {
355	411		case 20 => {
356	412		let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
357	413		&mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error {

431	487		sync::leave(guard); return result;
432	488		},
433	489		}
434	490		}
435	491
	492	+	/// Reserve remote dispatch capacity before activation or event publication.
	493	+	unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
	494	+	if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) {
	495	+	throw abi::Error::InvalidArg;
	496	+	}
	497	+	let guard = sync::enter();
	498	+	let domain = try domains::get(&domains::STORE, owner) catch error {
	499	+	sync::leave(guard);
	500	+	throw error;
	501	+	};
	502	+	if domain.state <> domains::Lifecycle::Active {
	503	+	sync::leave(guard);
	504	+	throw abi::Error::BadHandle;
	505	+	}
	506	+	let handle = abi::Handle(arguments[0]);
	507	+	let rights = abi::EXECUTE if operation == 21 else abi::WAKE;
	508	+	let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
	509	+	sync::leave(guard); throw error;
	510	+	};
	511	+	let source = dispatch::hart();
	512	+	let context = abi::reference(dispatch::current().context);
	513	+	let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
	514	+	let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error {
	515	+	sync::leave(guard);
	516	+	throw error;
	517	+	};
	518	+	let result = try invoke(owner, operation, arguments, now) catch error {
	519	+	remote::cancelReservation(&mut remote::STORE, pending);
	520	+	sync::leave(guard); throw error;
	521	+	};
	522	+	let batch = remote::publish(&mut remote::STORE, pending, remote::Action::Ready(context));
	523	+	dispatch::startReady(context, &batch);
	524	+	sync::leave(guard);
	525	+	dispatch::finishReady(context, &batch);
	526	+	return 0;
	527	+	}
	528	+
436	529		/// Complete a user ecall through the supplied execution boundary.
437	530		export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
438	531		execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
439	532		{
440	533		assert trap::fromUser(frame) and frame.cause == 8;

345	347		set frame.registers[3] = domain.graph.table.ptr as u64;
346	348		set frame.registers[10] = start.args;
347	349		set frame.registers[11] = start.size;
348	350		let object = slots::reference(&slot);
349	351		set store.contexts[object.index] = Context {
350		-	owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
	352	+	owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames, kernelStack, userStack, frame,
351	353		};
352	354		match slot {
353	355		case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
354	356		object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
355	357		}),

571	573		// MPIE enables interrupts after the first return into U-mode.
572	574		let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
573	575		set frame.registers[1] = returnAddress();
574	576		set frame.registers[3] = graph.table.ptr as u64;
575	577		set store.contexts[initial.index] = Context {
576		-	owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
	578	+	owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
577	579		userStack: range::Range { start: 0, end: 0 }, frame,
578	580		};
579	581		set store.records[object.index] = Domain {
580	582		state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
581	583		allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,

21	21		export mod instances;
22	22		export mod domains;
23	23		export mod budgets;
24	24		export mod dispatch;
25	25		export mod timers;
	26	+	export mod remote;
26	27		export mod calls;
27	28		export mod sys;
28	29		export mod boot;
29	30		@test export mod tests;

10	10		use super::registry;
11	11		use super::domains;
12	12		use super::budgets;
13	13		use super::dispatch;
14	14		use super::timers;
	15	+	use super::remote;
15	16		use std::arch::rv64::shared::catalog;
16	17
17	18		/// Platform data published by hart zero before secondary initialization.
18	19		export unsafe static PLATFORM: platform::Platform = undefined;
19	20		/// Release/acquire publication flag for PLATFORM.

23	24		/// Trap anchors owned by their physical harts.
24	25		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
25	26
26	27		/// Handle timer boundaries and user calls, and stop on unsupported traps.
27	28		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
28		-	if trap::classify(frame.cause) == trap::Cause::Timer {
	29	+	let cause = trap::classify(frame.cause);
	30	+	if cause == trap::Cause::Timer or cause == trap::Cause::Software {
29	31		dispatch::interrupt(frame, hart);
30	32		return;
31	33		}
32		-	if trap::classify(frame.cause) == trap::Cause::UserCall and trap::fromUser(frame) {
	34	+	if cause == trap::Cause::UserCall and trap::fromUser(frame) {
33	35		dispatch::call(frame, hart);
34	36		return;
35	37		}
36	38		print("kernel: unexpected trap\n");
37	39		trap::halt();

83	85		try! platform::protect(&mut PLATFORM, treeRange);
84	86		try! pages::initialize(&mut pages::STORE, &PLATFORM);
85	87		domains::initialize(&mut domains::STORE);
86	88		budgets::initialize(&mut budgets::STORE);
87	89		timers::initialize(&mut timers::STORE);
	90	+	remote::initialize(&mut remote::STORE, PLATFORM.harts);
88	91		assert entryCount == 2;
89	92		registry::initialize(&mut registry::STORE);
90	93		let bootCatalog = @sliceOf(entries, entryCount);
91	94		try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
92	95		let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";

295	295		/// Test half-open execution authority on the selected hart at the current tick.
296	296		export fn active(store: &Store, object: abi::Ref, hart: u32, now: u64) -> bool throws (abi::Error) {
297	297		let window = try get(store, object);
298	298		return window.context <> nil and window.hart == hart and window.start <= now and now < window.end;
299	299		}
	300	+
	301	+	/// Find all harts with retained authority for a receiver's live contexts.
	302	+	export fn wakeTargets(windows: &Store, contexts: &domains::Store, receiver: abi::Ref, now: u64) -> u32 {
	303	+	let mut targets: u32 = 0;
	304	+	for i in 0..limits::BUDGETS {
	305	+	if windows.slots[i].state <> slots::State::Live {
	306	+	continue;
	307	+	}
	308	+	let window = windows.windows[i];
	309	+	if window.owner <> receiver or window.end <= now {
	310	+	continue;
	311	+	}
	312	+	let context = window.context else {
	313	+	continue;
	314	+	};
	315	+	if not slots::matches(&contexts.contextSlots[..], context, slots::State::Live) {
	316	+	continue;
	317	+	}
	318	+	if contexts.contexts[context.index].owner <> receiver
	319	+	or contexts.contexts[context.index].state == domains::ContextState::Stopped {
	320	+	continue;
	321	+	}
	322	+	set targets \|= 1 << window.hart;
	323	+	}
	324	+	return targets;
	325	+	}

13	13		use super::transactions;
14	14		use super::loader;
15	15		use super::dispatch;
16	16		use super::timers;
17	17		use super::sync;
	18	+	use super::remote;
	19	+
	20	+	/// Resolve the owning hart for an operation that can change CPU authority.
	21	+	unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
	22	+	match operation {
	23	+	case 11, 12, 70, 71, 73 => {
	24	+	},
	25	+	else => return nil,
	26	+	}
	27	+	let entry = try capability::get(table, abi::Handle(arguments[0]));
	28	+	if entry.kind <> abi::Kind::Budget {
	29	+	return nil;
	30	+	}
	31	+	return (try budgets::get(&budgets::STORE, entry.object)).hart;
	32	+	}
	33	+
	34	+	/// Execute a queued budget change at its owning hart's dispatch boundary.
	35	+	export unsafe fn remoteBudget(call: remote::Call, hart: u32, now: u64) -> remote::Reply {
	36	+	let domain = try domains::get(&domains::STORE, call.owner) catch error {
	37	+	return remote::Reply::Error(error);
	38	+	};
	39	+	if domain.state <> domains::Lifecycle::Active {
	40	+	return remote::Reply::Error(abi::Error::BadHandle);
	41	+	}
	42	+	let target = try budgetHart(&domain.memory.table, call.operation, &call.arguments[..]) catch error {
	43	+	return remote::Reply::Error(error);
	44	+	};
	45	+	if target <> hart {
	46	+	return remote::Reply::Error(abi::Error::BadHandle);
	47	+	}
	48	+	let result = try invoke(call.owner, call.operation, &call.arguments[..], now) catch error {
	49	+	return remote::Reply::Error(error);
	50	+	};
	51	+	return remote::Reply::Value(result);
	52	+	}
18	53
19	54		/// Validate resident object generations for capabilities without page ownership.
20	55		unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
21	56		match entry.kind {
22	57		case abi::Kind::Domain, abi::Kind::Events => {

9	9		use super::trap;
10	10		use super::range;
11	11		use super::calls;
12	12		use super::timers;
13	13		use super::sync;
	14	+	use super::remote;
14	15
15	16		/// Dispatch ownership and idle state retained by one physical hart.
16	17		export record State: Copy {
17	18		/// Physical hart whose windows this state selects.
18	19		hart: u32,

30	31		export record Timer: Copy {
31	32		/// Shared monotonic mtime register.
32	33		clock: u64,
33	34		/// This hart's mtimecmp register.
34	35		compare: u64,
	36	+	/// This hart's machine software-interrupt word.
	37	+	software: u64,
35	38		}
36	39
37	40		/// Timer and dispatch state owned by one physical hart.
38	41		record Runtime: Copy {
39	42		/// Current context and retained idle frame.

47	50
48	51		/// Read the executing physical hart identifier.
49	52		export fn hart() -> u32;
50	53		/// Read the current machine stack pointer.
51	54		fn stackPointer() -> u64;
	55	+	/// Trap anchor of the hart currently executing this continuation.
	56	+	fn localAnchor() -> *mut trap::Hart;
52	57		/// Address of the M-mode idle loop.
53	58		fn idleAddress() -> u64;
54		-	/// Enable machine timer delivery while global interrupts remain disabled.
	59	+	/// Enable timer and software interrupt delivery with global interrupts disabled.
55	60		fn enable();
56	61
57	62		/// Initialize this hart's idle frame and validated timer before installing dispatch.
58	63		export unsafe fn initialize(machine: &platform::Platform, id: u32, kernelGp: u64) throws (abi::Error) {
59	64		let device = try timer(machine, id);

76	81		assert id < limits::HARTS;
77	82		let stack = HARTS[id].state.stack;
78	83		let pointer = stackPointer();
79	84		assert pointer >= stack.start and pointer < stack.end;
80	85		let guard = sync::enter();
	86	+	writeSoftware(HARTS[id].timer.software, 0);
	87	+	let previous = HARTS[id].state.current;
	88	+	save(&mut HARTS[id].state, &mut domains::STORE, frame);
	89	+	let mut deliveries: [remote::Delivery; limits::REMOTE_PER_HART] = undefined;
	90	+	let mut count: u32 = 0;
	91	+	for i in 0..limits::REMOTE_PER_HART {
	92	+	let delivery = try! remote::take(&mut remote::STORE, id) else {
	93	+	break;
	94	+	};
	95	+	match delivery.request.action {
	96	+	case remote::Action::InstructionSync => sync::syncInstructions(),
	97	+	case remote::Action::Reschedule, remote::Action::Ready(_) => {
	98	+	},
	99	+	case remote::Action::Budget(call) => {
	100	+	let reply = calls::remoteBudget(call, id, now(HARTS[id].timer));
	101	+	remote::respond(&mut remote::STORE, id, delivery.id, reply);
	102	+	},
	103	+	}
	104	+	set deliveries[count] = delivery;
	105	+	set count += 1;
	106	+	}
81	107		let clock = now(HARTS[id].timer);
82		-	let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
	108	+	let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
83	109		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
84	110		let mut deadline = choice.deadline;
85		-	if timeout < deadline {
86		-	set deadline = timeout;
	111	+	if timeout.deadline < deadline {
	112	+	set deadline = timeout.deadline;
	113	+	}
	114	+	for target in 0..limits::HARTS {
	115	+	if target <> id and (timeout.harts & (1 << target)) <> 0 {
	116	+	signal(target);
	117	+	}
87	118		}
88	119		arm(HARTS[id].timer, deadline);
	120	+	if previous <> choice.context {
	121	+	if let context = previous {
	122	+	signalContext(context, clock);
	123	+	}
	124	+	}
	125	+	for delivery in &deliveries[..count] {
	126	+	assert remote::acknowledge(&mut remote::STORE, id, delivery.id);
	127	+	if delivery.request.source <> id {
	128	+	signal(delivery.request.source);
	129	+	}
	130	+	if let case remote::Action::Budget(call) = delivery.request.action {
	131	+	signalContext(call.context, clock);
	132	+	}
	133	+	if let case remote::Action::Ready(context) = delivery.request.action {
	134	+	signalContext(context, clock);
	135	+	}
	136	+	}
89	137		sync::leave(guard);
90	138		}
91	139
	140	+	/// Signal an initialized online hart while request metadata is serialized.
	141	+	unsafe fn signal(target: u32) {
	142	+	writeSoftware(HARTS[target].timer.software, 1);
	143	+	}
	144	+
	145	+	/// Notify the current owner or the eligible hart after a context is released.
	146	+	unsafe fn signalContext(context: abi::Ref, now: u64) {
	147	+	if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) {
	148	+	return;
	149	+	}
	150	+	if let owner = domains::STORE.contexts[context.index].hart {
	151	+	signal(owner);
	152	+	return;
	153	+	}
	154	+	for i in 0..limits::BUDGETS {
	155	+	if budgets::STORE.slots[i].state <> slots::State::Live {
	156	+	continue;
	157	+	}
	158	+	let window = budgets::STORE.windows[i];
	159	+	if window.context == context and window.start <= now and now < window.end {
	160	+	signal(window.hart);
	161	+	}
	162	+	}
	163	+	}
	164	+
	165	+	/// Write one naturally aligned CLINT software-interrupt register.
	166	+	fn writeSoftware(address: u64, value: u32);
	167	+
	168	+	/// Post a local action and interrupt its target under metadata serialization.
	169	+	export unsafe fn request(target: u32, action: remote::Action) -> abi::Ref throws (abi::Error) {
	170	+	let id = try remote::send(&mut remote::STORE, hart(), target, action);
	171	+	signal(target);
	172	+	return id;
	173	+	}
	174	+
	175	+	/// Retain the caller and signal every published readiness request under the lock.
	176	+	export unsafe fn startReady(context: abi::Ref, batch: &remote::Batch) {
	177	+	assert domains::STORE.contexts[context.index].pending == nil;
	178	+	for target in 0..limits::HARTS {
	179	+	if (batch.targets & (1 << target)) == 0 {
	180	+	continue;
	181	+	}
	182	+	if domains::STORE.contexts[context.index].pending == nil {
	183	+	set domains::STORE.contexts[context.index].pending = batch.ids[target];
	184	+	}
	185	+	signal(target);
	186	+	}
	187	+	}
	188	+
	189	+	/// Consume every readiness acknowledgement before releasing the caller's storage.
	190	+	export unsafe fn finishReady(context: abi::Ref, batch: &remote::Batch) {
	191	+	for target in 0..limits::HARTS {
	192	+	if (batch.targets & (1 << target)) == 0 {
	193	+	continue;
	194	+	}
	195	+	waitRequest(batch.source, batch.ids[target]);
	196	+	}
	197	+	let guard = sync::enter();
	198	+	assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
	199	+	for target in 0..limits::HARTS {
	200	+	if (batch.targets & (1 << target)) == 0 {
	201	+	continue;
	202	+	}
	203	+	assert try! remote::result(&remote::STORE, batch.source, batch.ids[target]) == 0;
	204	+	try! remote::release(&mut remote::STORE, batch.source, batch.ids[target]);
	205	+	}
	206	+	set domains::STORE.contexts[context.index].pending = nil;
	207	+	sync::leave(guard);
	208	+	}
	209	+
	210	+	/// Wait for an interrupt with global interrupts masked, then enable delivery.
	211	+	fn waitInterrupt();
	212	+
	213	+	/// Release a completed call's context retention and request generation.
	214	+	unsafe fn releaseRequest(source: u32, request: abi::Ref) {
	215	+	if let case remote::Action::Budget(call) = remote::STORE.requests[request.index].action {
	216	+	assert slots::matches(&domains::STORE.contextSlots[..], call.context, slots::State::Live);
	217	+	assert domains::STORE.contexts[call.context.index].pending == request;
	218	+	set domains::STORE.contexts[call.context.index].pending = nil;
	219	+	}
	220	+	try! remote::release(&mut remote::STORE, source, request);
	221	+	}
	222	+
	223	+	/// Await acknowledgement while retaining the record and allowing preemption.
	224	+	unsafe fn waitRequest(source: u32, request: abi::Ref) {
	225	+	let interrupts = sync::maskInterrupts();
	226	+	loop {
	227	+	let guard = sync::enter();
	228	+	let complete = try! remote::completed(&remote::STORE, source, request);
	229	+	sync::leave(guard);
	230	+	if complete {
	231	+	break;
	232	+	}
	233	+	waitInterrupt();
	234	+	sync::maskInterrupts();
	235	+	}
	236	+	sync::restoreInterrupts(interrupts);
	237	+	}
	238	+
	239	+	/// Await a retained reply through a preemptible kernel continuation.
	240	+	export unsafe fn awaitRequest(source: u32, request: abi::Ref) -> u64 throws (abi::Error) {
	241	+	waitRequest(source, request);
	242	+	let guard = sync::enter();
	243	+	let result = try remote::result(&remote::STORE, source, request) catch error {
	244	+	releaseRequest(source, request);
	245	+	sync::leave(guard); throw error;
	246	+	};
	247	+	releaseRequest(source, request);
	248	+	sync::leave(guard); return result;
	249	+	}
	250	+
92	251		/// Read the running context's identity and domain-relative CPU authority.
93	252		export unsafe fn current() -> abi::CurrentContextInfo {
94	253		let id = hart();
95	254		assert id < limits::HARTS;
96	255		let context = HARTS[id].state.current else panic "no current context";

113	272		sync::leave(guard);
114	273		} else {
115	274		sync::leave(guard);
116	275		calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
117	276		}
118		-	interrupt(frame, anchor);
	277	+	interrupt(frame, &mut *localAnchor());
119	278		}
120	279
121	280		/// Enter the first authorized context or the hart's retained idle frame.
122	281		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
123	282		let id = hart();

145	304		}
146	305		if result <> nil or device.memory.end < device.memory.start
147	306		or device.memory.end - device.memory.start < 0xc000 or (device.memory.start & 7) <> 0 {
148	307		throw abi::Error::InvalidArg;
149	308		}
150		-	set result = Timer { clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8 };
	309	+	set result = Timer {
	310	+	clock: device.memory.start + 0xbff8, compare: device.memory.start + 0x4000 + hart as u64 * 8,
	311	+	software: device.memory.start + hart as u64 * 4,
	312	+	};
151	313		}
152	314		let selected = result else {
153	315		throw abi::Error::InvalidArg;
154	316		};
155	317		return selected;
156	318		}
157	319
158	320		/// Read the platform's monotonic clock in mtime ticks.
159		-	export fn now(timer: Timer) -> u64 {
160		-	return read(timer.clock);
161		-	}
	321	+	export fn now(timer: Timer) -> u64 { return read(timer.clock); }
162	322
163	323		/// Program the next boundary while this hart's timer handling is serialized.
164	324		export fn arm(timer: Timer, deadline: u64) {
165	325		write(timer.compare, deadline);
166	326		}

240	400		/// Interrupts remain disabled and shared metadata stays serialized through exchange.
241	401		export fn exchange(state: &mut State, windows: &budgets::Store, contexts: &mut domains::Store,
242	402		frame: &mut trap::Frame, anchor: &mut trap::Hart, now: u64) -> Choice throws (abi::Error)
243	403		{
244	404		let choice = try select(windows, contexts, state.hart, now);
245		-	if let current = state.current {
246		-	assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
247		-	set contexts.contexts[current.index].frame = *frame;
248		-	set contexts.contexts[current.index].hart = nil;
249		-	}
	405	+	save(state, contexts, frame);
250	406		if let next = choice.context {
251	407		let context = contexts.contexts[next.index];
252	408		assert context.kernelStack.start < context.kernelStack.end;
253	409		set *frame = context.frame;
254	410		set anchor.stackBottom = context.kernelStack.start;

261	417		}
262	418		set state.current = choice.context;
263	419		set state.budget = choice.budget;
264	420		return choice;
265	421		}
	422	+
	423	+	/// Retain a context frame before releasing this hart's execution ownership.
	424	+	fn save(state: &mut State, contexts: &mut domains::Store, frame: &trap::Frame) {
	425	+	if let current = state.current {
	426	+	assert slots::matches(&contexts.contextSlots[..], current, slots::State::Live);
	427	+	set contexts.contexts[current.index].frame = *frame;
	428	+	set contexts.contexts[current.index].hart = nil;
	429	+	}
	430	+	set state.current = nil;
	431	+	set state.budget = nil;
	432	+	}

1	1		//! Naturally aligned RV64 CLINT access with device ordering.
2	2		.text;
3	3		.export @kernel::dispatch::read;
4	4		.export @kernel::dispatch::write;
	5	+	.export @kernel::dispatch::writeSoftware;
5	6		.export @kernel::dispatch::hart;
6	7		.export @kernel::dispatch::stackPointer;
	8	+	.export @kernel::dispatch::localAnchor;
7	9		.export @kernel::dispatch::idleAddress;
8	10		.export @kernel::dispatch::enable;
9	11		.export @kernel::dispatch::interrupt;
	12	+	.export @kernel::dispatch::waitInterrupt;
	13	+
	14	+	// A pending enabled interrupt wakes wfi while global delivery is masked.
	15	+	@kernel::dispatch::waitInterrupt
	16	+	wfi;
	17	+	csrsi mstatus 8;
	18	+	ret;
10	19
11	20		// Retain the frame outside the context stack before publishing its release.
12	21		@kernel::dispatch::interrupt
13	22		mv %t0 %a0;
14	23		addi %t1 %a1 64;

35	44		// Dispatch releases context stacks only from the hart's firmware stack.
36	45		@kernel::dispatch::stackPointer
37	46		mv %a0 %sp;
38	47		ret;
39	48
	49	+	// A resumed kernel continuation uses its executing hart's trap storage.
	50	+	@kernel::dispatch::localAnchor
	51	+	csrr %a0 mscratch;
	52	+	ret;
	53	+
40	54		// Saved idle frames enter this loop with machine interrupts enabled by mret.
41	55		@kernel::dispatch::idleAddress
42	56		la %a0 @dispatchIdle;
43	57		ret;
44	58		@dispatchIdle

46	60		j @dispatchIdle;
47	61
48	62		// Global interrupt enable is restored from the selected frame by mret.
49	63		@kernel::dispatch::enable
50	64		csrr %t0 mie;
51		-	ori %t0 %t0 128;
	65	+	ori %t0 %t0 136;
52	66		csrw mie %t0;
53	67		ret;
54	68
55	69		// The validated CLINT mapping contains a naturally aligned mtime register.
56	70		@kernel::dispatch::read

63	77		@kernel::dispatch::write
64	78		fence iorw iorw;
65	79		sd %a1 0(%a0);
66	80		fence iorw iorw;
67	81		ret;
	82	+
	83	+	// CLINT software-interrupt registers are naturally aligned 32-bit words.
	84	+	@kernel::dispatch::writeSoftware
	85	+	fence iorw iorw;
	86	+	sw %a1 0(%a0);
	87	+	fence iorw iorw;
	88	+	ret;

56	56		owner: abi::Ref,
57	57		/// Whether this context can resume.
58	58		state: ContextState,
59	59		/// Hart retaining this context as its current execution owner.
60	60		hart: ?u32,
	61	+	/// Remote call whose retained continuation requires this context's storage.
	62	+	pending: ?abi::Ref,
61	63		/// Physical frames retained until this context stops.
62	64		kernelFrames: frames::Run,
63	65		/// Exclusive stack bounds used by trap entry and suspended M-mode calls.
64	66		kernelStack: range::Range,
65	67		/// Exclusive user stack extent; empty until initial activation.

204	206		let permit = try capability::authority(table, authority, abi::Rights(abi::DESTROY));
205	207		let target = try context(store, permit.object, object);
206	208		if store.records[permit.object.index].initial == object {
207	209		throw abi::Error::Denied;
208	210		}
209		-	if target.hart <> nil {
	211	+	if target.hart <> nil or target.pending <> nil {
210	212		throw abi::Error::Busy;
211	213		}
212	214		try! frames::release(&mut memory.backings.pool, target.kernelFrames);
213	215		try! slots::release(&mut store.contextSlots[..], object);
214	216		}

22	22		export constant CONTEXTS: u32 = 2048;
23	23		/// Maximum hardware threads.
24	24		export constant HARTS: u32 = 8;
25	25		/// One-shot timeouts retained by each hart, including pending delivery.
26	26		export constant TIMEOUTS_PER_HART: u32 = 64;
	27	+	/// Outstanding cross-hart requests, including acknowledgements awaiting release.
	28	+	export constant REMOTE_REQUESTS: u32 = 128;
	29	+	/// Queued remote requests accepted by each hart.
	30	+	export constant REMOTE_PER_HART: u32 = 16;
27	31		/// Maximum resident package records and gp table entries.
28	32		export constant PACKAGES: u32 = 256;
29	33		/// Generation values never wrap.
30	34		export constant LAST_GENERATION: u32 = 0xffffffff;

193	193		let child = slots::reference(&childSlot);
194	194		capability::initialize(&mut MEMORY.table, child);
195	195		let contextSlot = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
196	196		let context = slots::reference(&contextSlot);
197	197		set DOMAINS.contexts[context.index] = domains::Context {
198		-	owner: child, state: domains::ContextState::Ready, hart: nil,
	198	+	owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil,
199	199		kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },
200	200		userStack: range::Range { start: 0, end: 0 },
201	201		frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
202	202		};
203	203		set DOMAINS.records[child.index] = domains::Domain {

10	10		use kernel::slots;
11	11		use kernel::registry;
12	12		use kernel::limits;
13	13		use kernel::trap;
14	14		use kernel::budgets;
	15	+	use kernel::remote;
	16	+
	17	+	/// Deferred budget operations recheck authority and expiration at execution time.
	18	+	@test unsafe fn deferredBudget() throws (testing::TestError) {
	19	+	let owner = initialize();
	20	+	let handle = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 1, 10, 100);
	21	+	let entry = try! capability::get(&MEMORY.table, handle);
	22	+	let call = remote::Call { owner, context: owner, operation: 70, arguments: [*handle, 50, 0, 0] };
	23	+	let wrongHart = calls::remoteBudget(call, 0, 20);
	24	+	assert wrongHart == remote::Reply::Error(abi::Error::BadHandle);
	25	+	remote::initialize(&mut remote::STORE, 3);
	26	+	let request = try! remote::send(&mut remote::STORE, 0, 1, remote::Action::Budget(call));
	27	+	let delivery = try! remote::take(&mut remote::STORE, 1) else panic "missing expired call";
	28	+	assert delivery.id == request;
	29	+	let case remote::Action::Budget(queued) = delivery.request.action else panic "queued budget call";
	30	+	let expired = calls::remoteBudget(queued, 1, 100);
	31	+	assert expired == remote::Reply::Error(abi::Error::Busy);
	32	+	remote::respond(&mut remote::STORE, 1, request, expired);
	33	+	assert remote::acknowledge(&mut remote::STORE, 1, request);
	34	+	assert try! remote::completed(&remote::STORE, 0, request);
	35	+	try! remote::release(&mut remote::STORE, 0, request);
	36	+	let unchanged = try! budgets::get(&budgets::STORE, entry.object);
	37	+	assert unchanged.start == 10 and unchanged.end == 100;
	38	+	let reply = calls::remoteBudget(call, 1, 20);
	39	+	let case remote::Reply::Value(value) = reply else panic "deferred split";
	40	+	let left = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
	41	+	let right = try! budgets::query(&budgets::STORE, &MEMORY.table, abi::Handle(value));
	42	+	assert left.start == 20 and left.end == 50 and right.start == 50 and right.end == 100;
	43	+	let duplicate = calls::remoteBudget(call, 1, 25);
	44	+	assert duplicate == remote::Reply::Error(abi::Error::InvalidArg);
	45	+	let retained = try! budgets::query(&budgets::STORE, &MEMORY.table, handle);
	46	+	assert retained.start == 20 and retained.end == 50;
	47	+	let dropped = try! calls::invoke(owner, 12, &[*handle, 0, 0, 0], 25);
	48	+	assert dropped == 0;
	49	+	assert calls::remoteBudget(call, 1, 25) == remote::Reply::Error(abi::Error::BadHandle);
	50	+	}
15	51
16	52		/// Mapped physical storage for four pages plus alignment padding.
17	53		static RAM: [u64; 2560] = [0; 2560];
18	54		/// Kernel-only metadata for the calling domain.
19	55		unsafe static MEMORY: domains::Memory = undefined;

40	40		let b = try! capability::get(&TABLE, last);
41	41		set BUDGETS.windows[a.object.index].context = owner;
42	42		set BUDGETS.windows[b.object.index].context = owner;
43	43		}
44	44
	45	+	/// Wakeup reaches future windows and excludes expired or stopped contexts.
	46	+	@test unsafe fn wakeTargets() throws (testing::TestError) {
	47	+	initialize();
	48	+	set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
	49	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 1;
	50	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 25) == 1;
	51	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 40) == 0;
	52	+	set DOMAINS.contexts[0].state = domains::ContextState::Stopped;
	53	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
	54	+	set DOMAINS.contexts[0].state = domains::ContextState::Ready;
	55	+	set DOMAINS.contextSlots[0].generation += 1;
	56	+	assert budgets::wakeTargets(&BUDGETS, &DOMAINS, TABLE.owner, 0) == 0;
	57	+	}
	58	+
45	59		/// Dispatch selects only the half-open interval and wakes at the next runnable start.
46	60		@test unsafe fn boundaries() throws (testing::TestError) {
47	61		initialize();
48	62		let before = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
49	63		try testing::expect(before.context == nil and before.deadline == 10);

98	112		set PLATFORM.devices[0] = platform::Device {
99	113		kind: platform::Kind::Clint, memory: range::Range { start: 0x2000000, end: 0x2010000 },
100	114		};
101	115		let first = try! dispatch::timer(&PLATFORM, 0);
102	116		let last = try! dispatch::timer(&PLATFORM, 7);
	117	+	assert first.software == 0x2000000 and last.software == first.software + 28;
103	118		try testing::expect(first.clock == 0x200bff8 and first.compare == 0x2004000);
104	119		try testing::expect(last.clock == first.clock and last.compare == first.compare + 56);
105	120		let mut failures: u32 = 0;
106	121		try dispatch::timer(&PLATFORM, 1) catch err {
107	122		try testing::expect(err == abi::Error::InvalidArg); set failures += 1;

213	213		};
214	214		try! domains::returned(&mut DOMAINS, authority.object, context);
215	215		try testing::expect(DOMAINS.contexts[context.index].state == domains::ContextState::Stopped);
216	216		try testing::expect(DOMAINS.records[authority.object.index].state == domains::Lifecycle::Pending);
217	217		set DOMAINS.contexts[context.index].hart = nil;
	218	+	set DOMAINS.contexts[context.index].pending = abi::Ref { index: 0, generation: 1 };
	219	+	let mut retained = false;
	220	+	try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context) catch error {
	221	+	assert error == abi::Error::Busy; set retained = true;
	222	+	};
	223	+	assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
	224	+	for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
	225	+	assert not PAGES.backings.pool.free[i];
	226	+	}
	227	+	set DOMAINS.contexts[context.index].pending = nil;
218	228		try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context);
219	229		for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
220	230		try testing::expect(PAGES.backings.pool.free[i]);
221	231		}
222	232		try domains::context(&DOMAINS, authority.object, context) catch error {

4	4		use kernel::slots;
5	5		use kernel::domains;
6	6		use kernel::events;
7	7		use kernel::timers;
8	8		use kernel::limits;
	9	+	use kernel::budgets;
9	10
10	11		/// Timer records under test.
11	12		unsafe static TIMERS: timers::Store = undefined;
12	13		/// Domain metadata and trusted notification state.
13	14		unsafe static DOMAINS: domains::Store = undefined;
	15	+	/// CPU authority used to identify remote notification targets.
	16	+	unsafe static BUDGETS: budgets::Store = undefined;
14	17		/// Receiver ring and private metadata.
15	18		unsafe static MEMORY: domains::Memory = undefined;
16	19
17	20		/// Open an active domain queue without assigning CPU authority.
18	21		unsafe fn initialize() -> abi::Ref {
19	22		timers::initialize(&mut TIMERS);
	23	+	budgets::initialize(&mut BUDGETS);
20	24		domains::initialize(&mut DOMAINS);
21	25		let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
22	26		let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
23	27		set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
24	28		set DOMAINS.records[owner.index].memory = &mut MEMORY;
25	29		try! events::open(&mut DOMAINS.events, owner, &mut MEMORY.ring);
26	30		return owner;
27	31		}
28	32
	33	+	/// A timeout reports the hart of a receiver whose context moved before delivery.
	34	+	@test unsafe fn remoteReceiver() throws (testing::TestError) {
	35	+	let owner = initialize();
	36	+	set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };
	37	+	let context = abi::Ref { index: 0, generation: 1 };
	38	+	set DOMAINS.contexts[0].owner = owner;
	39	+	set DOMAINS.contexts[0].state = domains::ContextState::Waiting;
	40	+	set BUDGETS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
	41	+	set BUDGETS.windows[0] = budgets::Window {
	42	+	hart: 1, start: 10, end: 100, owner, handle: abi::Handle(0), context,
	43	+	};
	44	+	try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 15, 17);
	45	+	let result = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15);
	46	+	assert result.harts == 2 and result.deadline == 0xffffffffffffffff;
	47	+	assert DOMAINS.contexts[0].state == domains::ContextState::Ready;
	48	+	let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 16);
	49	+	assert again.harts == 0;
	50	+	}
	51	+
29	52		/// A timeout is delivered once, at its deadline, on its registered hart.
30	53		@test unsafe fn boundaries() throws (testing::TestError) {
31	54		let owner = initialize();
32	55		try! timers::request(&mut TIMERS, &DOMAINS, owner, 1, 10, 5, 17);
33		-	assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 15) == 0xffffffffffffffff);
34		-	assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 14) == 15);
	56	+	assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 15)).deadline == 0xffffffffffffffff);
	57	+	assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 14)).deadline == 15);
35	58		assert (try! events::pop(&mut MEMORY.ring) == nil);
36		-	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 15);
37		-	assert (next == 0xffffffffffffffff);
	59	+	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 15);
	60	+	assert (next.deadline == 0xffffffffffffffff);
38	61		let event = try! events::pop(&mut MEMORY.ring) else panic "timeout";
39	62		assert (event.kind == events::TIMEOUT and event.code == 17 and event.value == 0);
40		-	let again = try! timers::service(&mut TIMERS, &mut DOMAINS, 1, 16);
	63	+	let again = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 1, 16);
41	64		assert (try! events::pop(&mut MEMORY.ring) == nil);
42	65		}
43	66
44	67		/// Full queues retain due timeouts without requesting another immediate interrupt.
45	68		@test unsafe fn saturation() throws (testing::TestError) {

54	77		let mut full = false;
55	78		try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 1, 99) catch error {
56	79		assert (error == abi::Error::Busy); set full = true;
57	80		};
58	81		assert (full);
59		-	assert (try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1) == 0xffffffffffffffff);
	82	+	assert ((try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1)).deadline == 0xffffffffffffffff);
60	83		for i in 0..events::ORDINARY {
61	84		let event = try! events::pop(&mut MEMORY.ring) else panic "wake";
62	85		}
63		-	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 2);
	86	+	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 2);
64	87		for i in 0..limits::TIMEOUTS_PER_HART {
65	88		let event = try! events::pop(&mut MEMORY.ring) else panic "retained timeout";
66	89		assert (event.kind == events::TIMEOUT and event.code == i);
67	90		}
68	91		try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 2, 1, 100);

78	101		try timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0xffffffffffffffff, 1, 1) catch error {
79	102		assert (error == abi::Error::InvalidArg); set rejected += 1;
80	103		};
81	104		try! timers::request(&mut TIMERS, &DOMAINS, owner, 0, 0, 5, 1);
82	105		set DOMAINS.records[owner.index].state = domains::Lifecycle::Dead;
83		-	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, 0, 1);
84		-	assert (next == 0xffffffffffffffff and rejected == 2);
	106	+	let next = try! timers::service(&mut TIMERS, &mut DOMAINS, &BUDGETS, 0, 1);
	107	+	assert (next.deadline == 0xffffffffffffffff and rejected == 2);
85	108		assert (TIMERS.entries[0][0].owner == nil);
86	109		}

52	53		return;
53	54		}
54	55		throw abi::Error::Busy;
55	56		}
56	57
57		-	/// Deliver due entries and return the next future timeout boundary.
	58	+	/// Next timer boundary and dispatch notifications from delivered events.
	59	+	export record Result: Copy {
	60	+	/// Earliest future timeout, or the maximum tick when none is armed.
	61	+	deadline: u64,
	62	+	/// Harts with retained receiver authority that must refresh dispatch.
	63	+	harts: u32,
	64	+	}
	65	+
	66	+	/// Deliver due entries and report the next boundary and affected dispatch harts.
58	67		/// Full queues retain due entries for the next kernel entry without a timer spin.
59		-	export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, hart: u32, now: u64) -> u64 throws (abi::Error) {
	68	+	export unsafe fn service(store: &mut Store, receivers: &mut domains::Store, windows: &budgets::Store, hart: u32, now: u64) -> Result throws (abi::Error) {
60	69		if hart >= limits::HARTS {
61	70		throw abi::Error::InvalidArg;
62	71		}
63		-	let mut next: u64 = 0xffffffffffffffff;
	72	+	let mut result = Result { deadline: 0xffffffffffffffff, harts: 0 };
64	73		for i in 0..limits::TIMEOUTS_PER_HART {
65	74		let entry = store.entries[hart][i];
66	75		let owner = entry.owner else {
67	76		continue;
68	77		};