radiance · commit

kernel: Complete domain teardown and parenting

5e7ead57981261d3095812ae7f65daf329fb70fd5cf0666c09501552b7e6c22c

Alexis Sellier committed 13 days ago 1 parent 92262c08

Makefile +10 -1


# Verify the emulator binary exists.
EMU_PATH := $(shell command -v $(EMU) 2>/dev/null)

default: emulator $(RAD_BIN)
test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test
test: emulator seed-test std-test bin-test kernel-test module-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test termination-test

seed-test:
	@seed/test

module-test: emulator $(RAD_BIN)
	@RAD_EMULATOR="$(EMU)" sh test/modules/run

# Emulator command check

emulator:
ifeq ($(EMU_PATH),)
	$(error Emulator not found. Install it or set RAD_EMULATOR to its path)


# Concurrent user contexts and metadata transactions on every online hart.
.PHONY: smp-test
smp-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8" 7

# User exit, abort, and CPU faults preserve the surviving parent's execution.
termination-test: emulator $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run termination

.PHONY: termination-test

compiler/radiance.rad +2 -4

use std::sys;
use std::sys::unix;
use std::collections::dict;

/// Maximum number of modules we can load per package.
constant MAX_LOADED_MODULES: u32 = module::MAX_MODULES;
constant MAX_LOADED_MODULES: u32 = 128;
/// Maximum number of packages we can compile.
constant MAX_PACKAGES: u32 = 4;
/// Total module entries across all packages.
constant MAX_TOTAL_MODULES: u32 = 192;
/// Source code buffer arena (4 MB).
constant MAX_SOURCES_SIZE: u32 = 4194304;
/// Maximum number of test functions we can discover.
constant MAX_TESTS: u32 = 1024;
/// Maximum number of assembly source paths we can load per package.

static MAIN_ARENA: [u8; MAIN_ARENA_SIZE] = [0; MAIN_ARENA_SIZE];

/// Module source code.
static MODULE_SOURCES: [u8; MAX_SOURCES_SIZE] = [0; MAX_SOURCES_SIZE];
/// Module entries for all packages.
unsafe static MODULE_ENTRIES: [module::ModuleEntry; MAX_TOTAL_MODULES] = undefined;
unsafe static MODULE_ENTRIES: [module::ModuleEntry; module::MAX_MODULES] = undefined;
/// String pool.
unsafe static STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 };

/// Package scope.
unsafe static RESOLVER_PKG_SCOPE: resolver::Scope = undefined;

kernel/kernel.rad +1 -0

export mod backing;
export mod pages;
export mod loader;
export mod instances;
export mod domains;
export mod lifecycle;
export mod budgets;
export mod dispatch;
export mod timers;
export mod remote;
export mod calls;

kernel/kernel/abi.rad +2 -0

export constant DESTROY: u16 = 256;
/// All defined right bits.
export constant RIGHTS_MASK: u16 = 511;
/// Cascade destruction through creation ancestry.
export constant CASCADE: u64 = 1;
/// Fault code for explicit user Abort, separate from CPU exception codes.
export constant FAULT_ABORT: u32 = 256;

/// Object kind stored in the high four handle bits.
export union Kind: Copy {
    /// Unoccupied table entry.
    Empty = 0,

kernel/kernel/backing.rad +43 -10

    pool: frames::Pool,
    /// Backing-record lifetimes.
    slots: [slots::Slot; limits::ALLOCATIONS],
    /// Payloads for live backing records.
    records: [Allocation; limits::ALLOCATIONS],
    /// Exclusive upper bound of backing slots that have been published.
    used: u32,
    /// Live domain generation, or zero after all of that domain's exposures end.
    domains: [u32; limits::DOMAINS],
    /// Zero while open; otherwise one plus the next exposure slot to inspect.
    ending: [u32; limits::DOMAINS],
}

/// Initialize fresh backing and exposure-domain tables.
export fn initialize(store: &mut Store) {
    set store.used = 0;
    set store.pool.count = 0;
    set store.pool.retiredHead = 0;
    set store.pool.retiredCount = 0;
    slots::initialize(&mut store.slots[..]);
    for i in 0..limits::DOMAINS { set store.domains[i] = 0; }
    for i in 0..limits::DOMAINS { set store.domains[i] = 0; set store.ending[i] = 0; }
}

/// Register a domain before it can receive allocation exposure.
export fn registerDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
    if domain.index >= limits::DOMAINS or domain.generation == 0 { throw abi::Error::InvalidArg; }

    set store.domains[domain.index] = domain.generation;
}

/// Test exposure-domain generation without reading beyond the fixed table.
export fn domainLive(store: &Store, domain: abi::Ref) -> bool {
    return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation;
    return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation and store.ending[domain.index] == 0;
}

/// Publish reserved metadata with one page object and its live origin exposure.
/// The caller owns the committed run and has validated the origin domain.
export unsafe fn publish(store: &mut Store, reservation: slots::Reservation, run: frames::Run, origin: abi::Ref) -> abi::Ref {
    assert domainLive(store, origin);
    let object = slots::reference(&reservation);
    assert slots::matches(&store.slots[..], object, slots::State::Reserved);
    if object.index >= store.used { set store.used = object.index + 1; }
    set store.records[object.index].run = run;
    set store.records[object.index].pages = 1;
    set store.records[object.index].exposed = 1;
    let mut bits = bitset::init(&mut store.records[object.index].exposures[..]);
    bitset::put(&mut bits, origin.index);

    bitset::put(&mut bits, domain.index);
    set store.records[object.index].exposed += 1;
}

/// Reclaim only when no page object and no domain exposure remains.
fn reclaim(store: &mut Store, object: abi::Ref) {
fn reclaim(store: &mut Store, object: abi::Ref, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) {
    if store.records[object.index].pages <> 0 or store.records[object.index].exposed <> 0 { return; }
    let run = store.records[object.index].run;
    try! frames::release(&mut store.pool, run);
    try! release(&mut store.pool, run);
    try! slots::release(&mut store.slots[..], object);
}

/// Release one page object after its final handle disappears.
export fn releasePage(store: &mut Store, object: abi::Ref) throws (abi::Error) {
    try require(store, object);
    if store.records[object.index].pages == 0 { throw abi::Error::BadHandle; }
    set store.records[object.index].pages -= 1;
    reclaim(store, object);
    reclaim(store, object, frames::release);
}

/// End all exposure for one domain before its slot can be registered again.
/// Callers serialize this bounded scan with allocation and capability changes.
/// End all exposure while the caller has exclusive access to metadata.
export unsafe fn endDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
    if not domainLive(store, domain) { throw abi::Error::BadHandle; }
    for i in 0..limits::ALLOCATIONS {
    assert try end(store, domain, limits::ALLOCATIONS, frames::release);
}

/// Inspect at most limit backing slots and retire allocations whose lifetimes end.
/// Returns true after all exposure is clear and the domain slot can be registered again.
export unsafe fn endDomainStep(store: &mut Store, domain: abi::Ref, limit: u32) -> bool throws (abi::Error) {
    return try end(store, domain, limit, frames::retire);
}

/// Close exposure admission and advance one generation's retained scan cursor.
unsafe fn end(store: &mut Store, domain: abi::Ref, limit: u32,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) -> bool throws (abi::Error)
{
    if domain.index >= limits::DOMAINS or domain.generation == 0
        or store.domains[domain.index] <> domain.generation { throw abi::Error::BadHandle; }
    if limit == 0 { throw abi::Error::InvalidArg; }
    if store.ending[domain.index] == 0 { set store.ending[domain.index] = 1; }
    let first = store.ending[domain.index] - 1;
    let mut count = store.used - first;
    if count > limit { set count = limit; }
    let next = first + count;
    for i in first..next {
        if store.slots[i].state <> slots::State::Live { continue; }
        let mut bits = bitset::new(&mut store.records[i].exposures[..]);
        if not bitset::contains(&bits, domain.index) { continue; }
        bitset::clear(&mut bits, domain.index);
        set store.records[i].exposed -= 1;
        reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation });
        reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation }, release);
    }
    if next < store.used {
        set store.ending[domain.index] = next + 1;
        return false;
    }
    set store.domains[domain.index] = 0;
    set store.ending[domain.index] = 0;
    return true;
}

kernel/kernel/boot.rad +6 -3

use super::sync;
use super::trap;
use super::pages;
use super::registry;
use super::domains;
use super::lifecycle;
use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use super::interrupts;

/// Number of harts that validated their machine stack.
static ARRIVED: u64 = 0;
/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and user calls, and stop on unsupported traps.
/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    let cause = trap::classify(frame.cause);
    if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
        dispatch::interrupt(frame, hart);
        return;
    }
    if cause == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);
        return;
    }
    if trap::fromUser(frame) {
        dispatch::fault(frame, hart);
    }
    print("kernel: unexpected trap\n");
    trap::halt();
}


        let treeRange = range::new(treeAddress, size as u64) else panic "FDT range";
        assert platform::inRam(&PLATFORM, treeRange);
        try! platform::protect(&mut PLATFORM, treeRange);
        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        domains::initialize(&mut domains::STORE);
        lifecycle::initialize(&mut lifecycle::CALLS);
        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
        try! plic::initialize(&PLATFORM);

kernel/kernel/budgets.rad +2 -1

    let object = try owned(store, table, handle, abi::TRANSFER);
    let window = try get(store, object);
    if window.hart as u64 <> current.hart or window.context <> abi::reference(current.context)
        or now < window.start or now >= window.end { throw abi::Error::Busy; }
    let permit = try capability::lookup(table, target, abi::Kind::Domain, abi::Rights(abi::WAKE));
    let receiver = try domains::get(domainStore, permit.object) catch { throw abi::Error::Busy; };
    let receiver = try domains::get(domainStore, permit.object);
    if receiver.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
    if receiver.state <> domains::Lifecycle::Active or abi::id(receiver.initial) == current.context { throw abi::Error::Busy; }
    let context = try domains::context(domainStore, permit.object, receiver.initial) catch { throw abi::Error::Busy; };
    if context.state <> domains::ContextState::Ready or context.hart <> nil { throw abi::Error::Busy; }
    let result = try bind(store, domainStore, table, Binding { budget: handle, domain: target, context: receiver.initial }, now) catch error {
        if error == abi::Error::InvalidArg { throw abi::Error::Busy; }

kernel/kernel/calls.rad +114 -32

use super::timers;
use super::sync;
use super::remote;
use super::interrupts;
use super::plic;
use super::lifecycle;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
    match operation {
        case 11, 12, 70, 71, 73 => {},

}

/// Validate resident object generations for capabilities without page ownership.
unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },
        case abi::Kind::Domain, abi::Kind::Events => {
            let object = try domains::get(&domains::STORE, entry.object);
            if object.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
        },
        case abi::Kind::Image => { let object = try registry::get(&registry::STORE, entry.object); },
        else => throw abi::Error::BadHandle,
    }
}

/// Reject expired domain authority before an allocation can reserve resources.
unsafe fn allocationAuthority(table: &capability::Table, operation: u64, handle: abi::Handle) throws (abi::Error) {
    if operation <> 20 and operation <> 30 and operation <> 51 { return; }
    let rights = abi::CREATE if operation == 20 else abi::ALLOCATE;
    try resident(try capability::authority(table, handle, abi::Rights(rights)));
}

/// Grant or transfer authority while both domain tables remain serialized.
unsafe fn delegate(domain: &mut domains::Domain, handle: abi::Handle, target: abi::Handle, rights: u64, moving: bool, now: u64)
    -> abi::Handle throws (abi::Error)
{
    let owner = domain.memory.table.owner;

    }
    let output = @sliceOf(memory(arguments[1]), words.len);
    for i in 0..words.len { set output[i] = words[i]; }
}

/// Validate a termination call and return harts that must release stopped contexts.
export unsafe fn termination(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u32 throws (abi::Error) {
    assert arguments.len == 4;
    let domain = try domains::get(&domains::STORE, owner);
    if domain.state <> domains::Lifecycle::Active { throw abi::Error::BadHandle; }
    if arguments[2] <> 0 or arguments[3] <> 0 { throw abi::Error::InvalidArg; }
    match operation {
        case 49 => {
            if arguments[0] > 0xffffffff or arguments[1] <> 0 { throw abi::Error::InvalidArg; }
            return try domains::terminate(&mut domains::STORE, owner, events::CHILD_EXIT, arguments[0] as u32);
        },
        case 43 => {
            if arguments[0] <> 0 or arguments[1] <> 0 { throw abi::Error::InvalidArg; }
            return try domains::terminate(&mut domains::STORE, owner, events::FAULT, abi::FAULT_ABORT);
        },
        case 22 => {
            let flags = arguments[1];
            if (flags & ~abi::CASCADE) <> 0 or (arguments[0] == 0 and flags <> 0) { throw abi::Error::InvalidArg; }
            let mut target = owner;
            if arguments[0] <> 0 {
                let entry = try capability::lookup(&domain.memory.table, abi::Handle(arguments[0]), abi::Kind::Domain, abi::Rights(abi::DESTROY));
                set target = entry.object;
                if (try domains::get(&domains::STORE, target)).state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
            }
            if flags == abi::CASCADE { return try domains::terminateTree(&mut domains::STORE, target, events::CHILD_EXIT, 0); }
            return try domains::terminate(&mut domains::STORE, target, events::CHILD_EXIT, 0);
        },
        else => throw abi::Error::InvalidArg,
    }
}

/// Execute one operation with domain and resource metadata serialized.
export unsafe fn invoke(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    let mut domain = try domains::get(&domains::STORE, owner);
    let handle = abi::Handle(arguments[0]);
    try allocationAuthority(&domain.memory.table, operation, handle);
    match operation {
        case 22, 43, 49 => { let stopped = try termination(owner, operation, arguments); return 0; },
        case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
        case 12 => { try drop(&mut domain, handle); return 0; },
        case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,
            &registry::STORE, &mut domain.memory.table, handle, abi::Handle(arguments[1])),
        case 21 => {
            try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,
                handle, arguments[1], arguments[2], arguments[3]);
            return 0;
        },
        case 23 => {
            if arguments[2] <> 0 or arguments[3] <> 0 { throw abi::Error::InvalidArg; }
            try domains::reparent(&mut domains::STORE, &domain.memory.table, handle, abi::Handle(arguments[1]));
            return 0;
        },
        case 30 => return *try pages::allocate(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
        case 31 => return *try pages::split(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
        case 40 => {
            if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 { throw abi::Error::InvalidArg; }
            try budgets::handoff(&mut budgets::STORE, &domains::STORE, &mut domain.memory.table, dispatch::current(), handle, now);

            return 0;
        },
        case 47 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
            let target = try domains::get(&domains::STORE, entry.object);
            if target.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
            let mut parent: u64 = 0;
            if let object = target.parent { set parent = abi::id(object); }
            try reply(&domain, arguments, &[abi::id(entry.object), abi::id(target.creator), parent, target.state as u64]);
            return 0;
        },
        case 48 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Events, abi::Rights(0));
            let target = try domains::get(&domains::STORE, entry.object);
            if target.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
            try reply(&domain, arguments, &[events::CAPACITY as u64]);
            return 0;
        },
        case 51 => return *try loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut domain.memory.table,
            loader::Request { authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3] }),

            return 0;
        },
        case 62 => {
            let permission = try capability::authority(&domain.memory.table, handle, abi::Rights(0));
            let target = try domains::get(&domains::STORE, permission.object);
            if target.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
            let mut object = target.initial;
            if arguments[1] <> 0 { set object = abi::reference(arguments[1]); }
            let context = try domains::context(&domains::STORE, permission.object, object);
            let mut hart: u64 = 0xffffffffffffffff;
            if let id = context.hart { set hart = id as u64; }

/// Load a package with metadata locked only around resource transactions.
unsafe fn runtimeLoad(domain: &mut domains::Domain, request: loader::Request, guard: sync::Guard) -> u64 throws (abi::Error) {
    let owner = domain.memory.table.owner;
    let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
        catch error { sync::leave(guard); throw error; };
    let context = abi::reference(dispatch::current().context);
    let lease = lifecycle::holdLoad(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
    let retained = lifecycle::load(&lifecycle::CALLS, &lease);
    sync::leave(guard);
    let input = try loader::decode(&pending) catch error {
    let input = try loader::decodeInput(&retained.input) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(guard); throw error;
    };
    let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
    let existing = try loader::identifyInput(&registry::STORE, &retained.input, &input) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(guard); throw error;
    };
    let allocationGuard = sync::enter();
    if not live(owner) {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(allocationGuard); throw abi::Error::BadHandle;
    }
    if let object = existing {
        let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
        let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Existing(object))
            catch error { sync::leave(allocationGuard); throw error; };
        sync::leave(allocationGuard); return *result;
    }
    let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
    try lifecycle::reserveOutput(&mut lifecycle::CALLS, &lease, &mut pages::STORE, &mut registry::STORE) catch error {
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(allocationGuard); throw error;
    };
    let compiling = lifecycle::load(&lifecycle::CALLS, &lease);
    let output = compiling.output else panic "missing loader output";
    sync::leave(allocationGuard);
    let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
    let compiled = try loader::compileInput(&registry::STORE, &compiling.input, &input, &output) catch error {
        let guard = sync::enter();
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(guard); throw error;
    };
    let publicationGuard = sync::enter();
    if not live(owner) {
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(publicationGuard); throw abi::Error::BadHandle;
    }
    let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
    let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
    let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
        &mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Compiled(compiled))
        catch error { sync::leave(publicationGuard); throw error; };
    sync::leave(publicationGuard);
    return *result;
}

/// Execute a call with short metadata transactions and private bulk initialization.
/// Context ownership and pending calls retain storage across dispatch boundaries.
export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    if operation == 21 or operation == 50 { return try runtimeReady(owner, operation, arguments, now); }
    if operation == 21 or operation == 50 { return try runtimeReady(owner, operation, arguments); }
    let guard = sync::enter();
    let mut domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
    if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
    try allocationAuthority(&domain.memory.table, operation, abi::Handle(arguments[0])) catch error {
        sync::leave(guard); throw error;
    };
    if operation == 22 or operation == 43 or operation == 49 {
        let stopped = try termination(owner, operation, arguments) catch error { sync::leave(guard); throw error; };
        dispatch::reschedule(stopped);
        sync::leave(guard); return 0;
    }
    let target = try budgetHart(&domain.memory.table, operation, arguments) catch error { sync::leave(guard); throw error; };
    if let destination = target {
        let source = dispatch::hart();
        if destination <> source {
            let call = remote::Call { owner, context: abi::reference(dispatch::current().context), operation,

    let handle = abi::Handle(arguments[0]);
    match operation {
        case 20 => {
            let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
                &mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error { sync::leave(guard); throw error; };
            let context = abi::reference(dispatch::current().context);
            let lease = lifecycle::holdDomain(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
            sync::leave(guard);
            try domains::prepare(&registry::STORE, &pending) catch error {
            try lifecycle::prepareDomain(&lifecycle::CALLS, &registry::STORE, &lease) catch error {
                let guard = sync::enter();
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                let failed = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, failed);
                sync::leave(guard); throw error;
            };
            let guard = sync::enter();
            let ready = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
            if not live(owner) {
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
            let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready)
                catch error { sync::leave(guard); throw error; };
            sync::leave(guard); return *result;
        },
        case 30 => {
            let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
                catch error { sync::leave(guard); throw error; };
            let context = abi::reference(dispatch::current().context);
            let lease = lifecycle::holdPage(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
            sync::leave(guard);
            pages::clear(&pending);
            lifecycle::clearPage(&lifecycle::CALLS, &lease);
            let guard = sync::enter();
            let ready = lifecycle::takePage(&mut lifecycle::CALLS, &mut domains::STORE, lease);
            if not live(owner) {
                pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
                pages::cancel(&mut pages::STORE, &mut domain.memory.table, ready);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
            let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, ready);
            sync::leave(guard); return *result;
        },
        case 51 => return try runtimeLoad(&mut domain, loader::Request {
            authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
        }, guard),

            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
                &domain.memory.table, handle, start) catch error { sync::leave(guard); throw error; };
            let mut target: abi::Ref = undefined;
            match &pending { case domains::ContextReservation::Held(allocation) => { set target = allocation.owner; }, }
            let context = abi::reference(dispatch::current().context);
            let lease = lifecycle::holdContext(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
            sync::leave(guard);
            domains::contextClear(&pending);
            lifecycle::clearContext(&lifecycle::CALLS, &lease);
            let guard = sync::enter();
            let ready = lifecycle::takeContext(&mut lifecycle::CALLS, &mut domains::STORE, lease);
            if not live(owner) or not live(target) {
                domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
                domains::contextCancel(&mut domains::STORE, &mut pages::STORE, ready);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = domains::contextPublish(&mut domains::STORE, pending);
            let result = domains::contextPublish(&mut domains::STORE, ready);
            sync::leave(guard); return abi::id(result);
        },
        else => {
            let result = try invoke(owner, operation, arguments, now) catch error { sync::leave(guard); throw error; };
            let result = try invoke(owner, operation, arguments, dispatch::clock()) catch error { sync::leave(guard); throw error; };
            sync::leave(guard); return result;
        },
    }
}

/// Reserve remote dispatch capacity before activation or event publication.
unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u64 throws (abi::Error) {
    if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) { throw abi::Error::InvalidArg; }
    let guard = sync::enter();
    let domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
    if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
    let handle = abi::Handle(arguments[0]);

    let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
        sync::leave(guard); throw error;
    };
    let source = dispatch::hart();
    let context = abi::reference(dispatch::current().context);
    let now = dispatch::clock();
    let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
    let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error { sync::leave(guard); throw error; };
    let result = try invoke(owner, operation, arguments, now) catch error {
        remote::cancelReservation(&mut remote::STORE, pending);
        sync::leave(guard); throw error;

{
    assert trap::fromUser(frame) and frame.cause == 8;
    assert trap::advanceCall(frame);
    let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
    let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
        let operation = frame.registers[17];
        if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
            try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
                assert stopped == abi::Error::BadHandle;
            };
        }
        set frame.registers[10] = (0 as u64) - error as u64;
        return;
    };
    set frame.registers[10] = result;
}

kernel/kernel/dispatch.rad +59 -7

use super::abi;
use super::limits;
use super::slots;
use super::budgets;
use super::domains;
use super::lifecycle;
use super::loader;
use super::registry;
use super::frames;
use super::interrupts;
use super::events;
use super::pages;
use super::platform;
use super::trap;
use super::range;
use super::calls;
use super::timers;

record Runtime: Copy {
    /// Current context and retained idle frame.
    state: State,
    /// Validated timer registers for this hart.
    timer: Timer,
    /// Minimum ticks between scheduled maintenance passes.
    maintenance: u64,
    /// Earliest tick for another background maintenance pass.
    nextMaintenance: u64,
}

/// Runtime slots initialized before each hart enables timer interrupts.
unsafe static HARTS: [Runtime; limits::HARTS] = undefined;


    let device = try timer(machine, id);
    let stack = machine.stacks[id];
    let mut idle = trap::Frame { registers: [0; 32], pc: idleAddress(), status: 0x1880, cause: 0, value: 0 };
    set idle.registers[2] = stack.end;
    set idle.registers[3] = kernelGp;
    let mut maintenance = machine.timebase as u64 / 100;
    if maintenance == 0 { set maintenance = 1; }
    set HARTS[id] = Runtime {
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0,
    };
    arm(device, 0xffffffffffffffff);
}

/// Tail-enter dispatch on the firmware stack and resume the selected frame.

        set deliveries[count] = delivery;
        set count += 1;
    }
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
    let notified = timeout.harts | plic::service(clock);
    let notified = timeout.harts | plic::service(clock) | lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout.deadline < deadline { set deadline = timeout.deadline; }
    for target in 0..limits::HARTS {
        if target <> id and (notified & (1 << target)) <> 0 { signal(target); }
    }
    arm(HARTS[id].timer, deadline);
    reschedule(notified);
    if previous <> choice.context {
        if let context = previous { signalContext(context, clock); }
    }
    for delivery in &deliveries[..count] {
        assert remote::acknowledge(&mut remote::STORE, id, delivery.id);
        if delivery.request.source <> id { signal(delivery.request.source); }
        if let case remote::Action::Budget(call) = delivery.request.action { signalContext(call.context, clock); }
        if let case remote::Action::Ready(context) = delivery.request.action { signalContext(context, clock); }
    }
    lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE);
    let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE);
    let maintenanceClock = now(HARTS[id].timer);
    let mut maintenance = cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
    if maintenance and maintenanceClock >= HARTS[id].nextMaintenance {
        let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE);
        let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256);
        set maintenance = cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0;
        let after = now(HARTS[id].timer);
        let interval = HARTS[id].maintenance;
        set HARTS[id].nextMaintenance = 0xffffffffffffffff;
        if interval < 0xffffffffffffffff - after { set HARTS[id].nextMaintenance = after + interval; }
    }
    if maintenance and HARTS[id].nextMaintenance < deadline { set deadline = HARTS[id].nextMaintenance; }
    arm(HARTS[id].timer, deadline);
    sync::leave(guard);
}

/// Signal an initialized online hart while request metadata is serialized.
unsafe fn signal(target: u32) { writeSoftware(HARTS[target].timer.software, 1); }

/// Signal other harts that retain stopped contexts under the metadata lock.
export unsafe fn reschedule(harts: u32) {
    let source = hart();
    for target in 0..limits::HARTS {
        if target <> source and (harts & (1 << target)) <> 0 { signal(target); }
    }
}

/// Notify the current owner or the eligible hart after a context is released.
unsafe fn signalContext(context: abi::Ref, now: u64) {
    if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) { return; }
    if let owner = domains::STORE.contexts[context.index].hart { signal(owner); return; }
    for i in 0..limits::BUDGETS {

    };
    releaseRequest(source, request);
    sync::leave(guard); return result;
}

/// Read mtime through the executing hart's validated timer mapping.
export unsafe fn clock() -> u64 { return now(HARTS[hart()].timer); }

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else panic "no current context";

    let guard = sync::enter();
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    let owner = domains::STORE.contexts[context.index].owner;
    if frame.registers[17] == abi::Operation::ContextReturn as u64 {
        assert trap::advanceCall(frame);
        try! domains::returned(&mut domains::STORE, owner, context);
        let stopped = try! domains::returned(&mut domains::STORE, owner, context);
        reschedule(stopped);
        sync::leave(guard);
    } else {
        sync::leave(guard);
        sync::restoreInterrupts(8);
        calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
        let interrupts = sync::maskInterrupts();
    }
    interrupt(frame, &mut *localAnchor());
}

/// Stop a faulting user domain and resume another authorized context.
export unsafe fn fault(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
    assert trap::fromUser(frame);
    let guard = sync::enter();
    let context = HARTS[hart()].state.current else panic "fault without current context";
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    let owner = domains::STORE.contexts[context.index].owner;
    let stopped = try! domains::terminate(&mut domains::STORE, owner, events::FAULT, frame.cause as u32);
    reschedule(stopped);
    sync::leave(guard);
    interrupt(frame, &mut *localAnchor());
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let mut frame = HARTS[id].state.idle;

kernel/kernel/domains.rad +191 -32

export constant KERNEL_STACK_PAGES: u32 = 64;

/// Management rights returned to a domain's creator.
export constant MANAGEMENT_RIGHTS: u16 = abi::DESTROY | abi::EXECUTE | abi::GRANT | abi::TRANSFER | abi::WAKE;

/// Words needed to retain one ancestry bit per domain slot.
constant ANCESTRY_WORDS: u32 = (limits::DOMAINS + 31) / 32;

/// Protection-domain lifetime independent of execution budgets.
export union Lifecycle: Copy {
    /// Resources can be installed before one-time activation.
    Pending,
    /// The domain can execute when a context has CPU authority.
    Active,
    /// Execution has stopped and the identity can remain as a tombstone.
    /// Contexts cannot be selected; hart release and resource cleanup can remain.
    Dead,
}

/// Private domain metadata and its shared event ring.
export record Memory: Copy {

    state: ContextState,
    /// Hart retaining this context as its current execution owner.
    hart: ?u32,
    /// Remote call whose retained continuation requires this context's storage.
    pending: ?abi::Ref,
    /// A kernel call retains unpublished resources and requires this context's stack.
    reservation: bool,
    /// Physical frames retained until this context stops.
    kernelFrames: frames::Run,
    /// Exclusive stack bounds used by trap entry and suspended M-mode calls.
    kernelStack: range::Range,
    /// Exclusive user stack extent; empty until initial activation.

export record Store: Copy {
    /// Domain generations, including retained dead identities.
    slots: [slots::Slot; limits::DOMAINS],
    /// Payload for each published domain.
    records: [Domain; limits::DOMAINS],
    /// True after private storage is retired, while terminal identity can remain.
    reclaimed: [bool; limits::DOMAINS],
    /// True after quiescence admits a domain to resource teardown.
    reaping: [bool; limits::DOMAINS],
    /// Number of dead domain identities retained by the store.
    dead: u32,
    /// First domain slot considered by the next reaper pass.
    reapCursor: u32,
    /// Creation ancestors by domain slot. Clear a column before its ancestor slot is reused.
    ancestors: [[u32; ANCESTRY_WORDS]; limits::DOMAINS],
    /// Context generations shared by initial and additional contexts.
    contextSlots: [slots::Slot; limits::CONTEXTS],
    /// Payload for each published execution context.
    contexts: [Context; limits::CONTEXTS],
    /// Trusted event producers and terminal records.

/// Address of the U-mode normal-return call sequence.
fn returnAddress() -> u64;

/// Initialize fresh metadata before boot-domain publication.
export fn initialize(store: &mut Store) {
    set store.dead = 0;
    set store.reapCursor = 0;
    slots::initialize(&mut store.slots[..]);
    slots::initialize(&mut store.contextSlots[..]);
    for i in 0..limits::CONTEXTS { set store.contexts[i].reservation = false; }
    for i in 0..limits::DOMAINS { set store.ancestors[i] = [0; ANCESTRY_WORDS]; set store.reclaimed[i] = false; set store.reaping[i] = false; }
    events::initialize(&mut store.events);
}

/// Read a domain record after validating its retained generation.
export fn get(store: &Store, object: abi::Ref) -> Domain throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) { throw abi::Error::BadHandle; }
    if not slots::matches(&store.slots[..], object, slots::State::Live) or store.reclaimed[object.index] { throw abi::Error::BadHandle; }
    return store.records[object.index];
}

/// Test whether all published and unpublished execution ownership has ended.
export fn quiescent(store: &Store, owner: abi::Ref) -> bool throws (abi::Error) {
    let domain = try get(store, owner);
    if domain.state <> Lifecycle::Dead { return false; }
    for i in 0..limits::CONTEXTS {
        let state = store.contextSlots[i].state;
        if state <> slots::State::Live and state <> slots::State::Reserved { continue; }
        if store.contexts[i].owner <> owner { continue; }
        if state == slots::State::Reserved or store.contexts[i].state <> ContextState::Stopped
            or store.contexts[i].hart <> nil or store.contexts[i].pending <> nil or store.contexts[i].reservation { return false; }
    }
    return true;
}

/// Retire at most one stopped context after all domain execution ownership ends.
export fn retireContext(store: &mut Store, pool: &mut frames::Pool, owner: abi::Ref) -> bool throws (abi::Error) {
    if not try quiescent(store, owner) { throw abi::Error::Busy; }
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state <> slots::State::Live or store.contexts[i].owner <> owner { continue; }
        let object = abi::Ref { index: i, generation: store.contextSlots[i].generation };
        try! frames::retire(pool, store.contexts[i].kernelFrames);
        try! slots::release(&mut store.contextSlots[..], object);
        return true;
    }
    return false;
}

/// Retire private domain storage after contexts, handles, and exposures are gone.
export unsafe fn retireStorage(store: &mut Store, backings: &mut backing::Store, owner: abi::Ref) throws (abi::Error) {
    let domain = try get(store, owner);
    if not try quiescent(store, owner) { throw abi::Error::Busy; }
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state == slots::State::Live and store.contexts[i].owner == owner { throw abi::Error::Busy; }
    }
    for i in 0..limits::HANDLES {
        let state = domain.memory.table.slots[i].state;
        if state == slots::State::Live or state == slots::State::Reserved { throw abi::Error::Busy; }
    }
    if backings.domains[owner.index] <> 0 or store.events.queues[owner.index].generation <> 0 { throw abi::Error::Busy; }
    try! frames::retire(&mut backings.pool, domain.graph.frames);
    try! frames::retire(&mut backings.pool, domain.allocation);
    set store.reclaimed[owner.index] = true;
}

/// Release an acknowledged terminal identity and clear its creation-ancestry column.
export fn releaseIdentity(store: &mut Store, owner: abi::Ref) throws (abi::Error) {
    if not slots::matches(&store.slots[..], owner, slots::State::Live) { throw abi::Error::BadHandle; }
    if not store.reclaimed[owner.index] or not try events::acknowledged(&store.events, owner) { throw abi::Error::Busy; }
    let word = owner.index / 32;
    let bit = 1 << (owner.index % 32);
    for i in 0..limits::DOMAINS { set store.ancestors[i][word] &= ~bit; }
    try! events::release(&mut store.events, owner);
    try! slots::release(&mut store.slots[..], owner);
    assert store.dead > 0;
    set store.dead -= 1;
}

/// Read a live context after checking its generation and owning domain.
export fn context(store: &Store, owner: abi::Ref, object: abi::Ref) -> Context throws (abi::Error) {
    let domain = try get(store, owner);
    if not slots::matches(&store.contextSlots[..], object, slots::State::Live)
        or store.contexts[object.index].owner <> owner { throw abi::Error::BadHandle; }
    return store.contexts[object.index];
}

/// Stop a returned context; initial return ends execution of its whole domain.
export fn returned(store: &mut Store, owner: abi::Ref, object: abi::Ref) throws (abi::Error) {
    let current = try context(store, owner, object);
    if store.records[owner.index].initial == object {
        set store.records[owner.index].state = Lifecycle::Dead;
        for i in 0..limits::CONTEXTS {
            if store.contextSlots[i].state == slots::State::Live and store.contexts[i].owner == owner {
                set store.contexts[i].state = ContextState::Stopped;
            }
/// Change the live lifecycle receiver under shared metadata serialization.
export fn reparent(store: &mut Store, table: &capability::Table, child: abi::Handle, parent: abi::Handle)
    throws (abi::Error)
{
    let target = try capability::lookup(table, child, abi::Kind::Domain, abi::Rights(abi::DESTROY));
    let receiver = try capability::lookup(table, parent, abi::Kind::Domain, abi::Rights(abi::WAKE));
    let domain = try get(store, target.object);
    let destination = try get(store, receiver.object);
    if domain.state == Lifecycle::Dead or destination.state == Lifecycle::Dead { throw abi::Error::BadHandle; }
    set store.records[target.object.index].parent = receiver.object;
}

/// Retain a terminal event, stop execution, and report harts that must release contexts.
export fn terminate(store: &mut Store, owner: abi::Ref, kind: u16, code: u32) -> u32 throws (abi::Error) {
    return try stop(store, owner, false, kind, code);
}

/// Stop a domain and its creation descendants without changing creation ancestry.
export fn terminateTree(store: &mut Store, owner: abi::Ref, kind: u16, code: u32) -> u32 throws (abi::Error) {
    return try stop(store, owner, true, kind, code);
}

/// Commit terminal state for a selected set before releasing the metadata lock.
fn stop(store: &mut Store, owner: abi::Ref, tree: bool, kind: u16, code: u32) -> u32 throws (abi::Error) {
    if kind <> events::CHILD_EXIT and kind <> events::FAULT { throw abi::Error::InvalidArg; }
    let domain = try get(store, owner);
    let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
    for i in 0..limits::DOMAINS {
        if store.slots[i].state <> slots::State::Live or store.records[i].state == Lifecycle::Dead { continue; }
        if i <> owner.index and (not tree or (store.ancestors[i][owner.index / 32] & (1 << (owner.index % 32))) == 0) { continue; }
        if store.events.terminals[i].state <> events::State::Free { throw abi::Error::Busy; }
        if store.events.queues[i].generation <> store.slots[i].generation { throw abi::Error::BadHandle; }
        set generations[i] = store.slots[i].generation;
    }
    for i in 0..limits::DOMAINS {
        if generations[i] == 0 { continue; }
        let object = abi::Ref { index: i, generation: generations[i] };
        let terminal = try! events::reserve(&mut store.events, object);
        let mut receiver = abi::Ref { index: 0, generation: 0 };
        if let parent = store.records[i].parent { set receiver = parent; }
        events::finish(&mut store.events, terminal, receiver, kind, code);
        set store.records[i].state = Lifecycle::Dead;
        set store.dead += 1;
    }
    try! events::closeSet(&mut store.events, &generations[..]);
    let mut harts: u32 = 0;
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state <> slots::State::Live { continue; }
        let target = store.contexts[i].owner;
        if generations[target.index] == target.generation {
            set store.contexts[i].state = ContextState::Stopped;
            if let hart = store.contexts[i].hart { set harts |= 1 << hart; }
        }
    } else {
        set store.contexts[object.index].state = ContextState::Stopped;
    }
    for i in 0..limits::DOMAINS {
        if store.slots[i].state <> slots::State::Live { continue; }
        if let parent = store.records[i].parent {
            if generations[parent.index] == parent.generation { set store.records[i].parent = nil; }
        }
    }
    return harts;
}

/// Stop a returned context; initial return ends execution of its whole domain.
export fn returned(store: &mut Store, owner: abi::Ref, object: abi::Ref) -> u32 throws (abi::Error) {
    let current = try context(store, owner, object);
    if store.records[owner.index].initial == object { return try terminate(store, owner, events::CHILD_EXIT, 0); }
    set store.contexts[object.index].state = ContextState::Stopped;
    return 0;
}

/// Atomically check notifications and suspend the caller if its queue is empty.
/// The caller serializes this check with notification publication.
export unsafe fn wait(store: &mut Store, owner: abi::Ref, object: abi::Ref) throws (abi::Error) {

export fn contextDestroy(store: &mut Store, memory: &mut pages::Store, table: &capability::Table,
    authority: abi::Handle, object: abi::Ref) throws (abi::Error)
{
    let permit = try capability::authority(table, authority, abi::Rights(abi::DESTROY));
    let target = try context(store, permit.object, object);
    if store.records[permit.object.index].state == Lifecycle::Dead { throw abi::Error::BadHandle; }
    if store.records[permit.object.index].initial == object { throw abi::Error::Denied; }
    if target.hart <> nil or target.pending <> nil { throw abi::Error::Busy; }
    if target.hart <> nil or target.pending <> nil or target.reservation { throw abi::Error::Busy; }
    try! frames::release(&mut memory.backings.pool, target.kernelFrames);
    try! slots::release(&mut store.contextSlots[..], object);
}

/// Derive a descending stack extent from one writable Page capability.

export unsafe fn activate(store: &mut Store, memory: &pages::Store, table: &capability::Table,
    handle: abi::Handle, stack: u64, args: u64, size: u64) throws (abi::Error)
{
    let entry = try capability::lookup(table, handle, abi::Kind::Domain, abi::Rights(abi::EXECUTE));
    let mut domain = try get(store, entry.object);
    if domain.state == Lifecycle::Dead { throw abi::Error::BadHandle; }
    if domain.state <> Lifecycle::Pending { throw abi::Error::NotPending; }
    let userStack = try stackRange(memory, &domain.memory.table, stack);
    if size <> 0 and not pages::accessible(memory, &domain.memory.table, args, size, abi::Rights(abi::READ)) {
        throw abi::Error::InvalidArg;
    }

    set frame.registers[3] = domain.graph.table.ptr as u64;
    set frame.registers[10] = start.args;
    set frame.registers[11] = start.size;
    let object = slots::reference(&slot);
    set store.contexts[object.index] = Context {
        owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames, kernelStack, userStack, frame,
        owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames, kernelStack, userStack, frame,
    };
    match slot {
        case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
            object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
        }),
    }
}

/// Clear an unpublished context's private kernel stack.
export fn contextClear(reservation: &ContextReservation) {
    match reservation { case ContextReservation::Held(allocation) => zero(allocation.base, allocation.frames.count), }
    match reservation { case ContextReservation::Held(allocation) => contextClearAllocation(allocation), }
}

/// Clear retained private stack frames outside the metadata lock.
export fn contextClearAllocation(allocation: &ContextAllocation) {
    zero(allocation.base, allocation.frames.count);
}

/// Check the reserved context and its retained physical allocation.
fn contextRequired(store: &Store, allocation: &ContextAllocation) {
    assert slots::matches(&store.contextSlots[..], allocation.object, slots::State::Reserved);

    }
}

/// Return an unpublished context's stack and slot under metadata serialization.
export fn contextCancel(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation) {
    contextCancelUsing(store, memory, reservation, frames::release);
}

/// Cancel a context slot and transfer its private stack to the release operation.
export fn contextCancelUsing(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case ContextReservation::Held(allocation) => {
            contextRequired(store, &allocation);
            try! frames::release(&mut memory.backings.pool, allocation.frames);
            try! release(&mut memory.backings.pool, allocation.frames);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
        },
    }
}


        },
    }
}

/// Return storage that has never been exposed to a published domain.
fn discard(pool: &mut frames::Pool, prepared: Prepared) {
    try! frames::release(pool, prepared.kernelFrames);
    try! frames::release(pool, prepared.graph.frames);
    try! frames::release(pool, prepared.allocation);
fn discard(pool: &mut frames::Pool, prepared: Prepared, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) {
    try! release(pool, prepared.kernelFrames);
    try! release(pool, prepared.graph.frames);
    try! release(pool, prepared.allocation);
}

/// Unpublished domain metadata retained during private initialization.
export record Creation: Copy {
    /// Calling domain kept live until publication or cancellation.

    let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch err {
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let initialSlot = slots::reference(&contextSlot);
    set store.contexts[initialSlot.index].owner = slots::reference(&domainSlot);
    set store.contexts[initialSlot.index].userStack = range::Range { start: 0, end: 0 };
    let prepared = try allocate(&mut backings.pool, packages, root) catch err {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;

    }
}

/// Initialize private storage from resident packages while metadata can change.
export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
    match reservation {
        case Reservation::Held(creation) => {
            let mut storage = creation.storage;
            zero(storage.memory as u64, storage.allocation.count);
            zero(storage.kernelStack.start, storage.kernelFrames.count);
            try instances::fill(packages, &storage.graph.graph, storage.graph.base);
            capability::initialize(&mut storage.memory.table, creation.domain);
        },
    }
    match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
}

/// Initialize retained domain storage outside the metadata lock.
export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
    let mut storage = creation.storage;
    zero(storage.memory as u64, storage.allocation.count);
    zero(storage.kernelStack.start, storage.kernelFrames.count);
    try instances::fill(packages, &storage.graph.graph, storage.graph.base);
    capability::initialize(&mut storage.memory.table, creation.domain);
}

/// Check the unpublished capacity retained by one creation transaction.
fn require(store: &Store, table: &capability::Table, creation: &Creation) {
    assert table.owner == creation.owner;

    assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
}

/// Return private domain storage and reserved slots under metadata serialization.
export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
    cancelUsing(store, backings, table, reservation, frames::release);
}

/// Cancel private domain slots and transfer their storage to the release operation.
export fn cancelUsing(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            discard(&mut backings.pool, creation.storage);
            discard(&mut backings.pool, creation.storage, release);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
        },
    }

            // MPIE enables interrupts after the first return into U-mode.
            let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
            set frame.registers[1] = returnAddress();
            set frame.registers[3] = graph.table.ptr as u64;
            set store.contexts[initial.index] = Context {
                owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.reclaimed[object.index] = false;
            set store.reaping[object.index] = false;
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
            };
            set store.ancestors[object.index] = store.ancestors[creation.owner.index];
            set store.ancestors[object.index][creation.owner.index / 32] |= 1 << (creation.owner.index % 32);
            let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
            let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
            return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
                kind: abi::Kind::Domain, object: domain, rights: creation.rights,
            });

kernel/kernel/events.rad +41 -7

    state: State,
}

/// Fixed event metadata. Callers serialize all kernel-side mutations.
export record Store: Copy {
    /// Terminal events retained for publication into receiver rings.
    pending: u32,
    /// Producer metadata indexed by receiver domain.
    queues: [Queue; limits::DOMAINS],
    /// Terminal storage indexed by the terminating domain.
    terminals: [Terminal; limits::DOMAINS],
}

/// Advance a shared counter with explicit modulo arithmetic.
fn next(value: u32) -> u32 { return ((value as u64 + 1) & 0xffffffff) as u32; }

/// Initialize metadata before any queue or terminal record exists.
export fn initialize(store: &mut Store) {
    set store.pending = 0;
    for i in 0..limits::DOMAINS {
        set store.queues[i].generation = 0;
        set store.terminals[i].state = State::Free;
    }
}

        case Reservation::Held(subject) => {
            let terminal = &mut store.terminals[subject.index];
            assert terminal.subject == subject and terminal.state == State::Reserved;
            set terminal.receiver = receiver;
            set terminal.event = Event { kind, reserved: 0, code, value: abi::id(subject) };
            if active { set terminal.state = State::Pending; }
            if active { set terminal.state = State::Pending; set store.pending += 1; }
            else { set terminal.state = State::Acknowledged; }
        },
    }
}

/// Publish one retained terminal event when its receiver has ring capacity.
export fn deliver(store: &mut Store, subject: abi::Ref, ring: &mut Ring) -> bool throws (abi::Error) {
    if subject.index >= limits::DOMAINS or store.terminals[subject.index].subject <> subject
        or store.terminals[subject.index].state == State::Free { throw abi::Error::BadHandle; }
    let terminal = store.terminals[subject.index];
    if terminal.state <> State::Pending { return false; }
    try refresh(store, terminal.receiver, ring);
    let queue = &mut store.queues[terminal.receiver.index];
    if distance(queue.tail, queue.head) == CAPACITY { return false; }
    set store.terminals[subject.index].state = State::Queued;
    assert store.pending > 0;
    set store.pending -= 1;
    publish(queue, ring, terminal.event, subject);
    return true;
}

/// Publish retained terminal events in domain-index order as ring capacity permits.
export fn flush(store: &mut Store, receiver: abi::Ref, ring: &mut Ring) -> u32 throws (abi::Error) {
    try refresh(store, receiver, ring);
    let mut count: u32 = 0;
    for i in 0..limits::DOMAINS {
        if distance(store.queues[receiver.index].tail, store.queues[receiver.index].head) == CAPACITY { break; }
        if store.terminals[i].state <> State::Pending or store.terminals[i].receiver <> receiver { continue; }
        set store.terminals[i].state = State::Queued;
        let event = store.terminals[i].event;
        let subject = store.terminals[i].subject;
        publish(&mut store.queues[receiver.index], ring, event, subject);
        set count += 1;
        if try deliver(store, subject, ring) { set count += 1; }
    }
    return count;
}

/// Report whether a retained terminal identity can be released.

}

/// Close a stopped domain's queue and end all terminal references held by that receiver.
export fn close(store: &mut Store, receiver: abi::Ref) throws (abi::Error) {
    if not live(store, receiver) { throw abi::Error::BadHandle; }
    set store.queues[receiver.index].generation = 0;
    let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
    set generations[receiver.index] = receiver.generation;
    try! closeSet(store, &generations[..]);
}

/// Close selected receiver generations and acknowledge their retained terminal events.
/// Zero entries leave a receiver open. Validate all selected generations before mutation.
export fn closeSet(store: &mut Store, generations: &[u32]) throws (abi::Error) {
    if generations.len <> limits::DOMAINS { throw abi::Error::InvalidArg; }
    for i in 0..limits::DOMAINS {
        if generations[i] <> 0 and store.queues[i].generation <> generations[i] { throw abi::Error::BadHandle; }
    }
    for i in 0..limits::DOMAINS {
        if generations[i] <> 0 { set store.queues[i].generation = 0; }
    }
    for i in 0..limits::DOMAINS {
        if (store.terminals[i].state == State::Pending or store.terminals[i].state == State::Queued) and store.terminals[i].receiver == receiver {
        let terminal = store.terminals[i];
        if terminal.state <> State::Pending and terminal.state <> State::Queued { continue; }
        let receiver = terminal.receiver;
        if generations[receiver.index] == receiver.generation {
            if store.terminals[i].state == State::Pending { assert store.pending > 0; set store.pending -= 1; }
            set store.terminals[i].state = State::Acknowledged;
        }
    }
}

kernel/kernel/frames.rad +37 -0

    addresses: [u64; limits::FRAMES],
    /// True when the frame is available for reservation.
    free: [bool; limits::FRAMES],
    /// Number of valid frame-map entries.
    count: u32,
    /// Retired runs whose frames remain unavailable until reclamation.
    retired: [Run; limits::FRAMES],
    /// Index of the next retired run to reclaim.
    retiredHead: u32,
    /// Number of occupied retirement entries.
    retiredCount: u32,
}

/// Insert one range into an address-sorted prefix of fixed storage.
fn insert(ranges: &mut [range::Range], count: u32, value: range::Range) {
    let mut at = count;


/// Build a fresh frame map from complete RAM pages with no reserved byte.
/// The pool has zero valid entries if initialization fails.
export unsafe fn initialize(pool: &mut Pool, machine: &platform::Platform) throws (abi::Error) {
    set pool.count = 0;
    set pool.retiredHead = 0;
    set pool.retiredCount = 0;
    if machine.ramCount > platform::RAM_BANKS or machine.reservedCount > platform::RESERVATIONS {
        throw abi::Error::InvalidArg;
    }
    let mut reserved: [range::Range; platform::RESERVATIONS] = undefined;
    for i in 0..machine.reservedCount {

    let memory = try extent(pool, run);
    for i in run.first..run.first + run.count { if pool.free[i] { throw abi::Error::BadHandle; } }
    for i in run.first..run.first + run.count { set pool.free[i] = true; }
}

/// Transfer exclusive ownership of a committed run to bounded reclamation.
/// The caller must not release or retire this run again.
export fn retire(pool: &mut Pool, run: Run) throws (abi::Error) {
    let memory = try extent(pool, run);
    assert pool.retiredCount < pool.count;
    let tail = (pool.retiredHead + pool.retiredCount) % limits::FRAMES;
    set pool.retired[tail] = run;
    set pool.retiredCount += 1;
}

/// Return at most limit retired frames while metadata is serialized.
export fn reclaim(pool: &mut Pool, limit: u32) -> u32 {
    let mut returned: u32 = 0;
    while pool.retiredCount > 0 and returned < limit {
        let mut count = pool.retired[pool.retiredHead].count;
        if count > limit - returned { set count = limit - returned; }
        let batch = Run { first: pool.retired[pool.retiredHead].first, count };
        try! release(pool, batch);
        set pool.retired[pool.retiredHead].first += count;
        set pool.retired[pool.retiredHead].count -= count;
        set returned += count;
        if pool.retired[pool.retiredHead].count == 0 {
            set pool.retiredHead = (pool.retiredHead + 1) % limits::FRAMES;
            set pool.retiredCount -= 1;
        }
    }
    return returned;
}

/// Cancel unpublished frame ownership and restore its capacity.
export fn cancel(pool: &mut Pool, reservation: Reservation) throws (abi::Error) {
    match reservation { case Reservation::Held(run) => try release(pool, run), }
}

kernel/kernel/lifecycle.rad added +468 -0

1	+	//! Terminal notification delivery and execution eligibility.
2	+	use super::limits;
3	+	use super::slots;
4	+	use super::events;
5	+	use super::domains;
6	+	use super::budgets;
7	+	use super::abi;
8	+	use super::remote;
9	+	use super::pages;
10	+	use super::registry;
11	+	use super::loader;
12	+	use super::capability;
13	+	use super::frames;
14	+	use super::backing;
15	+	use super::interrupts;
16	+	use super::plic;
17	+
18	+	/// Words needed to track request ownership for every context slot.
19	+	constant CONTEXT_WORDS: u32 = (limits::CONTEXTS + 31) / 32;
20	+
21	+	/// Private input and optional native output retained by a loader call.
22	+	export record Loading: Copy {
23	+	/// Workspace and reserved caller handle.
24	+	input: loader::Input,
25	+	/// Output frames and package slot, when compilation has been admitted.
26	+	output: ?loader::Output,
27	+	}
28	+
29	+	/// Successful loader work ready for publication under the metadata lock.
30	+	export union LoadResult: Copy {
31	+	/// Matching immutable package already present in the registry.
32	+	Existing(abi::Ref),
33	+	/// Generated code and metadata in the retained output allocation.
34	+	Compiled(loader::Compiled),
35	+	}
36	+
37	+	/// Private storage retained by an unfinished syscall.
38	+	export union Allocation: Copy {
39	+	/// Page frames and unpublished capability metadata.
40	+	Page(pages::Allocation),
41	+	/// Additional context slot and private kernel stack.
42	+	Context(domains::ContextAllocation),
43	+	/// Reserved domain generation with its retained creation payload.
44	+	Domain(abi::Ref),
45	+	/// Private compiler workspace and unpublished output.
46	+	Load(Loading),
47	+	}
48	+
49	+	/// One unfinished allocation retained by a caller generation.
50	+	export record HeldAllocation: Copy {
51	+	/// Context whose continuation owns this allocation.
52	+	context: abi::Ref,
53	+	/// Calling domain, which can differ from the allocation target.
54	+	owner: abi::Ref,
55	+	/// Frames and metadata slots excluded from allocation.
56	+	allocation: Allocation,
57	+	}
58	+
59	+	/// Allocations retained while their contexts initialize private frames.
60	+	export record Calls: Copy {
61	+	/// One validity bit per occupied context slot.
62	+	active: [u32; CONTEXT_WORDS],
63	+	/// Reservation payloads indexed by context slot.
64	+	records: [HeldAllocation; limits::CONTEXTS],
65	+	/// Creation payloads valid while their domain slots are reserved.
66	+	creations: [domains::Creation; limits::DOMAINS],
67	+	}
68	+
69	+	/// A continuation's exclusive claim to its retained reservation.
70	+	export union Lease: Once {
71	+	/// Context generation that must take back the reservation before completion.
72	+	Held(abi::Ref),
73	+	}
74	+
75	+	/// Reservations serialized with context and physical-memory metadata.
76	+	export unsafe static CALLS: Calls = undefined;
77	+
78	+	/// Clear allocation ownership before contexts start.
79	+	export fn initialize(calls: &mut Calls) {
80	+	for i in 0..CONTEXT_WORDS { set calls.active[i] = 0; }
81	+	}
82	+
83	+	/// Retain private storage and pin the caller's kernel stack under the lock.
84	+	fn hold(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, allocation: Allocation) -> Lease {
85	+	assert context.index < limits::CONTEXTS and context.generation <> 0;
86	+	assert slots::matches(&store.contextSlots[..], context, slots::State::Live);
87	+	assert not store.contexts[context.index].reservation;
88	+	let bit = 1 << (context.index % 32);
89	+	assert (calls.active[context.index / 32] & bit) == 0;
90	+	set calls.records[context.index] = HeldAllocation { context, owner: store.contexts[context.index].owner, allocation };
91	+	set calls.active[context.index / 32] \|= bit;
92	+	set store.contexts[context.index].reservation = true;
93	+	return Lease::Held(context);
94	+	}
95	+
96	+	/// Transfer a page reservation into context-owned storage under the metadata lock.
97	+	export fn holdPage(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: pages::Reservation) -> Lease {
98	+	match reservation {
99	+	case pages::Reservation::Held(allocation) => {
100	+	assert store.contexts[context.index].owner == allocation.owner;
101	+	return hold(calls, store, context, Allocation::Page(allocation));
102	+	},
103	+	}
104	+	}
105	+
106	+	/// Transfer an unpublished context into its caller's retained storage.
107	+	export fn holdContext(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: domains::ContextReservation) -> Lease {
108	+	match reservation {
109	+	case domains::ContextReservation::Held(allocation) => return hold(calls, store, context, Allocation::Context(allocation)),
110	+	}
111	+	}
112	+
113	+	/// Retain unpublished domain storage under its reserved domain generation.
114	+	export fn holdDomain(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: domains::Reservation) -> Lease {
115	+	match reservation {
116	+	case domains::Reservation::Held(creation) => {
117	+	assert slots::matches(&store.slots[..], creation.domain, slots::State::Reserved);
118	+	assert store.contexts[context.index].owner == creation.owner;
119	+	set calls.creations[creation.domain.index] = creation;
120	+	return hold(calls, store, context, Allocation::Domain(creation.domain));
121	+	},
122	+	}
123	+	}
124	+
125	+	/// Read the private payload for an exact reserved domain generation.
126	+	fn creation(calls: &Calls, object: abi::Ref) -> domains::Creation {
127	+	assert object.index < limits::DOMAINS;
128	+	assert calls.creations[object.index].domain == object;
129	+	return calls.creations[object.index];
130	+	}
131	+
132	+	/// Initialize domain storage while the caller retains execution ownership.
133	+	export unsafe fn prepareDomain(calls: &Calls, packages: &registry::Store, lease: &Lease) throws (abi::Error) {
134	+	match lease {
135	+	case Lease::Held(context) => match held(calls, *context) {
136	+	case Allocation::Domain(object) => {
137	+	let retained = creation(calls, object);
138	+	try domains::prepareCreation(packages, &retained);
139	+	},
140	+	else => panic "domain reservation required",
141	+	},
142	+	}
143	+	}
144	+
145	+	/// Retain private loader input and pin its caller's kernel stack.
146	+	export fn holdLoad(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: loader::Reservation) -> Lease {
147	+	match reservation {
148	+	case loader::Reservation::Held(input) => {
149	+	assert store.contexts[context.index].owner == input.owner;
150	+	return hold(calls, store, context, Allocation::Load(Loading { input, output: nil }));
151	+	},
152	+	}
153	+	}
154	+
155	+	/// Read the private loader allocations through their caller's lease.
156	+	export fn load(calls: &Calls, lease: &Lease) -> Loading {
157	+	match lease {
158	+	case Lease::Held(context) => match held(calls, *context) {
159	+	case Allocation::Load(loading) => return loading,
160	+	else => panic "loader reservation required",
161	+	},
162	+	}
163	+	}
164	+
165	+	/// Attach reserved native output to its caller before compilation can start.
166	+	export fn reserveOutput(calls: &mut Calls, lease: &Lease, memory: &mut pages::Store, packages: &mut registry::Store) throws (abi::Error) {
167	+	let mut loading = load(calls, lease);
168	+	assert loading.output == nil;
169	+	let pending = try loader::reserveOutput(&mut memory.backings.pool, packages);
170	+	match pending { case loader::OutputReservation::Held(output) => { set loading.output = output; }, }
171	+	match lease { case Lease::Held(context) => { set calls.records[context.index].allocation = Allocation::Load(loading); }, }
172	+	}
173	+
174	+	/// Return all unpublished loader capacity under the metadata lock.
175	+	fn discardLoad(memory: &mut pages::Store, state: &mut loader::State, packages: &mut registry::Store,
176	+	table: &mut capability::Table, loading: Loading, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
177	+	{
178	+	if let output = loading.output { loader::cancelOutputUsing(&mut memory.backings.pool, packages, loader::OutputReservation::Held(output), release); }
179	+	loader::cancelUsing(state, memory, table, loader::Reservation::Held(loading.input), release);
180	+	}
181	+
182	+	/// Cancel a live continuation's load and release its stack pin under the lock.
183	+	export unsafe fn cancelLoad(calls: &mut Calls, store: &mut domains::Store, memory: &mut pages::Store,
184	+	state: &mut loader::State, packages: &mut registry::Store, lease: Lease)
185	+	{
186	+	match take(calls, store, lease) {
187	+	case Allocation::Load(loading) => {
188	+	let mut owner = try! domains::get(store, loading.input.owner);
189	+	discardLoad(memory, state, packages, &mut owner.memory.table, loading, frames::release);
190	+	},
191	+	else => panic "loader reservation required",
192	+	}
193	+	}
194	+
195	+	/// Publish successful loader work and release caller-owned temporary storage.
196	+	export unsafe fn finishLoad(calls: &mut Calls, store: &mut domains::Store, memory: &mut pages::Store,
197	+	state: &mut loader::State, packages: &mut registry::Store, lease: Lease, result: LoadResult) -> abi::Handle throws (abi::Error)
198	+	{
199	+	match take(calls, store, lease) {
200	+	case Allocation::Load(loading) => {
201	+	let mut owner = try! domains::get(store, loading.input.owner);
202	+	if owner.state == domains::Lifecycle::Dead {
203	+	discardLoad(memory, state, packages, &mut owner.memory.table, loading, frames::release);
204	+	throw abi::Error::BadHandle;
205	+	}
206	+	let mut object: abi::Ref = undefined;
207	+	match result {
208	+	case LoadResult::Existing(existing) => { assert loading.output == nil; set object = existing; },
209	+	case LoadResult::Compiled(compiled) => {
210	+	let output = loading.output else panic "missing loader output";
211	+	set object = loader::publishOutput(state, &mut memory.backings.pool, packages,
212	+	loader::OutputReservation::Held(output), compiled);
213	+	},
214	+	}
215	+	return try loader::finish(state, memory, &mut owner.memory.table, loader::Reservation::Held(loading.input), object);
216	+	},
217	+	else => panic "loader reservation required",
218	+	}
219	+	}
220	+
221	+	/// Read retained storage through its exact caller generation.
222	+	fn held(calls: &Calls, context: abi::Ref) -> Allocation {
223	+	assert context.index < limits::CONTEXTS;
224	+	assert (calls.active[context.index / 32] & (1 << (context.index % 32))) <> 0;
225	+	assert calls.records[context.index].context == context;
226	+	return calls.records[context.index].allocation;
227	+	}
228	+
229	+	/// Clear private page frames while the executing context retains hart ownership.
230	+	export fn clearPage(calls: &Calls, lease: &Lease) {
231	+	match lease {
232	+	case Lease::Held(context) => match held(calls, *context) {
233	+	case Allocation::Page(allocation) => pages::clearAllocation(&allocation),
234	+	else => panic "page reservation required",
235	+	},
236	+	}
237	+	}
238	+
239	+	/// Clear a private kernel stack while the caller retains hart ownership.
240	+	export fn clearContext(calls: &Calls, lease: &Lease) {
241	+	match lease {
242	+	case Lease::Held(context) => match held(calls, *context) {
243	+	case Allocation::Context(allocation) => domains::contextClearAllocation(&allocation),
244	+	else => panic "context reservation required",
245	+	},
246	+	}
247	+	}
248	+
249	+	/// Remove retained ownership before serialized publication or cancellation.
250	+	fn take(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> Allocation {
251	+	match lease {
252	+	case Lease::Held(context) => {
253	+	let allocation = held(calls, context);
254	+	set calls.active[context.index / 32] &= ~(1 << (context.index % 32));
255	+	assert slots::matches(&store.contextSlots[..], context, slots::State::Live);
256	+	set store.contexts[context.index].reservation = false;
257	+	return allocation;
258	+	},
259	+	}
260	+	}
261	+
262	+	/// Return a page reservation for publication or cancellation under the lock.
263	+	export fn takePage(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> pages::Reservation {
264	+	match take(calls, store, lease) {
265	+	case Allocation::Page(allocation) => return pages::Reservation::Held(allocation),
266	+	else => panic "page reservation required",
267	+	}
268	+	}
269	+
270	+	/// Return a context reservation for publication or cancellation under the lock.
271	+	export fn takeContext(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> domains::ContextReservation {
272	+	match take(calls, store, lease) {
273	+	case Allocation::Context(allocation) => return domains::ContextReservation::Held(allocation),
274	+	else => panic "context reservation required",
275	+	}
276	+	}
277	+
278	+	/// Return domain storage for publication or cancellation under the lock.
279	+	export fn takeDomain(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> domains::Reservation {
280	+	match take(calls, store, lease) {
281	+	case Allocation::Domain(object) => return domains::Reservation::Held(creation(calls, object)),
282	+	else => panic "domain reservation required",
283	+	}
284	+	}
285	+
286	+	/// Retire one dead caller's private storage after execution ownership ends.
287	+	export unsafe fn cancelAllocations(calls: &mut Calls, store: &mut domains::Store, memory: &mut pages::Store,
288	+	state: &mut loader::State, packages: &mut registry::Store) -> bool
289	+	{
290	+	for word in 0..CONTEXT_WORDS {
291	+	if calls.active[word] == 0 { continue; }
292	+	for offset in 0..32 as u32 {
293	+	let bit: u32 = 1 << offset;
294	+	if (calls.active[word] & bit) == 0 { continue; }
295	+	let index = word * 32 + offset;
296	+	let retained = calls.records[index];
297	+	let context = try domains::context(store, retained.owner, retained.context) catch { continue; };
298	+	if context.hart <> nil or context.state <> domains::ContextState::Stopped { continue; }
299	+	let mut owner = try! domains::get(store, retained.owner);
300	+	if owner.state <> domains::Lifecycle::Dead { continue; }
301	+	set calls.active[word] &= ~bit;
302	+	match retained.allocation {
303	+	case Allocation::Load(loading) => discardLoad(memory, state, packages, &mut owner.memory.table, loading, frames::retire),
304	+	case Allocation::Page(allocation) => pages::cancelUsing(memory, &mut owner.memory.table, pages::Reservation::Held(allocation), frames::retire),
305	+	case Allocation::Context(allocation) => domains::contextCancelUsing(store, memory, domains::ContextReservation::Held(allocation), frames::retire),
306	+	case Allocation::Domain(object) => domains::cancelUsing(store, &mut memory.backings,
307	+	&mut owner.memory.table, domains::Reservation::Held(creation(calls, object)), frames::retire),
308	+	}
309	+	set store.contexts[index].reservation = false;
310	+	return true;
311	+	}
312	+	}
313	+	return false;
314	+	}
315	+
316	+	/// Release stopped callers' remote work after hart ownership ends.
317	+	/// Taken requests keep the context pinned until the target acknowledges them.
318	+	export fn cancelRequests(store: &mut domains::Store, requests: &mut remote::Store) {
319	+	let mut touched: [u32; CONTEXT_WORDS] = [0; CONTEXT_WORDS];
320	+	let mut retained: [u32; CONTEXT_WORDS] = [0; CONTEXT_WORDS];
321	+	let mut cancelled = false;
322	+	for i in 0..limits::REMOTE_REQUESTS {
323	+	if requests.slots[i].state <> slots::State::Live { continue; }
324	+	let request = requests.requests[i];
325	+	let mut caller: ?abi::Ref = nil;
326	+	match request.action {
327	+	case remote::Action::Budget(call) => { set caller = call.context; },
328	+	case remote::Action::Ready(context) => { set caller = context; },
329	+	else => continue,
330	+	}
331	+	let object = caller else { continue; };
332	+	if not slots::matches(&store.contextSlots[..], object, slots::State::Live) { continue; }
333	+	if store.contexts[object.index].hart <> nil or store.contexts[object.index].state <> domains::ContextState::Stopped { continue; }
334	+	if not slots::matches(&store.slots[..], store.contexts[object.index].owner, slots::State::Live)
335	+	or store.records[store.contexts[object.index].owner.index].state <> domains::Lifecycle::Dead { continue; }
336	+	let word = object.index / 32;
337	+	let bit = 1 << (object.index % 32);
338	+	set touched[word] \|= bit;
339	+	if request.state == remote::State::Delivered {
340	+	set retained[word] \|= bit;
341	+	set store.contexts[object.index].pending = abi::Ref { index: i, generation: requests.slots[i].generation };
342	+	continue;
343	+	}
344	+	try! remote::cancel(requests, request.source, abi::Ref { index: i, generation: requests.slots[i].generation });
345	+	set cancelled = true;
346	+	}
347	+	if not cancelled { return; }
348	+	for i in 0..limits::CONTEXTS {
349	+	let bit = 1 << (i % 32);
350	+	if (touched[i / 32] & bit) <> 0 and (retained[i / 32] & bit) == 0 {
351	+	set store.contexts[i].pending = nil;
352	+	}
353	+	}
354	+	}
355	+
356	+	/// Advance one eligible domain from a rotating slot cursor.
357	+	export unsafe fn reapNext(store: &mut domains::Store, memory: &mut pages::Store, windows: &mut budgets::Store,
358	+	irqs: &mut interrupts::Store) -> bool
359	+	{
360	+	if store.dead == 0 { return false; }
361	+	let mut checked = false;
362	+	let mut blocked: [bool; limits::DOMAINS] = [false; limits::DOMAINS];
363	+	for offset in 0..limits::DOMAINS {
364	+	let index = (store.reapCursor + offset) % limits::DOMAINS;
365	+	if store.slots[index].state <> slots::State::Live or store.records[index].state <> domains::Lifecycle::Dead { continue; }
366	+	let owner = abi::Ref { index, generation: store.slots[index].generation };
367	+	if store.reclaimed[index] {
368	+	if not try! events::acknowledged(&store.events, owner) { continue; }
369	+	} else if not store.reaping[index] {
370	+	if not checked {
371	+	for i in 0..limits::CONTEXTS {
372	+	let state = store.contextSlots[i].state;
373	+	if state <> slots::State::Live and state <> slots::State::Reserved { continue; }
374	+	if state == slots::State::Reserved or store.contexts[i].state <> domains::ContextState::Stopped
375	+	or store.contexts[i].hart <> nil or store.contexts[i].pending <> nil or store.contexts[i].reservation {
376	+	set blocked[store.contexts[i].owner.index] = true;
377	+	}
378	+	}
379	+	set checked = true;
380	+	}
381	+	if blocked[index] { continue; }
382	+	}
383	+	if try! reap(store, memory, windows, irqs, owner) {
384	+	set store.reapCursor = (index + 1) % limits::DOMAINS;
385	+	return true;
386	+	}
387	+	}
388	+	return false;
389	+	}
390	+
391	+	/// Advance one dead domain's resource teardown under the metadata lock.
392	+	/// Returns false while execution ownership or terminal acknowledgement is pending.
393	+	export unsafe fn reap(store: &mut domains::Store, memory: &mut pages::Store, windows: &mut budgets::Store,
394	+	irqs: &mut interrupts::Store, owner: abi::Ref) -> bool throws (abi::Error)
395	+	{
396	+	if not slots::matches(&store.slots[..], owner, slots::State::Live) { throw abi::Error::BadHandle; }
397	+	if store.reclaimed[owner.index] {
398	+	if not try events::acknowledged(&store.events, owner) { return false; }
399	+	try! domains::releaseIdentity(store, owner);
400	+	return true;
401	+	}
402	+	if not store.reaping[owner.index] {
403	+	if not try domains::quiescent(store, owner) { return false; }
404	+	set store.reaping[owner.index] = true;
405	+	}
406	+	let mut domain = try! domains::get(store, owner);
407	+	for i in 0..limits::HANDLES {
408	+	let slot = domain.memory.table.slots[i];
409	+	if slot.state == slots::State::Reserved { return false; }
410	+	if slot.state <> slots::State::Live { continue; }
411	+	let entry = domain.memory.table.entries[i];
412	+	let handle = try! abi::handle(entry.kind, abi::Ref { index: i, generation: slot.generation });
413	+	match entry.kind {
414	+	case abi::Kind::Page => try! pages::drop(memory, &mut domain.memory.table, handle),
415	+	case abi::Kind::Budget => try! budgets::drop(windows, &mut domain.memory.table, handle),
416	+	case abi::Kind::Interrupt => {
417	+	let number = try! interrupts::query(irqs, &domain.memory.table, handle);
418	+	plic::mask(number);
419	+	try! interrupts::drop(irqs, &mut domain.memory.table, handle);
420	+	},
421	+	else => { let removed = try! capability::invalidate(&mut domain.memory.table, handle); },
422	+	}
423	+	return true;
424	+	}
425	+	if memory.backings.domains[owner.index] <> 0 {
426	+	let complete = try! backing::endDomainStep(&mut memory.backings, owner, 64);
427	+	return true;
428	+	}
429	+	if try! domains::retireContext(store, &mut memory.backings.pool, owner) { return true; }
430	+	try! domains::retireStorage(store, &mut memory.backings, owner);
431	+	return true;
432	+	}
433	+
434	+	/// Publish retained terminal events and report receiver harts under the metadata lock.
435	+	export unsafe fn service(store: &mut domains::Store, windows: &budgets::Store, now: u64) -> u32 {
436	+	if store.events.pending == 0 { return 0; }
437	+	let mut notified: [bool; limits::DOMAINS] = [false; limits::DOMAINS];
438	+	let mut changed = false;
439	+	for i in 0..limits::DOMAINS {
440	+	let terminal = store.events.terminals[i];
441	+	if terminal.state <> events::State::Pending { continue; }
442	+	let mut receiver = try domains::get(store, terminal.receiver) catch { continue; };
443	+	if receiver.state == domains::Lifecycle::Dead { continue; }
444	+	let delivered = try events::deliver(&mut store.events, terminal.subject, &mut receiver.memory.ring) catch { false };
445	+	if delivered { set notified[terminal.receiver.index] = true; set changed = true; }
446	+	}
447	+	if not changed { return 0; }
448	+	for i in 0..limits::CONTEXTS {
449	+	if store.contextSlots[i].state <> slots::State::Live { continue; }
450	+	if notified[store.contexts[i].owner.index] and store.contexts[i].state == domains::ContextState::Waiting {
451	+	set store.contexts[i].state = domains::ContextState::Ready;
452	+	}
453	+	}
454	+	let mut harts: u32 = 0;
455	+	for i in 0..limits::BUDGETS {
456	+	if windows.slots[i].state <> slots::State::Live { continue; }
457	+	let window = windows.windows[i];
458	+	if not slots::matches(&store.slots[..], window.owner, slots::State::Live)
459	+	or not notified[window.owner.index] or window.end <= now { continue; }
460	+	let context = window.context else { continue; };
461	+	if not slots::matches(&store.contextSlots[..], context, slots::State::Live) { continue; }
462	+	if store.contexts[context.index].owner == window.owner
463	+	and store.contexts[context.index].state <> domains::ContextState::Stopped {
464	+	set harts \|= 1 << window.hart;
465	+	}
466	+	}
467	+	return harts;
468	+	}

kernel/kernel/loader.rad +52 -18

    return Lease::Held(frames::commit(reservation));
}

/// Return temporary frames and release the compiler workspace under metadata serialization.
export fn release(state: &mut State, pool: &mut frames::Pool, lease: Lease) {
    releaseUsing(state, pool, lease, frames::release);
}

/// Transfer temporary frames to their release operation and clear workspace ownership.
fn releaseUsing(state: &mut State, pool: &mut frames::Pool, lease: Lease,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match lease {
        case Lease::Held(run) => { try! frames::release(pool, run); },
        case Lease::Held(run) => { try! release(pool, run); },
    }
    set state.owner = abi::Ref { index: 0, generation: 0 };
    set state.busy = false;
}


}

/// Snapshot and decode trusted input into the held private workspace.
/// The decoded package stays valid until this reservation is completed or cancelled.
export unsafe fn decode(reservation: &Reservation) -> binary::Package throws (abi::Error) {
    match reservation {
        case Reservation::Held(input) => {
            let mut work = workspace(input.work);
            try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
            let mut decoder = alloc::new(&mut work.decoded[..]);
            return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
                if err == binary::Error::Storage { throw abi::Error::Exhausted; }
                throw abi::Error::VerifyFailed;
            };
        },
    }
    match reservation { case Reservation::Held(input) => return try decodeInput(input), }
}

/// Snapshot retained input and decode it in its private workspace.
export unsafe fn decodeInput(input: &Input) -> binary::Package throws (abi::Error) {
    let mut work = workspace(input.work);
    try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
    let mut decoder = alloc::new(&mut work.decoded[..]);
    return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
        if err == binary::Error::Storage { throw abi::Error::Exhausted; }
        throw abi::Error::VerifyFailed;
    };
}

/// Reserved native output and its immutable physical extents.
export record Output: Copy {
    /// Reserved package slot selected for generated state references.

    -> Compiled throws (abi::Error)
{
    match reservation {
        case Reservation::Held(source) => match output {
            case OutputReservation::Held(target) => {
                let work = workspace(source.work);
                return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
                return try compileInput(packages, source, input, target);
            },
        },
    }
}

/// Generate native code into retained output while the caller owns its workspace.
export unsafe fn compileInput(packages: &registry::Store, source: &Input, input: &binary::Package, target: &Output)
    -> Compiled throws (abi::Error)
{
    let work = workspace(source.work);
    return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
}

/// Return unpublished output frames and its package slot under serialization.
export fn cancelOutput(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation) {
    cancelOutputUsing(pool, packages, reservation, frames::release);
}

/// Cancel a package slot and transfer private output to the release operation.
export fn cancelOutputUsing(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case OutputReservation::Held(output) => {
            assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
            try! frames::release(pool, output.metadata);
            try! frames::release(pool, output.code);
            try! release(pool, output.metadata);
            try! release(pool, output.code);
            registry::cancel(packages, slots::Reservation::Held(output.object));
        },
    }
}


}

/// Compare the snapshot with immutable resident content while the workspace is held.
export unsafe fn identify(packages: &registry::Store, reservation: &Reservation, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
    match reservation {
        case Reservation::Held(source) => return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]),
        case Reservation::Held(source) => return try identifyInput(packages, source, input),
    }
}

/// Compare retained source bytes with immutable resident package content.
export unsafe fn identifyInput(packages: &registry::Store, source: &Input, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
    return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]);
}

/// Reserve an Image handle and exclusive workspace under metadata serialization.
export unsafe fn reserve(state: &mut State, store: &mut pages::Store,
    table: &mut capability::Table, request: Request) -> Reservation throws (abi::Error)
{
    let address = try source(store, table, request);

    assert slots::matches(&table.slots[..], input.handle, slots::State::Reserved);
}

/// Return the workspace and reserved handle under metadata serialization.
export fn cancel(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation) {
    cancelUsing(state, store, table, reservation, frames::release);
}

/// Cancel a reserved Image handle and transfer its workspace to the release operation.
export fn cancelUsing(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(input) => {
            require(state, table, &input);
            release(state, &mut store.backings.pool, Lease::Held(input.frames));
            releaseUsing(state, &mut store.backings.pool, Lease::Held(input.frames), release);
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(input.handle));
        },
    }
}


kernel/kernel/pages.rad +12 -2

    }
}

/// Clear private frames while shared metadata can be used by other harts.
export fn clear(reservation: &Reservation) {
    match reservation { case Reservation::Held(allocation) => zero(allocation.base, allocation.run.count), }
    match reservation { case Reservation::Held(allocation) => clearAllocation(allocation), }
}

/// Clear frames held by an unpublished allocation's exclusive owner.
export fn clearAllocation(allocation: &Allocation) { zero(allocation.base, allocation.run.count); }

/// Check reserved generations before publishing or cancelling an allocation.
fn require(store: &Store, table: &capability::Table, allocation: &Allocation) {
    assert table.owner == allocation.owner;
    assert slots::matches(&table.slots[..], allocation.handle, slots::State::Reserved);
    assert slots::matches(&store.slots[..], allocation.page, slots::State::Reserved);

    }
}

/// Return private allocation capacity under metadata serialization.
export fn cancel(store: &mut Store, table: &mut capability::Table, reservation: Reservation) {
    cancelUsing(store, table, reservation, frames::release);
}

/// Cancel unpublished slots and transfer their frames to the selected release operation.
export fn cancelUsing(store: &mut Store, table: &mut capability::Table, reservation: Reservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(allocation) => {
            require(store, table, &allocation);
            try! frames::release(&mut store.backings.pool, allocation.run);
            try! release(&mut store.backings.pool, allocation.run);
            try! slots::cancel(&mut store.backings.slots[..], slots::Reservation::Held(allocation.backing));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(allocation.page));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
        },
    }

kernel/kernel/sys.rad +22 -0

/// Copy authority to a domain with optional rights narrowing.
export fn capabilityGrant(handle: abi::Handle, target: abi::Handle, rights: u64) -> abi::Handle throws (abi::Error) {
    return abi::Handle(try result(ecall(abi::Operation::CapabilityGrant as u32, *handle as i64, *target as i64, rights as i64, 0)));
}

/// Set a child's lifecycle receiver with Destroy and Wake authority.
export fn domainReparent(domain: abi::Handle, parent: abi::Handle) throws (abi::Error) {
    try result(ecall(abi::Operation::DomainReparent as u32, *domain as i64, *parent as i64, 0, 0));
}

/// Load a trusted binary package from a readable Page range.
export fn imageLoad(authority: abi::Handle, source: abi::Handle, offset: u64, length: u64) -> abi::Handle throws (abi::Error) {
    return abi::Handle(try result(ecall(abi::Operation::ImageLoad as u32, *authority as i64, *source as i64, offset as i64, length as i64)));
}

/// Destroy the named domain or the calling domain through the zero sentinel.
export fn domainDestroy(handle: abi::Handle, flags: u64) throws (abi::Error) {
    try result(ecall(abi::Operation::DomainDestroy as u32, *handle as i64, flags as i64, 0, 0));
}

/// End the calling domain and report its normal exit status to its parent.
export fn exit(status: u32) -> ! {
    ecall(abi::Operation::Exit as u32, status as i64, 0, 0, 0);
    panic "exit returned";
}

/// End the calling domain and report an explicit-abort fault to its parent.
export fn abort() -> ! {
    ecall(abi::Operation::Abort as u32, 0, 0, 0, 0);
    panic "abort returned";
}

/// Read resident package metadata through a readable Image capability.
export fn queryImage(handle: abi::Handle) -> abi::ImageInfo throws (abi::Error) {
    unsafe {
        let mut info: abi::ImageInfo = undefined;
        try result(ecall(abi::Operation::QueryImage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::ImageInfo) as i64, 0));

kernel/kernel/tests.rad +1 -0

export mod interrupts;
export mod registry;
export mod loader;
export mod instances;
export mod domains;
export mod lifecycle;
export mod budgets;
export mod dispatch;
export mod calls;
export mod timers;
export mod remote;

kernel/kernel/tests/backing.rad +58 -0

    let reservation = try! frames::reserve(&mut STORE.pool, 4);
    let run = frames::commit(reservation);
    return backing::publish(&mut STORE, slot, run, abi::Ref { index: 0, generation: 1 });
}

/// Exposure teardown advances in bounded steps and retains shared allocations.
@test unsafe fn boundedEnd() throws (testing::TestError) {
    initialize();
    let owner = abi::Ref { index: 0, generation: 1 };
    let receiver = abi::Ref { index: 1, generation: 1 };
    let mut objects: [abi::Ref; 3] = undefined;
    for i in 0..3 {
        let slot = try! slots::reserve(&mut STORE.slots[..]);
        let run = frames::commit(try! frames::reserve(&mut STORE.pool, 1));
        set objects[i] = backing::publish(&mut STORE, slot, run, owner);
    }
    assert STORE.used == 3;
    try! backing::expose(&mut STORE, objects[1], receiver);
    try! backing::releasePage(&mut STORE, objects[0]);
    try! backing::releasePage(&mut STORE, objects[1]);
    let mut invalid = false;
    try backing::endDomainStep(&mut STORE, owner, 0) catch error {
        assert error == abi::Error::InvalidArg; set invalid = true;
    };
    assert invalid and backing::domainLive(&STORE, owner);
    assert not try! backing::endDomainStep(&mut STORE, owner, 1);
    assert not backing::domainLive(&STORE, owner);
    assert STORE.slots[objects[0].index].state == slots::State::Free;
    assert STORE.pool.retiredCount == 1 and not STORE.pool.free[0];
    assert STORE.records[objects[1].index].exposed == 2;
    let mut busy = false;
    try backing::registerDomain(&mut STORE, abi::Ref { index: 0, generation: 2 }) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy;
    let mut denied = false;
    try backing::expose(&mut STORE, objects[2], owner) catch error {
        assert error == abi::Error::BadHandle; set denied = true;
    };
    assert denied;
    assert not try! backing::endDomainStep(&mut STORE, owner, 1);
    assert STORE.records[objects[1].index].exposed == 1;
    assert STORE.records[objects[2].index].exposed == 1;
    while not try! backing::endDomainStep(&mut STORE, owner, 64) {}
    assert STORE.records[objects[2].index].exposed == 0 and STORE.records[objects[2].index].pages == 1;
    assert STORE.pool.retiredCount == 1;
    assert frames::reclaim(&mut STORE.pool, 1) == 1 and STORE.pool.free[0];
    assert not STORE.pool.free[1] and not STORE.pool.free[2];
    let next = abi::Ref { index: 0, generation: 2 };
    try! backing::registerDomain(&mut STORE, next);
    assert backing::domainLive(&STORE, next);
    let mut stale = false;
    try backing::endDomainStep(&mut STORE, owner, 64) catch error {
        assert error == abi::Error::BadHandle; set stale = true;
    };
    assert stale and backing::domainLive(&STORE, next);
    try! backing::releasePage(&mut STORE, objects[2]);
    assert STORE.pool.free[2];
    while not try! backing::endDomainStep(&mut STORE, receiver, 64) {}
    assert STORE.pool.retiredCount == 1 and not STORE.pool.free[1];
    assert frames::reclaim(&mut STORE.pool, 1) == 1 and STORE.pool.free[1];
}

/// Dropping all page handles preserves capacity while an exposed domain lives.
@test unsafe fn exposures() throws (testing::TestError) {
    initialize();
    let object = allocate();
    let receiver = abi::Ref { index: 1, generation: 1 };

kernel/kernel/tests/budgets.rad +15 -2

    let source = abi::Ref { index: 7, generation: 1 };
    set STORE.windows[entry.object.index].context = source;
    let current = abi::CurrentContextInfo { context: abi::id(source), hart: 0, budget: *budget };
    let before = STORE.windows[entry.object.index];
    let mut rejected: u32 = 0;
    set DOMAINS.records[permission.object.index].state = domains::Lifecycle::Dead;
    for phase in 0..3 {
        if phase == 1 { set DOMAINS.reclaimed[permission.object.index] = true; }
        if phase == 2 { set DOMAINS.slots[permission.object.index].state = slots::State::Free; }
        try budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30) catch error {
            assert error == abi::Error::BadHandle; set rejected += 1;
        };
        assert STORE.windows[entry.object.index] == before;
        assert try! capability::get(&TABLE, budget) == entry;
    }
    set DOMAINS.slots[permission.object.index].state = slots::State::Live;
    set DOMAINS.reclaimed[permission.object.index] = false;
    set DOMAINS.records[permission.object.index].state = domains::Lifecycle::Pending;
    try budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30) catch error {
        assert (error == abi::Error::Busy); set rejected += 1;
    };
    set DOMAINS.records[permission.object.index].state = domains::Lifecycle::Active;
    set DOMAINS.contexts[initial.index].state = domains::ContextState::Waiting;

    set TABLE.entries[targetSlot.object.index].rights = permission.rights;
    for i in 0..MEMORY.table.slots.len { set MEMORY.table.slots[i].state = slots::State::Retired; }
    try budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30) catch error {
        assert (error == abi::Error::Busy); set rejected += 1;
    };
    assert (rejected == 7 and STORE.windows[entry.object.index] == before);
    assert (rejected == 10 and STORE.windows[entry.object.index] == before);
    assert (try! capability::get(&TABLE, budget) == entry);
    capability::initialize(&mut MEMORY.table, permission.object);
    try! budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30);
    let after = STORE.windows[entry.object.index];
    assert (after.start == 30 and after.end == 100 and after.hart == 0);

    let child = slots::reference(&childSlot);
    capability::initialize(&mut MEMORY.table, child);
    let contextSlot = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
    let context = slots::reference(&contextSlot);
    set DOMAINS.contexts[context.index] = domains::Context {
        owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil,
        owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil, reservation: false,
        kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },
        userStack: range::Range { start: 0, end: 0 },
        frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
    };
    set DOMAINS.records[child.index] = domains::Domain {

kernel/kernel/tests/calls.rad +206 -0

use kernel::limits;
use kernel::trap;
use kernel::budgets;
use kernel::remote;
use kernel::interrupts;
use kernel::events;

/// Event storage for administrative destruction targets.
unsafe static TARGETS: [domains::Memory; 2] = undefined;

/// Retained dead-domain records cannot be queried or delegated through handles.
@test unsafe fn deadHandles() throws (testing::TestError) {
    let owner = initialize();
    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
    let object = try! slots::commit(&mut domains::STORE.slots[..], pending);
    try! backing::registerDomain(&mut pages::STORE.backings, object);
    set domains::STORE.records[object.index].state = domains::Lifecycle::Dead;
    set domains::STORE.records[object.index].creator = owner;
    set domains::STORE.records[object.index].parent = nil;
    set domains::STORE.records[object.index].initial = object;
    set domains::STORE.contextSlots[object.index] = slots::Slot { generation: object.generation, state: slots::State::Live };
    set domains::STORE.contexts[object.index].owner = object;
    set domains::STORE.contexts[object.index].state = domains::ContextState::Stopped;
    set domains::STORE.contexts[object.index].hart = nil;
    set domains::STORE.contexts[object.index].userStack = { start: 0, end: 0 };
    for operation in [47 as u64, 48, 62, 10, 12, 21, 61, 20, 30, 51] {
        let mut kind = abi::Kind::Domain;
        if operation == 48 { set kind = abi::Kind::Events; }
        let handle = try! capability::install(&mut MEMORY.table, capability::Entry {
            kind: kind, object: object,
            rights: abi::Rights(abi::GRANT | abi::EXECUTE | abi::DESTROY | abi::CREATE | abi::ALLOCATE),
        });
        let mut rejected = false;
        let mut argument: u64 = 0;
        if operation == 61 { set argument = abi::id(object); }
        try calls::invoke(owner, operation, &[*handle, argument, 0, 0], 0) catch error {
            assert error == abi::Error::BadHandle;
            set rejected = true;
        };
        assert rejected;
        assert (try! capability::get(&MEMORY.table, handle)).object == object;
    }
}

/// Administrative destruction checks authority and follows creation ancestry only on request.
@test unsafe fn administrativeTermination() throws (testing::TestError) {
    for flags in [0 as u64, abi::CASCADE] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let mut objects: [abi::Ref; 2] = undefined;
        for i in 0..2 {
            let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
            let object = try! slots::commit(&mut domains::STORE.slots[..], pending);
            set objects[i] = object;
            set domains::STORE.records[object.index].state = domains::Lifecycle::Pending;
            set domains::STORE.records[object.index].creator = owner;
            set domains::STORE.records[object.index].parent = owner;
            set domains::STORE.records[object.index].memory = &mut TARGETS[i];
            try! events::open(&mut domains::STORE.events, object, &mut TARGETS[i].ring);
            set domains::STORE.contextSlots[object.index] = slots::Slot { generation: 1, state: slots::State::Live };
            set domains::STORE.contexts[object.index].owner = object;
            set domains::STORE.contexts[object.index].state = domains::ContextState::Ready;
            set domains::STORE.contexts[object.index].hart = 1 + 2 * i;
        }
        set domains::STORE.records[objects[1].index].creator = objects[0];
        set domains::STORE.records[objects[1].index].parent = objects[0];
        set domains::STORE.ancestors[objects[1].index][0] = 1 << objects[0].index;
        let handle = try! capability::install(&mut MEMORY.table, capability::Entry {
            kind: abi::Kind::Domain, object: objects[0], rights: abi::Rights(0),
        });
        let mut denied = false;
        try calls::termination(owner, 22, &[*handle, flags, 0, 0]) catch error {
            assert error == abi::Error::Denied; set denied = true;
        };
        assert denied and domains::STORE.dead == 0;
        let decoded = try! abi::decode(handle);
        set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::DESTROY);
        let mut invalid = false;
        try calls::termination(owner, 22, &[*handle, 2, 0, 0]) catch error {
            assert error == abi::Error::InvalidArg; set invalid = true;
        };
        assert invalid and domains::STORE.dead == 0;
        let stopped = try! calls::termination(owner, 22, &[*handle, flags, 0, 0]);
        assert domains::STORE.events.terminals[objects[0].index].event.code == 0;
        if flags == 0 {
            assert stopped == 2 and domains::STORE.records[objects[1].index].state == domains::Lifecycle::Pending;
            assert domains::STORE.records[objects[1].index].parent == nil;
        } else {
            assert stopped == 10 and domains::STORE.records[objects[1].index].state == domains::Lifecycle::Dead;
        }
        let mut dead = false;
        try calls::termination(owner, 22, &[*handle, 0, 0, 0]) catch error {
            assert error == abi::Error::BadHandle; set dead = true;
        };
        assert dead;
    }
}

/// Self termination validates arguments and reports the requested terminal kind.
@test unsafe fn terminationCalls() throws (testing::TestError) {
    for operation in [22 as u64, 43, 49] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let pending = try! slots::reserve(&mut domains::STORE.contextSlots[..]);
        let context = try! slots::commit(&mut domains::STORE.contextSlots[..], pending);
        set domains::STORE.contexts[context.index].owner = owner;
        set domains::STORE.contexts[context.index].state = domains::ContextState::Ready;
        set domains::STORE.contexts[context.index].hart = 2;
        let mut invalid = false;
        try calls::termination(owner, operation, &[0, 0, 1, 0]) catch error {
            assert error == abi::Error::InvalidArg; set invalid = true;
        };
        assert invalid and domains::STORE.records[owner.index].state == domains::Lifecycle::Active;
        let mut code: u64 = 0;
        if operation == 49 { set code = 0xffffffff; }
        assert try! calls::termination(owner, operation, &[code, 0, 0, 0]) == 4;
        let terminal = domains::STORE.events.terminals[owner.index];
        assert terminal.subject == owner;
        if operation == 43 {
            assert terminal.event.kind == events::FAULT and terminal.event.code == abi::FAULT_ABORT;
        } else {
            assert terminal.event.kind == events::CHILD_EXIT and terminal.event.code == code as u32;
        }
        assert domains::STORE.contexts[context.index].state == domains::ContextState::Stopped;
        assert domains::STORE.contexts[context.index].hart == 2;
    }
    let owner = initialize();
    for arguments in &[[0 as u64, 1, 0, 0], [0, 2, 0, 0]] {
        let mut invalid = false;
        try calls::termination(owner, 22, &arguments[..]) catch error {
            assert error == abi::Error::InvalidArg; set invalid = true;
        };
        assert invalid;
    }
    let mut overflow = false;
    try calls::termination(owner, 49, &[0x100000000, 0, 0, 0]) catch error {
        assert error == abi::Error::InvalidArg; set overflow = true;
    };
    assert overflow;
}

/// Parenting requires both capabilities and preserves creation ancestry.
@test unsafe fn reparent() throws (testing::TestError) {
    let owner = initialize();
    let childSlot = try! slots::reserve(&mut domains::STORE.slots[..]);
    let child = try! slots::commit(&mut domains::STORE.slots[..], childSlot);
    let parentSlot = try! slots::reserve(&mut domains::STORE.slots[..]);
    let parent = try! slots::commit(&mut domains::STORE.slots[..], parentSlot);
    set domains::STORE.records[child.index].state = domains::Lifecycle::Pending;
    set domains::STORE.records[child.index].creator = owner;
    set domains::STORE.records[child.index].parent = owner;
    set domains::STORE.records[parent.index].state = domains::Lifecycle::Active;
    let childHandle = try! capability::install(&mut MEMORY.table, capability::Entry {
        kind: abi::Kind::Domain, object: child, rights: abi::Rights(abi::DESTROY),
    });
    let parentHandle = try! capability::install(&mut MEMORY.table, capability::Entry {
        kind: abi::Kind::Domain, object: parent, rights: abi::Rights(abi::WAKE),
    });
    let changed = try! calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0);
    assert changed == 0 and domains::STORE.records[child.index].parent == parent;
    assert domains::STORE.records[child.index].creator == owner;
    let repeated = try! calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0);
    assert repeated == 0;
    let childCap = try! abi::decode(childHandle);
    let parentCap = try! abi::decode(parentHandle);
    let mut denied: u32 = 0;
    for index in &[childCap.object.index, parentCap.object.index] {
        let rights = MEMORY.table.entries[index].rights;
        set MEMORY.table.entries[index].rights = abi::Rights(0);
        try calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0) catch error {
            assert error == abi::Error::Denied; set denied += 1;
        };
        set MEMORY.table.entries[index].rights = rights;
    }
    assert denied == 2 and domains::STORE.records[child.index].parent == parent;
    let mut invalid: u32 = 0;
    for arguments in &[[*childHandle, *parentHandle, 1, 0], [*childHandle, *parentHandle, 0, 1]] {
        try calls::invoke(owner, 23, &arguments[..], 0) catch error {
            assert error == abi::Error::InvalidArg; set invalid += 1;
        };
    }
    assert invalid == 2;
    let mut dead: u32 = 0;
    for object in &[child, parent] {
        set domains::STORE.records[object.index].state = domains::Lifecycle::Dead;
        try calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0) catch error {
            assert error == abi::Error::BadHandle; set dead += 1;
        };
        set domains::STORE.records[object.index].state = domains::Lifecycle::Active;
    }
    assert dead == 2 and domains::STORE.records[child.index].parent == parent;
    try! slots::release(&mut domains::STORE.slots[..], parent);
    try calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0) catch error {
        assert error == abi::Error::BadHandle; set dead += 1;
    };
    assert dead == 3 and domains::STORE.records[child.index].creator == owner;
}

/// Interrupt queries write exactly four bytes at four-byte-aligned addresses.
@test unsafe fn interruptRecord() throws (testing::TestError) {
    let owner = initialize();
    interrupts::initialize(&mut interrupts::STORE, 8);

        try testing::expect(error == abi::Error::BadHandle); set rejected += 1;
    };
    try testing::expect(rejected == 6 and domains::STORE.contextSlots[0].state == slots::State::Free);
}

/// Invalid infallible queries terminate the caller at the syscall boundary.
@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
    for operation in [44 as u64, 46, 47, 48] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let mut frame = trap::Frame { registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
        set frame.registers[17] = operation;
        calls::handle(owner, &mut frame, 0, calls::invoke);
        assert domains::STORE.records[owner.index].state == domains::Lifecycle::Dead;
        assert domains::STORE.events.terminals[owner.index].event.code == abi::FAULT_ABORT;
        assert domains::STORE.events.terminals[owner.index].event.kind == events::FAULT;
    }
}

/// An unknown user operation advances PC and returns only a negative a0 error.
@test unsafe fn trapReply() throws (testing::TestError) {
    let owner = initialize();
    let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
    set frame.registers[17] = 0xffffffffffffffff;

kernel/kernel/tests/domains.rad +357 -4

use kernel::slots;
use kernel::backing;
use kernel::capability;
use kernel::registry;
use kernel::domains;
use kernel::lifecycle;
use kernel::loader;
use kernel::frames;
use kernel::budgets;
use kernel::interrupts;
use kernel::transactions;
use kernel::pages;
use kernel::events;

/// Exclusive resources released by the domain reaper.
unsafe static WINDOWS: budgets::Store = undefined;
/// Interrupt ownership released after hardware masking.
unsafe static IRQS: interrupts::Store = undefined;

/// Caller-owned storage used while a context stack is initialized.
unsafe static CALLS: lifecycle::Calls = undefined;

/// Mapped physical memory for child metadata and package instances.
static RAM: [u64; 131072] = [0; 131072];
/// Domain and context metadata under test.
unsafe static DOMAINS: domains::Store = undefined;
/// Physical allocation and exposure state.

        set BACKINGS.pool.addresses[i] = aligned + i as u64 * 4096;
        set BACKINGS.pool.free[i] = true;
    }
    let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
    let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
    set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
    set DOMAINS.records[owner.index].parent = nil;
    set DOMAINS.records[owner.index].creator = owner;
    try! backing::registerDomain(&mut BACKINGS, owner);
    capability::initialize(&mut TABLE, owner);
    let authority = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(rights),
    });

        template: &[7], memory: 8, alignment: 8, relocations: &[],
    });
    set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
}

/// Interrupted creation returns all private storage after its caller releases its hart.
@test unsafe fn cancelledCreation() throws (testing::TestError) {
    initialize(abi::CREATE);
    lifecycle::initialize(&mut CALLS);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let permit = try! capability::get(&TABLE, handle);
    let mut caller = try! domains::get(&DOMAINS, permit.object);
    set DOMAINS.records[permit.object.index].state = domains::Lifecycle::Active;
    let authority = try! capability::install(&mut caller.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: permit.object, rights: abi::Rights(abi::CREATE),
    });
    let root = caller.image;
    let image = try! registry::install(&PACKAGES, &mut caller.memory.table, root);
    set PAGES.backings = BACKINGS;
    let pending = try! domains::reserve(&mut DOMAINS, &mut PAGES.backings, &PACKAGES, &mut caller.memory.table, authority, image);
    let mut creation: domains::Creation = undefined;
    match &pending { case domains::Reservation::Held(value) => { set creation = *value; }, }
    assert DOMAINS.contexts[creation.context.index].owner == creation.domain;
    let lease = lifecycle::holdDomain(&mut CALLS, &mut DOMAINS, caller.initial, pending);
    try! lifecycle::prepareDomain(&CALLS, &PACKAGES, &lease);
    lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut loader::STATE, &mut registry::STORE);
    assert DOMAINS.slots[creation.domain.index].state == slots::State::Reserved;
    set DOMAINS.records[permit.object.index].state = domains::Lifecycle::Dead;
    set DOMAINS.contexts[caller.initial.index].state = domains::ContextState::Stopped;
    set DOMAINS.contexts[caller.initial.index].hart = 1;
    lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut loader::STATE, &mut registry::STORE);
    assert DOMAINS.contexts[caller.initial.index].reservation;
    assert not PAGES.backings.pool.free[creation.storage.allocation.first];
    match lease { case lifecycle::Lease::Held(context) => { assert context == caller.initial; }, }
    set DOMAINS.contexts[caller.initial.index].hart = nil;
    lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut loader::STATE, &mut registry::STORE);
    assert not DOMAINS.contexts[caller.initial.index].reservation;
    assert DOMAINS.slots[creation.domain.index].state == slots::State::Free;
    assert DOMAINS.contextSlots[creation.context.index].state == slots::State::Free;
    assert caller.memory.table.slots[creation.handle.index].state == slots::State::Free;
    assert PAGES.backings.pool.retiredCount == 3;
    while PAGES.backings.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    for run in [creation.storage.allocation, creation.storage.kernelFrames, creation.storage.graph.frames] {
        for i in run.first..run.first + run.count { assert PAGES.backings.pool.free[i]; }
    }
    set DOMAINS.records[permit.object.index].state = domains::Lifecycle::Active;
    let next = try! domains::reserve(&mut DOMAINS, &mut PAGES.backings, &PACKAGES, &mut caller.memory.table, authority, image);
    let active = lifecycle::holdDomain(&mut CALLS, &mut DOMAINS, caller.initial, next);
    try! lifecycle::prepareDomain(&CALLS, &PACKAGES, &active);
    let ready = lifecycle::takeDomain(&mut CALLS, &mut DOMAINS, active);
    let published = try! domains::publish(&mut DOMAINS, &mut PAGES.backings, &mut caller.memory.table, ready);
    let entry = try! capability::get(&caller.memory.table, published);
    assert entry.object.index == creation.domain.index and entry.object.generation <> creation.domain.generation;
    assert (try! domains::get(&DOMAINS, entry.object)).state == domains::Lifecycle::Pending;
    assert not DOMAINS.contexts[caller.initial.index].reservation;
}

/// Private storage and terminal identity have separate reclamation boundaries.
@test unsafe fn reclaimDomain() throws (testing::TestError) {
    initialize(abi::CREATE);
    let parentHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let entry = try! capability::get(&TABLE, handle);
    let child = try! domains::get(&DOMAINS, entry.object);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, parentHandle);
    let mut private = child.memory;
    let authority = try! capability::install(&mut private.table, capability::Entry {
        kind: abi::Kind::Domain, object: entry.object, rights: abi::Rights(abi::CREATE),
    });
    let image = try! registry::install(&PACKAGES, &mut private.table, child.image);
    let descendantHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut private.table, authority, image);
    let descendant = try! capability::get(&private.table, descendantHandle);
    let word = entry.object.index / 32;
    let bit = 1 << (entry.object.index % 32);
    assert (DOMAINS.ancestors[descendant.object.index][word] & bit) <> 0;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].hart = 1;
    assert try! domains::terminate(&mut DOMAINS, entry.object, events::CHILD_EXIT, 17) == 2;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].hart = nil;
    set DOMAINS.contexts[child.initial.index].pending = abi::Ref { index: 0, generation: 1 };
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].pending = nil;
    set DOMAINS.contexts[child.initial.index].reservation = true;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].reservation = false;
    let reserved = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
    let context = slots::reference(&reserved);
    set DOMAINS.contexts[context.index].owner = entry.object;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    let mut busy = false;
    try domains::retireContext(&mut DOMAINS, &mut BACKINGS.pool, entry.object) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy and BACKINGS.pool.retiredCount == 0;
    try! slots::cancel(&mut DOMAINS.contextSlots[..], reserved);
    assert try! domains::quiescent(&DOMAINS, entry.object);
    assert try! domains::retireContext(&mut DOMAINS, &mut BACKINGS.pool, entry.object);
    assert not try! domains::retireContext(&mut DOMAINS, &mut BACKINGS.pool, entry.object);
    let mut handles = false;
    try domains::retireStorage(&mut DOMAINS, &mut BACKINGS, entry.object) catch error {
        assert error == abi::Error::Busy; set handles = true;
    };
    assert handles;
    let mut memory = child.memory;
    for cap in [child.events, authority, image, descendantHandle] {
        let removed = try! capability::invalidate(&mut memory.table, cap);
    }
    let mut exposed = false;
    try domains::retireStorage(&mut DOMAINS, &mut BACKINGS, entry.object) catch error {
        assert error == abi::Error::Busy; set exposed = true;
    };
    assert exposed;
    while not try! backing::endDomainStep(&mut BACKINGS, entry.object, 64) {}
    try! domains::retireStorage(&mut DOMAINS, &mut BACKINGS, entry.object);
    assert BACKINGS.pool.retiredCount == 3;
    let mut stale = false;
    try domains::get(&DOMAINS, entry.object) catch error {
        assert error == abi::Error::BadHandle; set stale = true;
    };
    assert stale and DOMAINS.slots[entry.object.index].state == slots::State::Live;
    while BACKINGS.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut BACKINGS.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    assert not try! events::acknowledged(&DOMAINS.events, entry.object);
    let mut retained = false;
    try domains::releaseIdentity(&mut DOMAINS, entry.object) catch error {
        assert error == abi::Error::Busy; set retained = true;
    };
    assert retained;
    let parentEntry = try! capability::get(&TABLE, parentHandle);
    let mut parent = try! domains::get(&DOMAINS, parentEntry.object);
    assert try! events::flush(&mut DOMAINS.events, parentEntry.object, &mut parent.memory.ring) == 1;
    let event = try! events::pop(&mut parent.memory.ring) else panic "missing terminal event";
    assert event.value == abi::id(entry.object) and event.code == 17;
    try! events::refresh(&mut DOMAINS.events, parentEntry.object, &parent.memory.ring);
    try! domains::releaseIdentity(&mut DOMAINS, entry.object);
    assert DOMAINS.slots[entry.object.index].state == slots::State::Free;
    assert (DOMAINS.ancestors[descendant.object.index][word] & bit) == 0;
    let next = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let replacement = try! capability::get(&TABLE, next);
    assert replacement.object.index == entry.object.index and replacement.object.generation <> entry.object.generation;
    assert (try! domains::get(&DOMAINS, replacement.object)).state == domains::Lifecycle::Pending;
    assert try! domains::terminateTree(&mut DOMAINS, replacement.object, events::CHILD_EXIT, 0) == 0;
    assert (try! domains::get(&DOMAINS, descendant.object)).state == domains::Lifecycle::Pending;
}

/// Reaping releases private authority while surviving shared pages retain their frames.
@test unsafe fn reaper() throws (testing::TestError) {
    initialize(abi::CREATE | abi::ALLOCATE);
    budgets::initialize(&mut WINDOWS);
    interrupts::initialize(&mut IRQS, 8);
    let parentHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let target = (try! capability::get(&TABLE, handle)).object;
    let receiver = (try! capability::get(&TABLE, parentHandle)).object;
    let mut child = try! domains::get(&DOMAINS, target);
    let mut parent = try! domains::get(&DOMAINS, receiver);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, parentHandle);
    set PAGES.backings = BACKINGS;
    slots::initialize(&mut PAGES.slots[..]);
    let authority = try! capability::install(&mut child.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: target, rights: abi::Rights(abi::ALLOCATE),
    });
    let page = try! pages::allocate(&mut PAGES, &mut child.memory.table, authority, 1);
    let shared = try! pages::grant(&mut PAGES, &child.memory.table, &mut parent.memory.table, page, abi::READ as u64);
    let allocation = (try! pages::get(&PAGES, (try! capability::get(&child.memory.table, page)).object)).backing;
    let run = PAGES.backings.records[allocation.index].run;
    let budget = try! budgets::seed(&mut WINDOWS, &mut child.memory.table, 0, 0, 1000);
    let window = (try! capability::get(&child.memory.table, budget)).object;
    let irq = try! interrupts::seed(&mut IRQS, &mut child.memory.table, 7);
    let dangling = try! capability::install(&mut child.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: abi::Ref { index: 511, generation: 99 }, rights: abi::Rights(0),
    });
    set DOMAINS.contexts[child.initial.index].hart = 0;
    assert try! domains::terminate(&mut DOMAINS, target, events::CHILD_EXIT, 19) == 1;
    assert not lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    assert (try! capability::get(&child.memory.table, page)).kind == abi::Kind::Page;
    set DOMAINS.contexts[child.initial.index].hart = nil;
    assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    assert child.memory.table.slots[0].state == slots::State::Free;
    assert (try! capability::get(&child.memory.table, page)).kind == abi::Kind::Page;
    for step in 0..2000 {
        if DOMAINS.reclaimed[target.index] { break; }
        assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    }
    assert DOMAINS.reclaimed[target.index];
    assert WINDOWS.slots[window.index].state == slots::State::Free;
    assert IRQS.slots[6].state == slots::State::Free;
    assert not lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    while PAGES.backings.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    assert not PAGES.backings.pool.free[run.first];
    assert PAGES.backings.records[allocation.index].exposed == 1;
    assert (try! pages::get(&PAGES, (try! capability::get(&parent.memory.table, shared)).object)).count == 1;
    assert try! events::flush(&mut DOMAINS.events, receiver, &mut parent.memory.ring) == 1;
    let event = try! events::pop(&mut parent.memory.ring) else panic "missing reaped child event";
    assert event.value == abi::id(target) and event.code == 19;
    try! events::refresh(&mut DOMAINS.events, receiver, &parent.memory.ring);
    assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    assert DOMAINS.slots[target.index].state == slots::State::Free;
    assert try! domains::terminate(&mut DOMAINS, receiver, events::CHILD_EXIT, 0) == 0;
    for step in 0..2000 {
        if DOMAINS.slots[receiver.index].state == slots::State::Free { break; }
        assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    }
    assert DOMAINS.slots[receiver.index].state == slots::State::Free;
    while PAGES.backings.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    for i in 0..PAGES.backings.pool.count { assert PAGES.backings.pool.free[i]; }
}

/// Initial return reports to the current parent and retains its terminal identity.
@test unsafe fn terminalParent() throws (testing::TestError) {
    initialize(abi::CREATE);
    let first = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let second = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let target = try! capability::get(&TABLE, handle);
    let destination = try! capability::get(&TABLE, second);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, first);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, second);
    let child = try! domains::get(&DOMAINS, target.object);
    set DOMAINS.contexts[child.initial.index].hart = 3;
    set DOMAINS.contexts[child.initial.index].pending = abi::Ref { index: 7, generation: 1 };
    let stopped = try! domains::returned(&mut DOMAINS, target.object, child.initial);
    assert stopped == 8 and DOMAINS.contexts[child.initial.index].hart == 3;
    assert DOMAINS.contexts[child.initial.index].pending <> nil;
    let repeated = try! domains::terminate(&mut DOMAINS, target.object, events::FAULT, 99);
    assert repeated == 0;
    assert DOMAINS.records[target.object.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.events.terminals[target.object.index].state == events::State::Pending;
    assert DOMAINS.events.terminals[target.object.index].receiver == destination.object;
    assert DOMAINS.events.queues[target.object.index].generation == 0;
    let mut rejected = false;
    try domains::reparent(&mut DOMAINS, &TABLE, handle, first) catch error {
        assert error == abi::Error::BadHandle; set rejected = true;
    };
    assert rejected and DOMAINS.records[target.object.index].creator == TABLE.owner;
    let mut parent = try! domains::get(&DOMAINS, destination.object);
    assert try! events::flush(&mut DOMAINS.events, destination.object, &mut parent.memory.ring) == 1;
    assert not try! events::acknowledged(&DOMAINS.events, target.object);
    let event = try! events::pop(&mut parent.memory.ring) else panic "missing child exit";
    assert event.kind == events::CHILD_EXIT and event.code == 0 and event.value == abi::id(target.object);
    try! events::refresh(&mut DOMAINS.events, destination.object, &parent.memory.ring);
    assert try! events::acknowledged(&DOMAINS.events, target.object);
}

/// Parent exit leaves surviving children parentless without changing their creator.
@test unsafe fn parentExit() throws (testing::TestError) {
    initialize(abi::CREATE);
    let parentHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let childHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let parentCap = try! capability::get(&TABLE, parentHandle);
    let childCap = try! capability::get(&TABLE, childHandle);
    try! domains::reparent(&mut DOMAINS, &TABLE, childHandle, parentHandle);
    let parent = try! domains::get(&DOMAINS, parentCap.object);
    try! domains::returned(&mut DOMAINS, parentCap.object, parent.initial);
    let child = try! domains::get(&DOMAINS, childCap.object);
    assert child.state == domains::Lifecycle::Pending and child.parent == nil and child.creator == TABLE.owner;
    try! domains::returned(&mut DOMAINS, childCap.object, child.initial);
    assert try! events::acknowledged(&DOMAINS.events, childCap.object);
}

/// Create a child through its creator's own authority and resident image.
unsafe fn descendant(table: &mut capability::Table, image: abi::Ref) -> abi::Ref {
    let owner = table.owner;
    let authority = try! capability::install(table, capability::Entry {
        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE),
    });
    let executable = try! registry::install(&PACKAGES, table, image);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, table, authority, executable);
    return (try! capability::get(table, handle)).object;
}

/// Cascade retains creation ancestry after reparenting and an intermediate exit.
@test unsafe fn cascadeAncestry() throws (testing::TestError) {
    initialize(abi::CREATE);
    let package = try! capability::get(&TABLE, IMAGE);
    let first = descendant(&mut TABLE, package.object);
    let mut ancestor = try! domains::get(&DOMAINS, first);
    let second = descendant(&mut ancestor.memory.table, package.object);
    let mut middle = try! domains::get(&DOMAINS, second);
    let third = descendant(&mut middle.memory.table, package.object);
    let child = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: third, rights: abi::Rights(abi::DESTROY),
    });
    let root = TABLE.owner;
    let receiver = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: root, rights: abi::Rights(abi::WAKE),
    });
    try! domains::reparent(&mut DOMAINS, &TABLE, child, receiver);
    let reserved = try! events::reserve(&mut DOMAINS.events, third);
    let mut busy = false;
    try domains::terminateTree(&mut DOMAINS, first, events::CHILD_EXIT, 7) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy and DOMAINS.records[first.index].state == domains::Lifecycle::Pending;
    assert DOMAINS.records[second.index].state == domains::Lifecycle::Pending;
    assert DOMAINS.events.terminals[first.index].state == events::State::Free;
    events::cancel(&mut DOMAINS.events, reserved);
    let stopped = try! domains::terminate(&mut DOMAINS, second, events::CHILD_EXIT, 12);
    assert stopped == 0 and DOMAINS.records[third.index].state == domains::Lifecycle::Pending;
    assert DOMAINS.records[third.index].parent == TABLE.owner;
    set DOMAINS.contexts[ancestor.initial.index].hart = 1;
    let last = try! domains::get(&DOMAINS, third);
    set DOMAINS.contexts[last.initial.index].hart = 3;
    let harts = try! domains::terminateTree(&mut DOMAINS, first, events::CHILD_EXIT, 7);
    assert harts == 10;
    assert DOMAINS.records[first.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.records[second.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.records[third.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.records[TABLE.owner.index].state == domains::Lifecycle::Active;
    assert DOMAINS.events.terminals[second.index].event.code == 12;
    assert DOMAINS.events.terminals[third.index].event.code == 7;
    assert DOMAINS.records[third.index].creator == second;
    assert DOMAINS.contexts[last.initial.index].hart == 3;
}

/// Activation validates child authority before publishing its initial state.
@test unsafe fn activation() throws (testing::TestError) {
    initialize(abi::CREATE);
    slots::initialize(&mut PAGES.slots[..]);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);

    let stackBase = DOMAINS.contexts[index].kernelStack.start;
    let base = (&RAM[0]) as u64;
    let firstWord = ((stackBase - base) / 8) as u32;
    let lastWord = firstWord + domains::KERNEL_STACK_PAGES * 512 - 1;
    set RAM[firstWord] = 123; set RAM[lastWord] = 456;
    domains::contextClear(&pendingContext);
    lifecycle::initialize(&mut CALLS);
    let lease = lifecycle::holdContext(&mut CALLS, &mut DOMAINS, child.initial, pendingContext);
    assert DOMAINS.contexts[child.initial.index].reservation;
    lifecycle::clearContext(&CALLS, &lease);
    assert RAM[firstWord] == 0 and RAM[lastWord] == 0;
    let context = domains::contextPublish(&mut DOMAINS, pendingContext);
    let ready = lifecycle::takeContext(&mut CALLS, &mut DOMAINS, lease);
    assert not DOMAINS.contexts[child.initial.index].reservation;
    let context = domains::contextPublish(&mut DOMAINS, ready);
    let created = try! domains::context(&DOMAINS, authority.object, context);
    let initial = DOMAINS.contexts[child.initial.index];
    try testing::expect(context <> child.initial and created.owner == authority.object);
    try testing::expect(created.frame.registers[3] == initial.frame.registers[3]);
    try testing::expect(created.kernelStack.start >= initial.kernelStack.end or created.kernelStack.end <= initial.kernelStack.start);

    assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        assert not PAGES.backings.pool.free[i];
    }
    set DOMAINS.contexts[context.index].pending = nil;
    set DOMAINS.contexts[context.index].reservation = true;
    set retained = false;
    try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context) catch error {
        assert error == abi::Error::Busy; set retained = true;
    };
    assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
    set DOMAINS.contexts[context.index].reservation = false;
    try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context);
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        try testing::expect(PAGES.backings.pool.free[i]);
    }
    try domains::context(&DOMAINS, authority.object, context) catch error {

    let next = try! domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    try testing::expect(next.index == context.index and next.generation <> context.generation);
    try! domains::returned(&mut DOMAINS, authority.object, child.initial);
    try testing::expect(DOMAINS.records[authority.object.index].state == domains::Lifecycle::Dead);
    try testing::expect(DOMAINS.contexts[next.index].state == domains::ContextState::Stopped);
    try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, next);
    try testing::expect(rejected == 11);
    try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, next) catch error {
        assert error == abi::Error::BadHandle; set rejected += 1;
    };
    assert DOMAINS.contextSlots[next.index].state == slots::State::Live;
    assert rejected == 12;
}

/// Both serialized wait/wakeup orders preserve notification and runnable state.
@test unsafe fn waitWakeup() throws (testing::TestError) {
    initialize(abi::CREATE);

kernel/kernel/tests/frames.rad +26 -0

//! Contiguous frame reservation and reserved-memory exclusion.

use std::testing;
use kernel::abi;
use kernel::frames;
use kernel::limits;
use kernel::platform;
use kernel::range;

/// Frame-map workspace is static because its size exceeds a small kernel stack.
unsafe static POOL: frames::Pool = undefined;

    set machine.reservedCount = 1;
    set machine.reserved[0] = range::Range { start: 0x80003000, end: 0x80005000 };
    try! frames::initialize(&mut POOL, &machine);
}

/// Retired frames stay unavailable until bounded reclamation returns them.
@test unsafe fn retirement() throws (testing::TestError) {
    initialize();
    set POOL.retiredHead = limits::FRAMES - 1;
    let left = frames::commit(try! frames::reserve(&mut POOL, 3));
    let right = frames::commit(try! frames::reserve(&mut POOL, 5));
    try! frames::retire(&mut POOL, left);
    try! frames::retire(&mut POOL, right);
    assert POOL.retiredCount == 2;
    for i in 0..8 { assert not POOL.free[i]; }
    assert frames::reclaim(&mut POOL, 0) == 0;
    assert frames::reclaim(&mut POOL, 2) == 2;
    assert POOL.free[0] and POOL.free[1] and not POOL.free[2];
    let reused = frames::commit(try! frames::reserve(&mut POOL, 2));
    assert reused.first == 0;
    assert frames::reclaim(&mut POOL, 3) == 3;
    assert POOL.retiredCount == 1 and not POOL.free[5];
    assert not POOL.free[0] and not POOL.free[1];
    try! frames::retire(&mut POOL, reused);
    assert frames::reclaim(&mut POOL, 20) == 5;
    assert POOL.retiredCount == 0;
    for i in 0..8 { assert POOL.free[i]; }
    assert frames::reclaim(&mut POOL, 20) == 0;
}

/// Adjacent physical frames form runs; a reserved hole cannot be crossed.
@test unsafe fn fragmentation() throws (testing::TestError) {
    initialize();
    try testing::expect(POOL.count == 8);
    let left = try! frames::reserve(&mut POOL, 3);

kernel/kernel/tests/lifecycle.rad added +307 -0

1	+	//! Terminal delivery, receiver eligibility, and retained full-queue events.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::domains;
6	+	use kernel::budgets;
7	+	use kernel::events;
8	+	use kernel::lifecycle;
9	+	use kernel::registry;
10	+	use kernel::loader;
11	+	use kernel::limits;
12	+	use kernel::remote;
13	+	use kernel::pages;
14	+	use kernel::backing;
15	+	use kernel::capability;
16	+	use kernel::frames;
17	+
18	+	/// Receiver generations, contexts, and terminal events.
19	+	unsafe static STORE: domains::Store = undefined;
20	+	/// CPU authority retained by receivers.
21	+	unsafe static WINDOWS: budgets::Store = undefined;
22	+	/// Queued calls and acknowledgements retained by stopped contexts.
23	+	unsafe static REQUESTS: remote::Store = undefined;
24	+	/// Physical allocation metadata used for interrupted page calls.
25	+	unsafe static MEMORY: pages::Store = undefined;
26	+	/// Mapped frames available to interrupted page calls.
27	+	static RAM: [u64; 4096] = [0; 4096];
28	+	/// Reservations retained across private initialization.
29	+	unsafe static CALLS: lifecycle::Calls = undefined;
30	+	/// Shared queue storage for the first receiver.
31	+	unsafe static FIRST: domains::Memory = undefined;
32	+	/// Shared queue storage for the second receiver.
33	+	unsafe static SECOND: domains::Memory = undefined;
34	+
35	+	/// Open one receiver with a waiting initial context.
36	+	unsafe fn open(index: u32, memory: &mut domains::Memory) {
37	+	let owner = abi::Ref { index, generation: 1 };
38	+	set STORE.slots[index] = slots::Slot { generation: 1, state: slots::State::Live };
39	+	set STORE.records[index].state = domains::Lifecycle::Active;
40	+	set STORE.records[index].memory = (&mut memory) as unsafe mut domains::Memory;
41	+	set STORE.contextSlots[index] = slots::Slot { generation: 1, state: slots::State::Live };
42	+	set STORE.contexts[index].owner = owner;
43	+	set STORE.contexts[index].state = domains::ContextState::Waiting;
44	+	set STORE.contexts[index].hart = nil;
45	+	set STORE.contexts[index].pending = nil;
46	+	try! events::open(&mut STORE.events, owner, &mut memory.ring);
47	+	}
48	+
49	+	/// Initialize two independent receivers and one future window on hart one.
50	+	unsafe fn initialize() {
51	+	domains::initialize(&mut STORE);
52	+	budgets::initialize(&mut WINDOWS);
53	+	remote::initialize(&mut REQUESTS, 3);
54	+	open(0, &mut FIRST);
55	+	open(1, &mut SECOND);
56	+	let owner = abi::Ref { index: 0, generation: 1 };
57	+	set WINDOWS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
58	+	set WINDOWS.windows[0] = budgets::Window {
59	+	hart: 1, start: 30, end: 100, owner, handle: abi::Handle(0), context: owner,
60	+	};
61	+	}
62	+
63	+	/// Retain a normal exit for a chosen receiver.
64	+	unsafe fn retain(index: u32, receiver: u32) {
65	+	let pending = try! events::reserve(&mut STORE.events, abi::Ref { index, generation: 1 });
66	+	events::finish(&mut STORE.events, pending, abi::Ref { index: receiver, generation: 1 }, events::CHILD_EXIT, index);
67	+	}
68	+
69	+	/// Dead page callers release unpublished capacity only after hart ownership ends.
70	+	@test unsafe fn cancelPage() throws (testing::TestError) {
71	+	initialize();
72	+	lifecycle::initialize(&mut CALLS);
73	+	backing::initialize(&mut MEMORY.backings);
74	+	slots::initialize(&mut MEMORY.slots[..]);
75	+	let owner = abi::Ref { index: 0, generation: 1 };
76	+	try! backing::registerDomain(&mut MEMORY.backings, owner);
77	+	capability::initialize(&mut FIRST.table, owner);
78	+	let authority = try! capability::install(&mut FIRST.table, capability::Entry {
79	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
80	+	});
81	+	let base = ((&RAM[0]) as u64 + 4095) & ~4095;
82	+	set MEMORY.backings.pool.count = 2;
83	+	for i in 0..2 {
84	+	set MEMORY.backings.pool.addresses[i] = base + i as u64 * 4096;
85	+	set MEMORY.backings.pool.free[i] = true;
86	+	}
87	+	let reservation = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 2);
88	+	let held = lifecycle::holdPage(&mut CALLS, &mut STORE, owner, reservation);
89	+	assert STORE.contexts[0].reservation;
90	+	lifecycle::clearPage(&CALLS, &held);
91	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
92	+	assert not MEMORY.backings.pool.free[0] and not MEMORY.backings.pool.free[1];
93	+	set STORE.records[0].state = domains::Lifecycle::Dead;
94	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
95	+	set STORE.contexts[0].hart = 0;
96	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
97	+	assert not MEMORY.backings.pool.free[0];
98	+	// Model the end of the stopped continuation before hart release.
99	+	match held { case lifecycle::Lease::Held(context) => { assert context == owner; }, }
100	+	set STORE.contexts[0].hart = nil;
101	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
102	+	assert MEMORY.backings.pool.retiredCount == 1 and not MEMORY.backings.pool.free[0];
103	+	assert frames::reclaim(&mut MEMORY.backings.pool, 256) == 2;
104	+	assert MEMORY.backings.pool.free[0] and MEMORY.backings.pool.free[1];
105	+	assert MEMORY.slots[0].state == slots::State::Free;
106	+	assert MEMORY.backings.slots[0].state == slots::State::Free;
107	+	assert FIRST.table.slots[1].state == slots::State::Free and CALLS.active[0] == 0;
108	+	assert not STORE.contexts[0].reservation;
109	+	set STORE.records[0].state = domains::Lifecycle::Active;
110	+	let next = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 1);
111	+	let lease = lifecycle::holdPage(&mut CALLS, &mut STORE, owner, next);
112	+	lifecycle::clearPage(&CALLS, &lease);
113	+	let ready = lifecycle::takePage(&mut CALLS, &mut STORE, lease);
114	+	let handle = pages::publish(&mut MEMORY, &mut FIRST.table, ready);
115	+	assert CALLS.active[0] == 0;
116	+	let entry = try! capability::get(&FIRST.table, handle);
117	+	assert (try! pages::get(&MEMORY, entry.object)).count == 1;
118	+	}
119	+
120	+	/// One cancellation pass retires at most one caller's private allocation.
121	+	@test unsafe fn cancellationBatch() throws (testing::TestError) {
122	+	initialize();
123	+	lifecycle::initialize(&mut CALLS);
124	+	backing::initialize(&mut MEMORY.backings);
125	+	slots::initialize(&mut MEMORY.slots[..]);
126	+	let owner = abi::Ref { index: 0, generation: 1 };
127	+	let second = abi::Ref { index: 1, generation: 1 };
128	+	set STORE.contexts[1].owner = owner;
129	+	try! backing::registerDomain(&mut MEMORY.backings, owner);
130	+	capability::initialize(&mut FIRST.table, owner);
131	+	let authority = try! capability::install(&mut FIRST.table, capability::Entry {
132	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
133	+	});
134	+	let base = ((&RAM[0]) as u64 + 4095) & ~4095;
135	+	set MEMORY.backings.pool.count = 2;
136	+	for i in 0..2 {
137	+	set MEMORY.backings.pool.addresses[i] = base + i as u64 * 4096;
138	+	set MEMORY.backings.pool.free[i] = true;
139	+	}
140	+	let firstPage = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 1);
141	+	let first = lifecycle::holdPage(&mut CALLS, &mut STORE, owner, firstPage);
142	+	let secondPage = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 1);
143	+	let other = lifecycle::holdPage(&mut CALLS, &mut STORE, second, secondPage);
144	+	match first { case lifecycle::Lease::Held(context) => { assert context == owner; }, }
145	+	match other { case lifecycle::Lease::Held(context) => { assert context == second; }, }
146	+	set STORE.records[0].state = domains::Lifecycle::Dead;
147	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
148	+	set STORE.contexts[1].state = domains::ContextState::Stopped;
149	+	assert lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
150	+	assert CALLS.active[0] == 2 and MEMORY.backings.pool.retiredCount == 1;
151	+	assert not STORE.contexts[0].reservation and STORE.contexts[1].reservation;
152	+	assert lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
153	+	assert CALLS.active[0] == 0 and MEMORY.backings.pool.retiredCount == 2;
154	+	assert not lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
155	+	assert not MEMORY.backings.pool.free[0] and not MEMORY.backings.pool.free[1];
156	+	assert frames::reclaim(&mut MEMORY.backings.pool, 1) == 1;
157	+	assert frames::reclaim(&mut MEMORY.backings.pool, 1) == 1;
158	+	assert MEMORY.backings.pool.retiredCount == 0;
159	+	}
160	+
161	+	/// A context call belongs to its caller even when its target domain dies.
162	+	@test unsafe fn cancelContext() throws (testing::TestError) {
163	+	initialize();
164	+	lifecycle::initialize(&mut CALLS);
165	+	backing::initialize(&mut MEMORY.backings);
166	+	let caller = abi::Ref { index: 0, generation: 1 };
167	+	let target = abi::Ref { index: 1, generation: 1 };
168	+	let base = ((&RAM[0]) as u64 + 4095) & ~4095;
169	+	set MEMORY.backings.pool.count = 1;
170	+	set MEMORY.backings.pool.addresses[0] = base;
171	+	set MEMORY.backings.pool.free[0] = true;
172	+	let slot = try! slots::reserve(&mut STORE.contextSlots[..]);
173	+	let object = slots::reference(&slot);
174	+	let run = frames::commit(try! frames::reserve(&mut MEMORY.backings.pool, 1));
175	+	set STORE.contexts[object.index].owner = target;
176	+	set STORE.contexts[object.index].kernelFrames = run;
177	+	set STORE.contexts[object.index].hart = nil;
178	+	match slot { case slots::Reservation::Held(id) => { assert id == object; }, }
179	+	let pending = domains::ContextReservation::Held(domains::ContextAllocation {
180	+	object, owner: target, frames: run, base,
181	+	});
182	+	let lease = lifecycle::holdContext(&mut CALLS, &mut STORE, caller, pending);
183	+	lifecycle::clearContext(&CALLS, &lease);
184	+	set STORE.records[1].state = domains::Lifecycle::Dead;
185	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
186	+	assert STORE.contexts[0].reservation and not MEMORY.backings.pool.free[0];
187	+	set STORE.records[0].state = domains::Lifecycle::Dead;
188	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
189	+	set STORE.contexts[0].hart = 1;
190	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
191	+	assert STORE.contextSlots[object.index].state == slots::State::Reserved;
192	+	match lease { case lifecycle::Lease::Held(id) => { assert id == caller; }, }
193	+	set STORE.contexts[0].hart = nil;
194	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
195	+	assert MEMORY.backings.pool.retiredCount == 1 and not MEMORY.backings.pool.free[0];
196	+	assert frames::reclaim(&mut MEMORY.backings.pool, 256) == 1;
197	+	assert MEMORY.backings.pool.free[0];
198	+	assert STORE.contextSlots[object.index].state == slots::State::Free;
199	+	assert not STORE.contexts[0].reservation and CALLS.active[0] == 0;
200	+	}
201	+
202	+	/// Dead contexts retain requests until hart release and completion of taken work.
203	+	@test unsafe fn cancelRequests() throws (testing::TestError) {
204	+	initialize();
205	+	let context = abi::Ref { index: 0, generation: 1 };
206	+	let pending = try! remote::send(&mut REQUESTS, 0, 1, remote::Action::Ready(context));
207	+	let other = try! remote::send(&mut REQUESTS, 0, 1, remote::Action::Ready(context));
208	+	let delivered = try! remote::take(&mut REQUESTS, 1) else panic "missing readiness request";
209	+	assert delivered.id == pending;
210	+	set STORE.contexts[0].pending = other;
211	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
212	+	assert REQUESTS.slots[other.index].state == slots::State::Live;
213	+	set STORE.records[0].state = domains::Lifecycle::Dead;
214	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
215	+	set STORE.contexts[0].hart = 0;
216	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
217	+	assert REQUESTS.slots[other.index].state == slots::State::Live;
218	+	set STORE.contexts[0].hart = nil;
219	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
220	+	assert REQUESTS.slots[other.index].state == slots::State::Free;
221	+	assert REQUESTS.slots[pending.index].state == slots::State::Live;
222	+	assert STORE.contexts[0].pending == pending;
223	+	assert remote::acknowledge(&mut REQUESTS, 1, pending);
224	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
225	+	assert STORE.contexts[0].pending == nil and REQUESTS.slots[pending.index].state == slots::State::Free;
226	+	assert try! remote::take(&mut REQUESTS, 1) == nil;
227	+	assert REQUESTS.queues[1].count == 0;
228	+	let budget = try! remote::send(&mut REQUESTS, 0, 1, remote::Action::Budget(remote::Call {
229	+	owner: context, context, operation: 70, arguments: [0; 4],
230	+	}));
231	+	set STORE.contexts[0].pending = budget;
232	+	let work = try! remote::take(&mut REQUESTS, 1) else panic "missing budget request";
233	+	remote::respond(&mut REQUESTS, 1, work.id, remote::Reply::Error(abi::Error::BadHandle));
234	+	assert remote::acknowledge(&mut REQUESTS, 1, work.id);
235	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
236	+	assert STORE.contexts[0].pending == nil and REQUESTS.slots[budget.index].state == slots::State::Free;
237	+	}
238	+
239	+	/// Closing multiple receivers validates all generations before changing any queue.
240	+	@test unsafe fn closeReceivers() throws (testing::TestError) {
241	+	initialize();
242	+	retain(2, 0);
243	+	retain(3, 1);
244	+	let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
245	+	set generations[0] = 1;
246	+	set generations[1] = 2;
247	+	let mut rejected = false;
248	+	try events::closeSet(&mut STORE.events, &generations[..]) catch error {
249	+	assert error == abi::Error::BadHandle; set rejected = true;
250	+	};
251	+	assert rejected and STORE.events.queues[0].generation == 1 and STORE.events.queues[1].generation == 1;
252	+	assert STORE.events.pending == 2;
253	+	set generations[1] = 1;
254	+	try! events::closeSet(&mut STORE.events, &generations[..]);
255	+	assert STORE.events.pending == 0;
256	+	assert STORE.events.queues[0].generation == 0 and STORE.events.queues[1].generation == 0;
257	+	assert try! events::acknowledged(&STORE.events, abi::Ref { index: 2, generation: 1 });
258	+	assert try! events::acknowledged(&STORE.events, abi::Ref { index: 3, generation: 1 });
259	+	}
260	+
261	+	/// Events ready both receivers but signal only existing CPU authority.
262	+	@test unsafe fn receivers() throws (testing::TestError) {
263	+	initialize();
264	+	retain(2, 0);
265	+	retain(3, 1);
266	+	assert STORE.events.pending == 2;
267	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2;
268	+	assert STORE.events.pending == 0;
269	+	assert STORE.contexts[0].state == domains::ContextState::Ready;
270	+	assert STORE.contexts[1].state == domains::ContextState::Ready;
271	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 0;
272	+	assert STORE.events.queues[0].tail == 1 and STORE.events.queues[1].tail == 1;
273	+	let first = try! events::pop(&mut FIRST.ring) else panic "first exit";
274	+	let second = try! events::pop(&mut SECOND.ring) else panic "second exit";
275	+	assert first.kind == events::CHILD_EXIT and first.code == 2;
276	+	assert second.kind == events::CHILD_EXIT and second.code == 3;
277	+	retain(4, 0);
278	+	assert lifecycle::service(&mut STORE, &WINDOWS, 100) == 0;
279	+	assert STORE.events.terminals[4].state == events::State::Queued;
280	+	}
281	+
282	+	/// Full receivers retain terminal events while other receivers make progress.
283	+	@test unsafe fn saturation() throws (testing::TestError) {
284	+	initialize();
285	+	let receiver = abi::Ref { index: 0, generation: 1 };
286	+	for i in 0..events::ORDINARY {
287	+	assert try! events::send(&mut STORE.events, receiver, &mut FIRST.ring,
288	+	events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });
289	+	}
290	+	for i in 64..128 { retain(i, 0); }
291	+	assert try! events::flush(&mut STORE.events, receiver, &mut FIRST.ring) == 64;
292	+	retain(2, 0);
293	+	retain(3, 1);
294	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 0;
295	+	assert STORE.events.terminals[2].state == events::State::Pending;
296	+	assert STORE.events.pending == 1;
297	+	assert STORE.events.terminals[3].state == events::State::Queued;
298	+	let consumed = try! events::pop(&mut FIRST.ring) else panic "full queue";
299	+	assert consumed.kind == events::WAKEUP;
300	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2;
301	+	assert STORE.events.terminals[2].state == events::State::Queued;
302	+	assert STORE.events.queues[0].tail == events::CAPACITY + 1;
303	+	retain(4, 0);
304	+	assert STORE.events.pending == 1;
305	+	try! events::close(&mut STORE.events, receiver);
306	+	assert STORE.events.pending == 0;
307	+	}

kernel/kernel/tests/loader.rad +81 -0

use kernel::frames;
use kernel::backing;
use kernel::capability;
use kernel::registry;
use kernel::loader;
use kernel::domains;
use kernel::lifecycle;

/// Caller metadata used for interrupted loader calls.
unsafe static DOMAINS: domains::Store = undefined;
/// Caller-owned private reservations.
unsafe static CALLS: lifecycle::Calls = undefined;
/// Capability storage retained until the stopped caller is reclaimed.
unsafe static MEMORY: domains::Memory = undefined;

/// Mapped frames for temporary compiler storage and persistent native output.
static RAM: [u64; 18874368] = [0; 18874368];
/// Physical frame and page metadata.
unsafe static PAGES: pages::Store = undefined;

    let mut count: u32 = 0;
    for i in 0..PAGES.backings.pool.count { if PAGES.backings.pool.free[i] { set count += 1; } }
    return count;
}

/// Dead loaders retain private resources until hart release, then permit a retry.
@test unsafe fn cancelledCall() throws (testing::TestError) {
    for stage in 0..3 {
        initialize();
        domains::initialize(&mut DOMAINS);
        lifecycle::initialize(&mut CALLS);
        let request = input(43, false);
        let before = free();
        let owner = TABLE.owner;
        set MEMORY.table = TABLE;
        set DOMAINS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
        set DOMAINS.records[0].state = domains::Lifecycle::Active;
        set DOMAINS.records[0].memory = &mut MEMORY;
        set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };
        set DOMAINS.contexts[0].owner = owner;
        set DOMAINS.contexts[0].state = domains::ContextState::Ready;
        set DOMAINS.contexts[0].hart = nil;
        let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
        let lease = lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, pending);
        if stage > 0 {
            try! lifecycle::reserveOutput(&mut CALLS, &lease, &mut PAGES, &mut PACKAGES);
        }
        if stage == 2 {
            let retained = lifecycle::load(&CALLS, &lease);
            let decoded = try! loader::decodeInput(&retained.input);
            let output = retained.output else panic "missing output";
            let compiled = try! loader::compileInput(&PACKAGES, &retained.input, &decoded, &output);
            assert compiled.code > 0;
        }
        lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER, &mut PACKAGES);
        assert LOADER.busy and DOMAINS.contexts[0].reservation;
        set DOMAINS.records[0].state = domains::Lifecycle::Dead;
        set DOMAINS.contexts[0].state = domains::ContextState::Stopped;
        set DOMAINS.contexts[0].hart = 1;
        lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER, &mut PACKAGES);
        assert LOADER.busy and free() < before;
        match lease { case lifecycle::Lease::Held(context) => { assert context == owner; }, }
        set DOMAINS.contexts[0].hart = nil;
        lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER, &mut PACKAGES);
        assert not LOADER.busy and not DOMAINS.contexts[0].reservation;
        assert free() < before and PAGES.backings.pool.retiredCount > 0;
        while PAGES.backings.pool.retiredCount > 0 {
            let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
            assert reclaimed > 0 and reclaimed <= 256;
        }
        assert free() == before and MEMORY.table.slots[2].state == slots::State::Free;
        assert PACKAGES.slots[0].state == slots::State::Free and not LOADER.live[0];
        set DOMAINS.records[0].state = domains::Lifecycle::Active;
        let retry = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
        let active = lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, retry);
        try! lifecycle::reserveOutput(&mut CALLS, &active, &mut PAGES, &mut PACKAGES);
        let retained = lifecycle::load(&CALLS, &active);
        let decoded = try! loader::decodeInput(&retained.input);
        let output = retained.output else panic "missing retry output";
        let compiled = try! loader::compileInput(&PACKAGES, &retained.input, &decoded, &output);
        let handle = try! lifecycle::finishLoad(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER,
            &mut PACKAGES, active, lifecycle::LoadResult::Compiled(compiled));
        let object = try! registry::image(&PACKAGES, &MEMORY.table, handle, abi::Rights(0));
        assert object.index == 0 and not DOMAINS.contexts[0].reservation;
        let available = free();
        let duplicate = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
        let reuse = lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, duplicate);
        let source = lifecycle::load(&CALLS, &reuse);
        let program = try! loader::decodeInput(&source.input);
        assert try! loader::identifyInput(&PACKAGES, &source.input, &program) == object;
        let same = try! lifecycle::finishLoad(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER,
            &mut PACKAGES, reuse, lifecycle::LoadResult::Existing(object));
        assert try! registry::image(&PACKAGES, &MEMORY.table, same, abi::Rights(0)) == object;
        assert free() == available and not LOADER.busy;
    }
}

/// Reserved input stays private across allocation, source mutation, and contention.
@test unsafe fn stagedInput() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();

lib/std/lang/module.rad +1 -1

use std::lang::alloc;
use std::lang::ast;
use std::lang::strings;

/// Maximum number of modules tracked in a single compilation graph.
export constant MAX_MODULES: u32 = 128;
export constant MAX_MODULES: u32 = 192;
/// Maximum number of characters for a module path.
constant MAX_PATH_LEN: u32 = 256;
/// Maximum number of components that make up a logical module path.
constant MAX_MODULE_PATH_DEPTH: u32 = 16;
/// Filesystem separator used when constructing child paths.

test/dispatch/kernel/dispatchcheck.rad +23 -3

use kernel::domains;
use kernel::budgets;
use kernel::dispatch;
use kernel::boot;
use kernel::events;
use kernel::sync;
use kernel::dispatchinput;

/// Bootstrap authority used to create fixture domains and CPU windows.
unsafe static TABLE: capability::Table = undefined;
/// Domain handles used for budget binding.

    let executable = try! registry::install(&registry::STORE, &mut user.memory.table, root);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let stackCap = try! capability::get(&TABLE, stack);
    let stackPage = try! pages::get(&pages::STORE, stackCap.object);
    let args = @sliceOf(memory(stackPage.base), 32);
    let parent = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: domains::STORE.contexts[CONTEXTS[2].index].owner, rights: abi::Rights(abi::WAKE),
    });
    let args = @sliceOf(memory(stackPage.base), 40);
    let words = [*granted, *executable, *selfAuthority];
    for word in 0..3 {
        for byte in 0..8 { set args[word * 8 + byte] = (words[word] >> (byte as u64 * 8)) as u8; }
    }
    try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, HANDLES[0], stackPage.base + 4096, stackPage.base, 32);
    for byte in 0..8 { set args[32 + byte] = (*parent >> (byte as u64 * 8)) as u8; }
    try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, HANDLES[0], stackPage.base + 4096, stackPage.base, 40);
    let imageRef = try! registry::image(&registry::STORE, &TABLE, image, abi::Rights(abi::EXECUTE));
    let target = try! registry::exported(&registry::STORE, imageRef, &"spin::count"[..]);
    let case shared::Target::Data(data) = target else panic "counter";
    let timer = try! dispatch::timer(&boot::PLATFORM, 0);
    let clock = dispatch::now(timer);
    let start = clock + 1000000;
    let span: u64 = 4000000;
    let span: u64 = 8000000;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.pc = worker();
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[10] = user.graph.table[data.slot] + data.offset as u64;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[11] = timer.clock;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[12] = start + 3 * span;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[13] = start + 4 * span;

    for byte in 0..8 { set progress |= bytes[byte] as u64 << (byte as u64 * 8); }
    if progress == 0 { failed(6); }
    let checked = try! registry::exported(&registry::STORE, domain.image, &"spin::destroyed"[..]);
    let case shared::Target::Data(flag) = checked else panic "destruction flag";
    if *memory(domain.graph.table[flag.slot] + flag.offset as u64) <> 1 { failed(7); }
    let child = try! registry::exported(&registry::STORE, domain.image, &"spin::childId"[..]);
    let case shared::Target::Data(identity) = child else panic "child identity";
    let idBytes = @sliceOf(memory(domain.graph.table[identity.slot] + identity.offset as u64), 8);
    let mut childId: u64 = 0;
    for byte in 0..8 { set childId |= idBytes[byte] as u64 << (byte as u64 * 8); }
    let owner = domains::STORE.contexts[CONTEXTS[2].index].owner;
    let guard = sync::enter();
    let mut checker = try! domains::get(&domains::STORE, owner);
    assert try! events::flush(&mut domains::STORE.events, owner, &mut checker.memory.ring) == 0;
    assert not try! events::acknowledged(&domains::STORE.events, abi::reference(childId));
    let exited = try! events::pop(&mut checker.memory.ring) else panic "child exit";
    assert exited.kind == events::CHILD_EXIT and exited.code == 0 and exited.value == childId;
    try! events::refresh(&mut domains::STORE.events, owner, &checker.memory.ring);
    assert try! events::acknowledged(&domains::STORE.events, abi::reference(childId));
    sync::leave(guard);
    let wake = try! events::pop(&mut domain.memory.ring) else panic "wake token";
    assert wake.kind == events::WAKEUP and wake.code == 123;
    let timeout = try! events::pop(&mut domain.memory.ring) else panic "timeout token";
    assert timeout.kind == events::TIMEOUT and timeout.code == 77 and timeout.value == 0;
    assert try! events::pop(&mut domain.memory.ring) == nil;

test/dispatch/spin.rad +10 -5

    image: u64,
    /// Local self-Domain capability with allocation authority.
    domain: u64,
    /// Future window that the user can partition between its contexts.
    future: u64,
    /// Completion checker with authority to receive child lifecycle events.
    parent: u64,
}

/// Startup environment supplied by the kernel.
record Env: Copy {
    /// Readable startup capabilities.

export static count: u64 = 0;
/// Additional-context progress in the same private package state.
export static extraCount: u64 = 0;
/// Successful user-side return and destruction checks.
export static destroyed: u64 = 0;
/// Generation-bearing identity of the child that returns normally.
export static childId: u64 = 0;
/// Local self-Domain authority shared by both user contexts.
static SELF: abi::Handle = abi::Handle(0);

/// Update shared private state and return from an additional context.
export fn worker(args: *opaque, size: u64) {
    assert size == 0;
    set extraCount += 1;
    try! sys::timeout(1000000, 77);
    try! sys::timeout(2000000, 77);
    sys::wait();
    set extraCount += 1;
    set count += 1;
}
/// Check direct calls and handoff, then exercise independently bound contexts.

    if env.argsSize == 0 {
        try! sys::timeout(10000, 88);
        sys::wait();
        return;
    }
    assert env.argsSize == 32;
    assert env.argsSize == 40;
    set SELF = abi::Handle(env.argsPointer.domain);
    let handle = env.argsPointer.stack as i64;
    let info = sys::queryPage(abi::Handle(handle as u64));
    assert info.base <> 0 and info.count == 1;
    let mut rejected = false;

    let domain = sys::queryDomain(abi::Handle(env.argsPointer.domain));
    assert domain.id <> 0 and domain.state == 1;
    let image = try! sys::queryImage(abi::Handle(env.argsPointer.image));
    assert image.entry <> 0 and image.codeBase <> 0 and image.codeBytes > 0 and image.dataBytes > 0;
    let child = try! sys::domainCreate(abi::Handle(0), abi::Handle(env.argsPointer.image));
    set childId = sys::queryDomain(child).id;
    try! sys::domainReparent(child, abi::Handle(env.argsPointer.parent));
    let stack = try! sys::pageAllocate(abi::Handle(0), 1);
    let granted = try! sys::capabilityGrant(stack, child, (abi::READ | abi::WRITE) as u64);
    let childStack = sys::queryPage(stack);
    try! sys::domainActivate(child, childStack.base + 4096, (&*env) as *unsafe opaque, 0);
    assert sys::queryDomain(child).state == 1;

    assert rejected and sys::currentContext().budget == current.budget;
    set count = 1;
    try! sys::yield(child);
    let continued = sys::currentContext();
    assert continued.context == current.context and continued.budget == *future;
    assert sys::queryDomain(child).state == 2;
    while extraCount == 0 { set count += 1; }
    let waiting = try! sys::queryContext(SELF, extra);
    assert waiting.state == 2 and waiting.hart == 0xffffffffffffffff;
    try! sys::wakeup(SELF, 123);
    assert extraCount == 1;
    assert extraCount == 1 or extraCount == 2;
    while extraCount == 1 { set count += 1; }
    let stopped = try! sys::queryContext(abi::Handle(env.argsPointer.domain), extra);
    assert stopped.state == 1 and stopped.hart == 0xffffffffffffffff;
    try! sys::contextDestroy(abi::Handle(env.argsPointer.domain), extra);
    set rejected = false;
    try sys::queryContext(abi::Handle(env.argsPointer.domain), extra) catch error {
        assert error == abi::Error::BadHandle;
        set rejected = true;
    };
    assert rejected and sys::queryDomain(child).state == 2;
    assert rejected;
    set destroyed = 1;
    while true { set count += 1; }
}

test/modules/run added +21 -0

1	+	#!/bin/sh
2	+	# Resolve and execute references beyond module index 127.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	work=$(mktemp -d)
6	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
7	+	mkdir "$work/capacity"
8	+	set -- -pkg capacity -mod "$work/capacity.rad"
9	+	index=0
10	+	while [ "$index" -lt 127 ]; do
11	+	printf 'export mod m%s;\n' "$index" >> "$work/capacity.rad"
12	+	printf '/// Return this module index.\nexport fn value() -> u64 { return %s; }\n' "$index" > "$work/capacity/m$index.rad"
13	+	set -- "$@" -mod "$work/capacity/m$index.rad"
14	+	index=$((index + 1))
15	+	done
16	+	printf 'use capacity::m126;\n/// Check a package root beyond module index 127.\n@default fn main() -> u64 { assert m126::value() == 126; return 0; }\n' > "$work/entry.rad"
17	+	set -- "$@" -pkg entry -mod "$work/entry.rad"
18	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
19	+	-run bin/radiance.rv64.dev "$@" -entry entry -o "$work/capacity.rv64"
20	+	"$emulator" -run "$work/capacity.rv64"
21	+	printf 'module scopes: references beyond index 127 passed\n'

test/smp/kernel/dispatchcheck.rad +207 -5

use kernel::calls;
use kernel::events;
use kernel::timers;
use kernel::interrupts;
use kernel::plic;
use kernel::lifecycle;
use kernel::dispatchinput;

/// Bootstrap resource authority.
unsafe static TABLE: capability::Table = undefined;
/// Release/acquire barrier after fixture publication.
static READY: u64 = 0;
/// Shared user domain generation.
unsafe static USER: abi::Ref = undefined;
/// Pending domain in the user's creation subtree.
unsafe static DESCENDANT: abi::Ref = undefined;
/// Per-hart user execution contexts.
unsafe static CONTEXTS: [abi::Ref; 8] = undefined;
/// Mapped progress counters in the user's private state.
static COUNTERS: u64 = 0;
/// Per-context migration observations in the user's private state.

static IDLE_STACK: u64 = 0;
/// Generation of the idle-wakeup domain.
unsafe static IDLE_DOMAIN: abi::Ref = undefined;
/// Completion checker's live domain generation.
unsafe static CHECKER: abi::Ref = undefined;
/// Unbound window that expires before the checker starts.
unsafe static EXPIRED: abi::Handle = undefined;
/// Checker's authority to activate and notify the idle-wakeup domain.
unsafe static RECEIVER: abi::Handle = undefined;
/// Unbound authority reserved for the idle-wakeup check on hart one.
unsafe static IDLE_WINDOW: abi::Handle = undefined;
/// Validated physical memory mapping.

    let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
        loader::Request { authority, source, offset: 0, length: length as u64 });
    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let cap = try! capability::get(&TABLE, handle);
    set USER = cap.object;
    createDescendant();
    let mut user = try! domains::get(&domains::STORE, USER);
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),
    });
    assert boot::PLATFORM.irqSources == 31;

                &TABLE, handle, abi::ContextStart { entry, stack: page.base + 4096, args: page.base, size: 8 });
        }
    }
    let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let checkerCap = try! capability::get(&TABLE, checkerHandle);
    let checker = try! domains::get(&domains::STORE, checkerCap.object);
    let mut checker = try! domains::get(&domains::STORE, checkerCap.object);
    set CHECKER = checkerCap.object;
    set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;
    set domains::STORE.contexts[checker.initial.index].frame.pc = completion();
    set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;
    set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;

    let end = start + 10000000 * harts * harts;
    let migrationEnd = end + 10000000 * harts * harts;
    let windowBase = dataAddress(&user, &"spin::windows"[..]);
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        let budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);
        let begin = clock if hart == 0 else start;
        let mut budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, begin, 0xffffffffffffffff);
        if hart == 0 {
            let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, start, clock);
            set EXPIRED = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut checker.memory.table,
                budget, budgets::DEFAULT_RIGHTS as u64, clock);
            set budget = next;
        }
        let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);
        let checking = try! budgets::split(&mut budgets::STORE, &mut TABLE, final, migrationEnd, clock);
        let future = try! budgets::split(&mut budgets::STORE, &mut TABLE, checking, migrationEnd + 10000000000, clock);
        let transferred = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut user.memory.table,
            future, budgets::DEFAULT_RIGHTS as u64, clock);

        }
        if hart == 1 { set IDLE_WINDOW = checking; }
    }
}

/// Create an actual pending descendant before secondary harts enter dispatch.
unsafe fn createDescendant() {
    let mut user = try! domains::get(&domains::STORE, USER);
    let root = user.image;
    let authority = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::CREATE),
    });
    let image = try! registry::install(&registry::STORE, &mut user.memory.table, root);
    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
        &registry::STORE, &mut user.memory.table, authority, image);
    set DESCENDANT = (try! capability::get(&user.memory.table, handle)).object;
}

/// Resolve one exported private-data address in the fixture's user instance.
unsafe fn dataAddress(user: &domains::Domain, name: &[u8]) -> u64 {
    let target = try! registry::exported(&registry::STORE, user.image, name);
    let case shared::Target::Data(data) = target else panic "fixture data export";
    return user.graph.table[data.slot] + data.offset as u64;

    set IDLE_HANDLE = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
        &registry::STORE, &mut TABLE, authority, image);
    let permit = try! capability::get(&TABLE, IDLE_HANDLE);
    set IDLE_DOMAIN = permit.object;
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let allocation = try! capability::install(&mut domain.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::ALLOCATE),
    });
    set IDLE_CONTEXT = domain.initial;
    set WAKE_STATE = dataAddress(&domain, &"spin::wakeState"[..]);
    sync::storeRelease(memory(dataAddress(&domain, &"spin::idle"[..])) as *mut u64, 1);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut domain.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let permission = try! capability::get(&TABLE, stack);
    let page = try! pages::get(&pages::STORE, permission.object);
    set IDLE_STACK = page.base + 4096;
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE),
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE | abi::DESTROY),
    });
}

/// Check shared progress and context isolation after the user windows finish.
export unsafe fn verify() {
    let guard = sync::enter();
    let user = try! domains::get(&domains::STORE, USER);
    let mut counts: [u64; 8] = [0; 8];
    let mut notifications: u32 = 0;
    for hart in 0..8 { if (boot::PLATFORM.harts & (1 << hart)) <> 0 { set notifications += 1; } }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
        let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);
        set counts[hart] = value;
        let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);
        assert context.owner == USER and (context.frame.status & 0x1800) == 0;
        assert context.owner == USER;
        let mode = context.frame.status & 0x1800;
        assert mode == 0 or mode == 0x1800;
        if mode == 0x1800 {
            assert context.frame.registers[3] == kernelGp();
            assert context.frame.registers[2] >= context.kernelStack.start and context.frame.registers[2] <= context.kernelStack.end;
        } else {
            assert context.frame.registers[3] == user.graph.table.ptr as u64;
        }
        let destination = (hart + notifications - 1) % notifications;
        assert context.hart == nil or context.hart == destination;
        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;
        assert context.pending == nil;
    }
    assert domains::STORE.events.queues[USER.index].tail == notifications + 1;
    let user = try! domains::get(&domains::STORE, USER);
    assert user.memory.ring.data[0].kind == events::INTERRUPT and user.memory.ring.data[0].code == 7;
    sync::leave(guard);
    for byte in "smp completed transactions:" { put(byte); }
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }

            }
            assert acknowledged;
        }
    }
    for byte in "smp request acknowledgements passed\n" { put(byte); }
    expiredCall();
    if (boot::PLATFORM.harts & 2) <> 0 { idleWakeup(); }
    interruptOwnership();
    interruptStorm();
    if (boot::PLATFORM.harts & 2) <> 0 { terminalWakeup(); cancelAllocation(); reclaimedDomains(); }
    let maximum = sync::maximum();
    assert maximum > 0 and maximum < 1000000;
    for byte in "smp metadata instructions: 0x" { put(byte); }
    number(maximum);
    put(10);
    finish();
}

/// Validate elapsed authority against the current clock at the serialized boundary.
unsafe fn expiredCall() {
    let guard = sync::enter();
    let checker = try! domains::get(&domains::STORE, CHECKER);
    let before = try! budgets::query(&budgets::STORE, &checker.memory.table, EXPIRED);
    assert before.end < dispatch::clock();
    sync::leave(guard);
    let mut rejected = false;
    try calls::synchronized(CHECKER, 70, &[*EXPIRED, before.start + (before.end - before.start) / 2, 0, 0], 0) catch error {
        assert error == abi::Error::Busy; set rejected = true;
    };
    assert rejected;
    let checking = sync::enter();
    let after = try! budgets::query(&budgets::STORE, &checker.memory.table, EXPIRED);
    assert after.start == before.start and after.end == before.end;
    sync::leave(checking);
    for byte in "smp resumed call rejected expired authority\n" { put(byte); }
}

/// Transfer a queued IRQ, rearm after consumption, and mask it before capability drop.
unsafe fn interruptOwnership() {
    let guard = sync::enter();
    let mut user = try! domains::get(&domains::STORE, USER);
    let mut checker = try! domains::get(&domains::STORE, CHECKER);

    }
    assert resumed;
    for byte in "smp remote timeout passed\n" { put(byte); }
}

/// Wake a parent waiting on hart one through terminal delivery on hart zero.
unsafe fn terminalWakeup() {
    let guard = sync::enter();
    let mut receiver = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let timeout = try! events::pop(&mut receiver.memory.ring) else panic "missing idle timeout";
    assert timeout.kind == events::TIMEOUT and timeout.code == 0x66;
    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 7);
    sync::leave(guard);
    assert waiting();
    let stopping = sync::enter();
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    let child = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::DESTROY),
    });
    let parent = try! calls::invoke(CHECKER, 23, &[*child, *RECEIVER, 0, 0], 0);
    assert parent == 0;
    let cancelled = try! dispatch::request(1, remote::Action::Budget(remote::Call {
        owner: USER, context: CONTEXTS[0], operation: 70, arguments: [0; 4],
    }));
    set domains::STORE.contexts[CONTEXTS[0].index].pending = cancelled;
    let readiness = try! remote::reserve(&mut remote::STORE, 0, 2);
    let batch = remote::publish(&mut remote::STORE, readiness, remote::Action::Ready(CONTEXTS[1]));
    dispatch::startReady(CONTEXTS[1], &batch);
    let stopped = try! domains::terminateTree(&mut domains::STORE, USER, events::CHILD_EXIT, 23);
    assert stopped == 0;
    assert domains::STORE.records[DESCENDANT.index].state == domains::Lifecycle::Dead;
    assert try! events::acknowledged(&domains::STORE.events, DESCENDANT);
    let request = try! dispatch::request(0, remote::Action::Reschedule);
    sync::leave(stopping);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let mut resumed = false;
    for attempt in 0..100000 {
        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 9 { set resumed = true; break; }
    }
    assert resumed;
    let checking = sync::enter();
    assert remote::STORE.slots[cancelled.index].state == slots::State::Free;
    assert remote::STORE.slots[batch.ids[1].index].state == slots::State::Free;
    assert domains::STORE.contexts[CONTEXTS[0].index].pending == nil;
    assert domains::STORE.contexts[CONTEXTS[1].index].pending == nil;
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let event = try! events::pop(&mut domain.memory.ring) else panic "missing remote exit";
    assert event.kind == events::CHILD_EXIT and event.code == 23 and event.value == abi::id(USER);
    sync::leave(checking);
    for byte in "smp cascade cancelled calls and woke remote parent\n" { put(byte); }
}

/// Stop a real page-clear continuation and recover its unpublished frames.
unsafe fn cancelAllocation() {
    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 10);
    let mut allocation: pages::Allocation = undefined;
    let mut request: ?abi::Ref = nil;
    for attempt in 0..100000 {
        let guard = sync::enter();
        let index = IDLE_CONTEXT.index;
        if (lifecycle::CALLS.active[index / 32] & (1 << (index % 32))) <> 0 {
            match lifecycle::CALLS.records[index].allocation {
                case lifecycle::Allocation::Page(value) => { set allocation = value; },
                else => panic "page allocation required",
            }
            assert allocation.owner == IDLE_DOMAIN and allocation.run.count == 4096;
            sync::leave(guard);
            let result = try! calls::synchronized(CHECKER, 22, &[*RECEIVER, 0, 0, 0], dispatch::clock());
            assert result == 0;
            let checking = sync::enter();
            assert domains::STORE.records[IDLE_DOMAIN.index].state == domains::Lifecycle::Dead;
            set request = try! dispatch::request(1, remote::Action::Reschedule);
            sync::leave(checking);
            break;
        }
        sync::leave(guard);
    }
    let stopped = request else panic "page call did not reserve frames";
    let acknowledged = try! dispatch::awaitRequest(0, stopped);
    assert acknowledged == 0;
    let snapshot = sync::enter();
    let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);
    for byte in "smp cancelled page status and progress: " { put(byte); }
    number(context.frame.status); put(32);
    number(sync::loadAcquire(memory(WAKE_STATE) as *u64)); put(10);
    assert context.hart == nil and context.state == domains::ContextState::Stopped;
    assert (context.frame.status & 0x1800) == 0x1800;
    assert sync::loadAcquire(memory(WAKE_STATE) as *u64) == 11;
    sync::leave(snapshot);
    let mut reclaimed = false;
    for attempt in 0..1000 {
        let checking = sync::enter();
        set reclaimed = pages::STORE.backings.pool.retiredCount == 0;
        if reclaimed { sync::leave(checking); break; }
        let progress = try! dispatch::request(0, remote::Action::Reschedule);
        sync::leave(checking);
        let result = try! dispatch::awaitRequest(0, progress);
        assert result == 0;
    }
    assert reclaimed;
    let guard = sync::enter();
    assert (lifecycle::CALLS.active[IDLE_CONTEXT.index / 32] & (1 << (IDLE_CONTEXT.index % 32))) == 0;
    assert pages::STORE.slots[allocation.page.index].state == slots::State::Free;
    assert pages::STORE.backings.slots[allocation.backing.index].state == slots::State::Free;
    for i in allocation.run.first..allocation.run.first + allocation.run.count {
        assert pages::STORE.backings.pool.free[i];
    }
    sync::leave(guard);
    for byte in "smp cancelled preempted page allocation\n" { put(byte); }
}

/// Wait for dispatch-driven teardown and verify all dead execution ownership is gone.
unsafe fn reclaimedDomains() {
    for attempt in 0..100000 {
        let guard = sync::enter();
        let mut done = pages::STORE.backings.pool.retiredCount == 0;
        for owner in [USER, IDLE_DOMAIN, DESCENDANT] {
            if slots::matches(&domains::STORE.slots[..], owner, slots::State::Live) { set done = false; }
        }
        if done {
            assert domains::STORE.dead == 0;
            for owner in [USER, IDLE_DOMAIN, DESCENDANT] {
                assert domains::STORE.slots[owner.index].state == slots::State::Free;
                assert pages::STORE.backings.domains[owner.index] == 0;
                for i in 0..domains::STORE.contextSlots.len {
                    if domains::STORE.contextSlots[i].state == slots::State::Live {
                        assert domains::STORE.contexts[i].owner <> owner;
                    }
                }
                for i in 0..budgets::STORE.slots.len {
                    if budgets::STORE.slots[i].state == slots::State::Live { assert budgets::STORE.windows[i].owner <> owner; }
                }
                for i in 0..interrupts::STORE.slots.len {
                    if interrupts::STORE.slots[i].state == slots::State::Live { assert interrupts::STORE.records[i].owner <> owner; }
                }
            }
            sync::leave(guard);
            for byte in "smp reaped dead domains and recovered ownership\n" { put(byte); }
            return;
        }
        let progress = try! dispatch::request(0, remote::Action::Reschedule);
        sync::leave(guard);
        let result = try! dispatch::awaitRequest(0, progress);
        assert result == 0;
    }
    panic "domain reaper did not finish";
}

/// Confirm the waiting context has released hart ownership before its wakeup.
unsafe fn waiting() -> bool {
    for attempt in 0..100000 {
        let guard = sync::enter();
        let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);

test/smp/spin.rad +9 -1

export static interrupt: u64 = 0;

/// Observed destination hart plus one after each context changes budget.
export static migrations: [u64; 8] = [0; 8];

/// Wait handshake: 2/3 bracket wakeup; 4 requests a second wait, bracketed by 5/6.
/// Progress shared with the machine fixture for waits and interrupted allocation.
export static wakeState: u64 = 0;

/// Nonzero selects the independently activated idle-wakeup context.
export static idle: u64 = 0;


        set wakeState = 3;
        while wakeState <> 4 {}
        set wakeState = 5;
        sys::wait();
        set wakeState = 6;
        while wakeState <> 7 {}
        set wakeState = 8;
        sys::wait();
        set wakeState = 9;
        while wakeState <> 10 {}
        set wakeState = 11;
        let allocation = try! sys::pageAllocate(abi::Handle(0), 4096);
        set wakeState = 12;
        while true {}
    }
    run(0);
}

test/termination/kernel/dispatchcheck.rad added +101 -0

1	+	//! Native exit and fault reporting to a surviving parent.
2	+	use std::mem;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::backing;
6	+	use kernel::pages;
7	+	use kernel::capability;
8	+	use kernel::registry;
9	+	use kernel::loader;
10	+	use kernel::domains;
11	+	use kernel::budgets;
12	+	use kernel::dispatch;
13	+	use kernel::boot;
14	+	use kernel::events;
15	+	use kernel::sync;
16	+	use kernel::dispatchinput;
17	+
18	+	/// Bootstrap creation, allocation, and budget authority.
19	+	unsafe static TABLE: capability::Table = undefined;
20	+	/// Child domain identities retained by terminal events.
21	+	static CHILDREN: [abi::Ref; 5] = undefined;
22	+	/// Surviving event receiver and native completion checker.
23	+	unsafe static PARENT: abi::Ref = undefined;
24	+	/// Map a validated physical page.
25	+	fn memory(address: u64) -> *mut u8;
26	+	/// Current kernel package-state base.
27	+	fn kernelGp() -> u64;
28	+	/// Native entry for the completion checker.
29	+	fn completion() -> u64;
30	+	/// Signal successful completion to the machine fixture.
31	+	fn finish();
32	+
33	+	/// Give four children disjoint windows, followed by the surviving parent.
34	+	export unsafe fn setup() {
35	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
36	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
37	+	capability::initialize(&mut TABLE, owner);
38	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
39	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
40	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
41	+	});
42	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
43	+	let page = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, source)).object);
44	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
45	+	let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
46	+	let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
47	+	loader::Request { authority, source, offset: 0, length: length as u64 });
48	+	let parentHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
49	+	set PARENT = (try! capability::get(&TABLE, parentHandle)).object;
50	+	let parent = try! domains::get(&domains::STORE, PARENT);
51	+	set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;
52	+	set domains::STORE.contexts[parent.initial.index].frame.pc = completion();
53	+	set domains::STORE.contexts[parent.initial.index].frame.status = 0x1880;
54	+	set domains::STORE.contexts[parent.initial.index].frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;
55	+	set domains::STORE.contexts[parent.initial.index].frame.registers[3] = kernelGp();
56	+	let mut handles: [abi::Handle; 5] = undefined;
57	+	let mut contexts: [abi::Ref; 5] = undefined;
58	+	for i in 0..5 {
59	+	let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
60	+	let object = (try! capability::get(&TABLE, handle)).object;
61	+	let mut child = try! domains::get(&domains::STORE, object);
62	+	set CHILDREN[i] = object;
63	+	set handles[i] = handle;
64	+	set contexts[i] = child.initial;
65	+	try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);
66	+	let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
67	+	let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
68	+	let storage = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, stack)).object);
69	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, i as u64);
70	+	}
71	+	let timer = try! dispatch::timer(&boot::PLATFORM, 0);
72	+	let clock = dispatch::now(timer);
73	+	let start = clock + 1000000;
74	+	let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 12000000);
75	+	for i in 0..5 {
76	+	let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, remaining, start + (i as u64 + 1) * 2000000, clock);
77	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
78	+	budgets::Binding { budget: remaining, domain: handles[i], context: contexts[i] }, clock);
79	+	set remaining = next;
80	+	}
81	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
82	+	budgets::Binding { budget: remaining, domain: parentHandle, context: parent.initial }, clock);
83	+	}
84	+
85	+	/// Check each terminal kind, status, and generation while the parent continues to run.
86	+	export unsafe fn verify() {
87	+	let guard = sync::enter();
88	+	let mut parent = try! domains::get(&domains::STORE, PARENT);
89	+	let codes = [37 as u32, abi::FAULT_ABORT, 3, 0, abi::FAULT_ABORT];
90	+	for i in 0..5 {
91	+	let event = try! events::pop(&mut parent.memory.ring) else panic "missing termination event";
92	+	assert event.value == abi::id(CHILDREN[i]) and event.code == codes[i];
93	+	if i == 0 or i == 3 { assert event.kind == events::CHILD_EXIT; }
94	+	else { assert event.kind == events::FAULT; }
95	+	assert domains::STORE.records[CHILDREN[i].index].state == domains::Lifecycle::Dead;
96	+	}
97	+	try! events::refresh(&mut domains::STORE.events, PARENT, &parent.memory.ring);
98	+	assert try! events::pop(&mut parent.memory.ring) == nil;
99	+	sync::leave(guard);
100	+	finish();
101	+	}

test/termination/machine.ras added +25 -0

1	+	//! Boot one hart and check user termination through production dispatch.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::setup;
5	+	call @kernel::boot::run;
6	+	.export @kernel::dispatchcheck::memory;
7	+	.export @kernel::dispatchcheck::kernelGp;
8	+	.export @kernel::dispatchcheck::completion;
9	+	.export @kernel::dispatchcheck::finish;
10	+	@kernel::dispatchcheck::memory
11	+	ret;
12	+	@kernel::dispatchcheck::kernelGp
13	+	mv %a0 %gp;
14	+	ret;
15	+	@kernel::dispatchcheck::completion
16	+	la %a0 @complete;
17	+	ret;
18	+	@complete
19	+	call @kernel::dispatchcheck::verify;
20	+	ebreak;
21	+	@kernel::dispatchcheck::finish
22	+	li %t0 0x10001000;
23	+	li %t1 0x5555;
24	+	sw %t1 0(%t0);
25	+	ebreak;

test/termination/spin.rad added +30 -0

1	+	//! User termination and CPU-fault entry paths.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Issue a raw call to test the kernel error boundary.
6	+	@intrinsic fn ecall(operation: u32, a0: i64, a1: i64, a2: i64, a3: i64) -> i64;
7	+
8	+	/// Startup values used to select the termination operation.
9	+	record Env: Copy {
10	+	/// Optional argument bytes supplied by the parent.
11	+	argsPointer: *u8,
12	+	/// Termination case selected by the parent.
13	+	argsSize: u64,
14	+	/// Installed event handle.
15	+	eventsHandle: u64,
16	+	/// Shared event-ring address.
17	+	eventsPointer: u64,
18	+	}
19	+
20	+	/// Exercise an explicit exit, explicit abort, or a user breakpoint fault.
21	+	@default fn main(env: *Env) {
22	+	if env.argsSize == 0 { sys::exit(37); }
23	+	if env.argsSize == 1 { sys::abort(); }
24	+	if env.argsSize == 3 { try! sys::domainDestroy(abi::Handle(0), 0); }
25	+	if env.argsSize == 4 {
26	+	let result = ecall(47, 0, 0, 0, 0);
27	+	sys::exit(99);
28	+	}
29	+	assert false;
30	+	}

470	617		}
471	618		}
472	619
473	620		/// Initialize private storage from resident packages while metadata can change.
474	621		export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
475		-	match reservation {
476		-	case Reservation::Held(creation) => {
477		-	let mut storage = creation.storage;
478		-	zero(storage.memory as u64, storage.allocation.count);
479		-	zero(storage.kernelStack.start, storage.kernelFrames.count);
480		-	try instances::fill(packages, &storage.graph.graph, storage.graph.base);
481		-	capability::initialize(&mut storage.memory.table, creation.domain);
482		-	},
483		-	}
	622	+	match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
	623	+	}
	624	+
	625	+	/// Initialize retained domain storage outside the metadata lock.
	626	+	export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
	627	+	let mut storage = creation.storage;
	628	+	zero(storage.memory as u64, storage.allocation.count);
	629	+	zero(storage.kernelStack.start, storage.kernelFrames.count);
	630	+	try instances::fill(packages, &storage.graph.graph, storage.graph.base);
	631	+	capability::initialize(&mut storage.memory.table, creation.domain);
484	632		}
485	633
486	634		/// Check the unpublished capacity retained by one creation transaction.
487	635		fn require(store: &Store, table: &capability::Table, creation: &Creation) {
488	636		assert table.owner == creation.owner;

491	639		assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
492	640		}
493	641
494	642		/// Return private domain storage and reserved slots under metadata serialization.
495	643		export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
	644	+	cancelUsing(store, backings, table, reservation, frames::release);
	645	+	}
	646	+
	647	+	/// Cancel private domain slots and transfer their storage to the release operation.
	648	+	export fn cancelUsing(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation,
	649	+	release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
	650	+	{
496	651		match reservation {
497	652		case Reservation::Held(creation) => {
498	653		require(store, table, &creation);
499		-	discard(&mut backings.pool, creation.storage);
	654	+	discard(&mut backings.pool, creation.storage, release);
500	655		try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
501	656		try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
502	657		try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
503	658		},
504	659		}

532	687		// MPIE enables interrupts after the first return into U-mode.
533	688		let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
534	689		set frame.registers[1] = returnAddress();
535	690		set frame.registers[3] = graph.table.ptr as u64;
536	691		set store.contexts[initial.index] = Context {
537		-	owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
	692	+	owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
538	693		userStack: range::Range { start: 0, end: 0 }, frame,
539	694		};
	695	+	set store.reclaimed[object.index] = false;
	696	+	set store.reaping[object.index] = false;
540	697		set store.records[object.index] = Domain {
541	698		state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
542	699		allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
543	700		};
	701	+	set store.ancestors[object.index] = store.ancestors[creation.owner.index];
	702	+	set store.ancestors[object.index][creation.owner.index / 32] \|= 1 << (creation.owner.index % 32);
544	703		let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
545	704		let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
546	705		return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
547	706		kind: abi::Kind::Domain, object: domain, rights: creation.rights,
548	707		});

21	21
22	22		# Verify the emulator binary exists.
23	23		EMU_PATH := $(shell command -v $(EMU) 2>/dev/null)
24	24
25	25		default: emulator $(RAD_BIN)
26		-	test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test
	26	+	test: emulator seed-test std-test bin-test kernel-test module-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test termination-test
27	27
28	28		seed-test:
29	29		@seed/test
30	30
	31	+	module-test: emulator $(RAD_BIN)
	32	+	@RAD_EMULATOR="$(EMU)" sh test/modules/run
	33	+
31	34		# Emulator command check
32	35
33	36		emulator:
34	37		ifeq ($(EMU_PATH),)
35	38		$(error Emulator not found. Install it or set RAD_EMULATOR to its path)

211	214
212	215		# Concurrent user contexts and metadata transactions on every online hart.
213	216		.PHONY: smp-test
214	217		smp-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
215	218		@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8" 7
	219	+
	220	+	# User exit, abort, and CPU faults preserve the surviving parent's execution.
	221	+	termination-test: emulator $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	222	+	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run termination
	223	+
	224	+	.PHONY: termination-test

26	26		use std::sys;
27	27		use std::sys::unix;
28	28		use std::collections::dict;
29	29
30	30		/// Maximum number of modules we can load per package.
31		-	constant MAX_LOADED_MODULES: u32 = module::MAX_MODULES;
	31	+	constant MAX_LOADED_MODULES: u32 = 128;
32	32		/// Maximum number of packages we can compile.
33	33		constant MAX_PACKAGES: u32 = 4;
34		-	/// Total module entries across all packages.
35		-	constant MAX_TOTAL_MODULES: u32 = 192;
36	34		/// Source code buffer arena (4 MB).
37	35		constant MAX_SOURCES_SIZE: u32 = 4194304;
38	36		/// Maximum number of test functions we can discover.
39	37		constant MAX_TESTS: u32 = 1024;
40	38		/// Maximum number of assembly source paths we can load per package.

56	54		static MAIN_ARENA: [u8; MAIN_ARENA_SIZE] = [0; MAIN_ARENA_SIZE];
57	55
58	56		/// Module source code.
59	57		static MODULE_SOURCES: [u8; MAX_SOURCES_SIZE] = [0; MAX_SOURCES_SIZE];
60	58		/// Module entries for all packages.
61		-	unsafe static MODULE_ENTRIES: [module::ModuleEntry; MAX_TOTAL_MODULES] = undefined;
	59	+	unsafe static MODULE_ENTRIES: [module::ModuleEntry; module::MAX_MODULES] = undefined;
62	60		/// String pool.
63	61		unsafe static STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 };
64	62
65	63		/// Package scope.
66	64		unsafe static RESOLVER_PKG_SCOPE: resolver::Scope = undefined;

20	20		export mod backing;
21	21		export mod pages;
22	22		export mod loader;
23	23		export mod instances;
24	24		export mod domains;
	25	+	export mod lifecycle;
25	26		export mod budgets;
26	27		export mod dispatch;
27	28		export mod timers;
28	29		export mod remote;
29	30		export mod calls;

133	133		export constant DESTROY: u16 = 256;
134	134		/// All defined right bits.
135	135		export constant RIGHTS_MASK: u16 = 511;
136	136		/// Cascade destruction through creation ancestry.
137	137		export constant CASCADE: u64 = 1;
	138	+	/// Fault code for explicit user Abort, separate from CPU exception codes.
	139	+	export constant FAULT_ABORT: u32 = 256;
138	140
139	141		/// Object kind stored in the high four handle bits.
140	142		export union Kind: Copy {
141	143		/// Unoccupied table entry.
142	144		Empty = 0,

27	27		pool: frames::Pool,
28	28		/// Backing-record lifetimes.
29	29		slots: [slots::Slot; limits::ALLOCATIONS],
30	30		/// Payloads for live backing records.
31	31		records: [Allocation; limits::ALLOCATIONS],
	32	+	/// Exclusive upper bound of backing slots that have been published.
	33	+	used: u32,
32	34		/// Live domain generation, or zero after all of that domain's exposures end.
33	35		domains: [u32; limits::DOMAINS],
	36	+	/// Zero while open; otherwise one plus the next exposure slot to inspect.
	37	+	ending: [u32; limits::DOMAINS],
34	38		}
35	39
36	40		/// Initialize fresh backing and exposure-domain tables.
37	41		export fn initialize(store: &mut Store) {
	42	+	set store.used = 0;
38	43		set store.pool.count = 0;
	44	+	set store.pool.retiredHead = 0;
	45	+	set store.pool.retiredCount = 0;
39	46		slots::initialize(&mut store.slots[..]);
40		-	for i in 0..limits::DOMAINS { set store.domains[i] = 0; }
	47	+	for i in 0..limits::DOMAINS { set store.domains[i] = 0; set store.ending[i] = 0; }
41	48		}
42	49
43	50		/// Register a domain before it can receive allocation exposure.
44	51		export fn registerDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
45	52		if domain.index >= limits::DOMAINS or domain.generation == 0 { throw abi::Error::InvalidArg; }

47	54		set store.domains[domain.index] = domain.generation;
48	55		}
49	56
50	57		/// Test exposure-domain generation without reading beyond the fixed table.
51	58		export fn domainLive(store: &Store, domain: abi::Ref) -> bool {
52		-	return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation;
	59	+	return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation and store.ending[domain.index] == 0;
53	60		}
54	61
55	62		/// Publish reserved metadata with one page object and its live origin exposure.
56	63		/// The caller owns the committed run and has validated the origin domain.
57	64		export unsafe fn publish(store: &mut Store, reservation: slots::Reservation, run: frames::Run, origin: abi::Ref) -> abi::Ref {
58	65		assert domainLive(store, origin);
59	66		let object = slots::reference(&reservation);
60	67		assert slots::matches(&store.slots[..], object, slots::State::Reserved);
	68	+	if object.index >= store.used { set store.used = object.index + 1; }
61	69		set store.records[object.index].run = run;
62	70		set store.records[object.index].pages = 1;
63	71		set store.records[object.index].exposed = 1;
64	72		let mut bits = bitset::init(&mut store.records[object.index].exposures[..]);
65	73		bitset::put(&mut bits, origin.index);

88	96		bitset::put(&mut bits, domain.index);
89	97		set store.records[object.index].exposed += 1;
90	98		}
91	99
92	100		/// Reclaim only when no page object and no domain exposure remains.
93		-	fn reclaim(store: &mut Store, object: abi::Ref) {
	101	+	fn reclaim(store: &mut Store, object: abi::Ref, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) {
94	102		if store.records[object.index].pages <> 0 or store.records[object.index].exposed <> 0 { return; }
95	103		let run = store.records[object.index].run;
96		-	try! frames::release(&mut store.pool, run);
	104	+	try! release(&mut store.pool, run);
97	105		try! slots::release(&mut store.slots[..], object);
98	106		}
99	107
100	108		/// Release one page object after its final handle disappears.
101	109		export fn releasePage(store: &mut Store, object: abi::Ref) throws (abi::Error) {
102	110		try require(store, object);
103	111		if store.records[object.index].pages == 0 { throw abi::Error::BadHandle; }
104	112		set store.records[object.index].pages -= 1;
105		-	reclaim(store, object);
	113	+	reclaim(store, object, frames::release);
106	114		}
107	115
108		-	/// End all exposure for one domain before its slot can be registered again.
109		-	/// Callers serialize this bounded scan with allocation and capability changes.
	116	+	/// End all exposure while the caller has exclusive access to metadata.
110	117		export unsafe fn endDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
111		-	if not domainLive(store, domain) { throw abi::Error::BadHandle; }
112		-	for i in 0..limits::ALLOCATIONS {
	118	+	assert try end(store, domain, limits::ALLOCATIONS, frames::release);
	119	+	}
	120	+
	121	+	/// Inspect at most limit backing slots and retire allocations whose lifetimes end.
	122	+	/// Returns true after all exposure is clear and the domain slot can be registered again.
	123	+	export unsafe fn endDomainStep(store: &mut Store, domain: abi::Ref, limit: u32) -> bool throws (abi::Error) {
	124	+	return try end(store, domain, limit, frames::retire);
	125	+	}
	126	+
	127	+	/// Close exposure admission and advance one generation's retained scan cursor.
	128	+	unsafe fn end(store: &mut Store, domain: abi::Ref, limit: u32,
	129	+	release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) -> bool throws (abi::Error)
	130	+	{
	131	+	if domain.index >= limits::DOMAINS or domain.generation == 0
	132	+	or store.domains[domain.index] <> domain.generation { throw abi::Error::BadHandle; }
	133	+	if limit == 0 { throw abi::Error::InvalidArg; }
	134	+	if store.ending[domain.index] == 0 { set store.ending[domain.index] = 1; }
	135	+	let first = store.ending[domain.index] - 1;
	136	+	let mut count = store.used - first;
	137	+	if count > limit { set count = limit; }
	138	+	let next = first + count;
	139	+	for i in first..next {
113	140		if store.slots[i].state <> slots::State::Live { continue; }
114	141		let mut bits = bitset::new(&mut store.records[i].exposures[..]);
115	142		if not bitset::contains(&bits, domain.index) { continue; }
116	143		bitset::clear(&mut bits, domain.index);
117	144		set store.records[i].exposed -= 1;
118		-	reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation });
	145	+	reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation }, release);
	146	+	}
	147	+	if next < store.used {
	148	+	set store.ending[domain.index] = next + 1;
	149	+	return false;
119	150		}
120	151		set store.domains[domain.index] = 0;
	152	+	set store.ending[domain.index] = 0;
	153	+	return true;
121	154		}

7	7		use super::sync;
8	8		use super::trap;
9	9		use super::pages;
10	10		use super::registry;
11	11		use super::domains;
	12	+	use super::lifecycle;
12	13		use super::budgets;
13	14		use super::dispatch;
14	15		use super::timers;
15	16		use super::remote;
16	17		use super::interrupts;

24	25		/// Number of harts that validated their machine stack.
25	26		static ARRIVED: u64 = 0;
26	27		/// Trap anchors owned by their physical harts.
27	28		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
28	29
29		-	/// Handle timer boundaries and user calls, and stop on unsupported traps.
	30	+	/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
30	31		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
31	32		let cause = trap::classify(frame.cause);
32	33		if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
33	34		dispatch::interrupt(frame, hart);
34		-	return;
35	35		}
36	36		if cause == trap::Cause::UserCall and trap::fromUser(frame) {
37	37		dispatch::call(frame, hart);
38		-	return;
	38	+	}
	39	+	if trap::fromUser(frame) {
	40	+	dispatch::fault(frame, hart);
39	41		}
40	42		print("kernel: unexpected trap\n");
41	43		trap::halt();
42	44		}
43	45

79	81		let treeRange = range::new(treeAddress, size as u64) else panic "FDT range";
80	82		assert platform::inRam(&PLATFORM, treeRange);
81	83		try! platform::protect(&mut PLATFORM, treeRange);
82	84		try! pages::initialize(&mut pages::STORE, &PLATFORM);
83	85		domains::initialize(&mut domains::STORE);
	86	+	lifecycle::initialize(&mut lifecycle::CALLS);
84	87		budgets::initialize(&mut budgets::STORE);
85	88		timers::initialize(&mut timers::STORE);
86	89		remote::initialize(&mut remote::STORE, PLATFORM.harts);
87	90		interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
88	91		try! plic::initialize(&PLATFORM);

234	234		let object = try owned(store, table, handle, abi::TRANSFER);
235	235		let window = try get(store, object);
236	236		if window.hart as u64 <> current.hart or window.context <> abi::reference(current.context)
237	237		or now < window.start or now >= window.end { throw abi::Error::Busy; }
238	238		let permit = try capability::lookup(table, target, abi::Kind::Domain, abi::Rights(abi::WAKE));
239		-	let receiver = try domains::get(domainStore, permit.object) catch { throw abi::Error::Busy; };
	239	+	let receiver = try domains::get(domainStore, permit.object);
	240	+	if receiver.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
240	241		if receiver.state <> domains::Lifecycle::Active or abi::id(receiver.initial) == current.context { throw abi::Error::Busy; }
241	242		let context = try domains::context(domainStore, permit.object, receiver.initial) catch { throw abi::Error::Busy; };
242	243		if context.state <> domains::ContextState::Ready or context.hart <> nil { throw abi::Error::Busy; }
243	244		let result = try bind(store, domainStore, table, Binding { budget: handle, domain: target, context: receiver.initial }, now) catch error {
244	245		if error == abi::Error::InvalidArg { throw abi::Error::Busy; }

45	46		}
46	47
47	48		/// Validate resident object generations for capabilities without page ownership.
48	49		unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
49	50		match entry.kind {
50		-	case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },
	51	+	case abi::Kind::Domain, abi::Kind::Events => {
	52	+	let object = try domains::get(&domains::STORE, entry.object);
	53	+	if object.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
	54	+	},
51	55		case abi::Kind::Image => { let object = try registry::get(&registry::STORE, entry.object); },
52	56		else => throw abi::Error::BadHandle,
53	57		}
54	58		}
55	59
	60	+	/// Reject expired domain authority before an allocation can reserve resources.
	61	+	unsafe fn allocationAuthority(table: &capability::Table, operation: u64, handle: abi::Handle) throws (abi::Error) {
	62	+	if operation <> 20 and operation <> 30 and operation <> 51 { return; }
	63	+	let rights = abi::CREATE if operation == 20 else abi::ALLOCATE;
	64	+	try resident(try capability::authority(table, handle, abi::Rights(rights)));
	65	+	}
	66	+
56	67		/// Grant or transfer authority while both domain tables remain serialized.
57	68		unsafe fn delegate(domain: &mut domains::Domain, handle: abi::Handle, target: abi::Handle, rights: u64, moving: bool, now: u64)
58	69		-> abi::Handle throws (abi::Error)
59	70		{
60	71		let owner = domain.memory.table.owner;

140	151		}
141	152		let output = @sliceOf(memory(arguments[1]), words.len);
142	153		for i in 0..words.len { set output[i] = words[i]; }
143	154		}
144	155
	156	+	/// Validate a termination call and return harts that must release stopped contexts.
	157	+	export unsafe fn termination(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u32 throws (abi::Error) {
	158	+	assert arguments.len == 4;
	159	+	let domain = try domains::get(&domains::STORE, owner);
	160	+	if domain.state <> domains::Lifecycle::Active { throw abi::Error::BadHandle; }
	161	+	if arguments[2] <> 0 or arguments[3] <> 0 { throw abi::Error::InvalidArg; }
	162	+	match operation {
	163	+	case 49 => {
	164	+	if arguments[0] > 0xffffffff or arguments[1] <> 0 { throw abi::Error::InvalidArg; }
	165	+	return try domains::terminate(&mut domains::STORE, owner, events::CHILD_EXIT, arguments[0] as u32);
	166	+	},
	167	+	case 43 => {
	168	+	if arguments[0] <> 0 or arguments[1] <> 0 { throw abi::Error::InvalidArg; }
	169	+	return try domains::terminate(&mut domains::STORE, owner, events::FAULT, abi::FAULT_ABORT);
	170	+	},
	171	+	case 22 => {
	172	+	let flags = arguments[1];
	173	+	if (flags & ~abi::CASCADE) <> 0 or (arguments[0] == 0 and flags <> 0) { throw abi::Error::InvalidArg; }
	174	+	let mut target = owner;
	175	+	if arguments[0] <> 0 {
	176	+	let entry = try capability::lookup(&domain.memory.table, abi::Handle(arguments[0]), abi::Kind::Domain, abi::Rights(abi::DESTROY));
	177	+	set target = entry.object;
	178	+	if (try domains::get(&domains::STORE, target)).state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
	179	+	}
	180	+	if flags == abi::CASCADE { return try domains::terminateTree(&mut domains::STORE, target, events::CHILD_EXIT, 0); }
	181	+	return try domains::terminate(&mut domains::STORE, target, events::CHILD_EXIT, 0);
	182	+	},
	183	+	else => throw abi::Error::InvalidArg,
	184	+	}
	185	+	}
	186	+
145	187		/// Execute one operation with domain and resource metadata serialized.
146	188		export unsafe fn invoke(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
147	189		assert arguments.len == 4;
148	190		let mut domain = try domains::get(&domains::STORE, owner);
149	191		let handle = abi::Handle(arguments[0]);
	192	+	try allocationAuthority(&domain.memory.table, operation, handle);
150	193		match operation {
	194	+	case 22, 43, 49 => { let stopped = try termination(owner, operation, arguments); return 0; },
151	195		case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
152	196		case 12 => { try drop(&mut domain, handle); return 0; },
153	197		case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,
154	198		&registry::STORE, &mut domain.memory.table, handle, abi::Handle(arguments[1])),
155	199		case 21 => {
156	200		try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,
157	201		handle, arguments[1], arguments[2], arguments[3]);
158	202		return 0;
159	203		},
	204	+	case 23 => {
	205	+	if arguments[2] <> 0 or arguments[3] <> 0 { throw abi::Error::InvalidArg; }
	206	+	try domains::reparent(&mut domains::STORE, &domain.memory.table, handle, abi::Handle(arguments[1]));
	207	+	return 0;
	208	+	},
160	209		case 30 => return *try pages::allocate(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
161	210		case 31 => return *try pages::split(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
162	211		case 40 => {
163	212		if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 { throw abi::Error::InvalidArg; }
164	213		try budgets::handoff(&mut budgets::STORE, &domains::STORE, &mut domain.memory.table, dispatch::current(), handle, now);

190	239		return 0;
191	240		},
192	241		case 47 => {
193	242		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
194	243		let target = try domains::get(&domains::STORE, entry.object);
	244	+	if target.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
195	245		let mut parent: u64 = 0;
196	246		if let object = target.parent { set parent = abi::id(object); }
197	247		try reply(&domain, arguments, &[abi::id(entry.object), abi::id(target.creator), parent, target.state as u64]);
198	248		return 0;
199	249		},
200	250		case 48 => {
201	251		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Events, abi::Rights(0));
202	252		let target = try domains::get(&domains::STORE, entry.object);
	253	+	if target.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
203	254		try reply(&domain, arguments, &[events::CAPACITY as u64]);
204	255		return 0;
205	256		},
206	257		case 51 => return *try loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut domain.memory.table,
207	258		loader::Request { authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3] }),

235	286		return 0;
236	287		},
237	288		case 62 => {
238	289		let permission = try capability::authority(&domain.memory.table, handle, abi::Rights(0));
239	290		let target = try domains::get(&domains::STORE, permission.object);
	291	+	if target.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
240	292		let mut object = target.initial;
241	293		if arguments[1] <> 0 { set object = abi::reference(arguments[1]); }
242	294		let context = try domains::context(&domains::STORE, permission.object, object);
243	295		let mut hart: u64 = 0xffffffffffffffff;
244	296		if let id = context.hart { set hart = id as u64; }

276	328		/// Load a package with metadata locked only around resource transactions.
277	329		unsafe fn runtimeLoad(domain: &mut domains::Domain, request: loader::Request, guard: sync::Guard) -> u64 throws (abi::Error) {
278	330		let owner = domain.memory.table.owner;
279	331		let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
280	332		catch error { sync::leave(guard); throw error; };
	333	+	let context = abi::reference(dispatch::current().context);
	334	+	let lease = lifecycle::holdLoad(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
	335	+	let retained = lifecycle::load(&lifecycle::CALLS, &lease);
281	336		sync::leave(guard);
282		-	let input = try loader::decode(&pending) catch error {
	337	+	let input = try loader::decodeInput(&retained.input) catch error {
283	338		let guard = sync::enter();
284		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	339	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	340	+	&mut loader::STATE, &mut registry::STORE, lease);
285	341		sync::leave(guard); throw error;
286	342		};
287		-	let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
	343	+	let existing = try loader::identifyInput(&registry::STORE, &retained.input, &input) catch error {
288	344		let guard = sync::enter();
289		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	345	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	346	+	&mut loader::STATE, &mut registry::STORE, lease);
290	347		sync::leave(guard); throw error;
291	348		};
292	349		let allocationGuard = sync::enter();
293	350		if not live(owner) {
294		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	351	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	352	+	&mut loader::STATE, &mut registry::STORE, lease);
295	353		sync::leave(allocationGuard); throw abi::Error::BadHandle;
296	354		}
297	355		if let object = existing {
298		-	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	356	+	let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	357	+	&mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Existing(object))
299	358		catch error { sync::leave(allocationGuard); throw error; };
300	359		sync::leave(allocationGuard); return *result;
301	360		}
302		-	let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
303		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	361	+	try lifecycle::reserveOutput(&mut lifecycle::CALLS, &lease, &mut pages::STORE, &mut registry::STORE) catch error {
	362	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	363	+	&mut loader::STATE, &mut registry::STORE, lease);
304	364		sync::leave(allocationGuard); throw error;
305	365		};
	366	+	let compiling = lifecycle::load(&lifecycle::CALLS, &lease);
	367	+	let output = compiling.output else panic "missing loader output";
306	368		sync::leave(allocationGuard);
307		-	let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
	369	+	let compiled = try loader::compileInput(&registry::STORE, &compiling.input, &input, &output) catch error {
308	370		let guard = sync::enter();
309		-	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
310		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	371	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	372	+	&mut loader::STATE, &mut registry::STORE, lease);
311	373		sync::leave(guard); throw error;
312	374		};
313	375		let publicationGuard = sync::enter();
314		-	if not live(owner) {
315		-	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
316		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
317		-	sync::leave(publicationGuard); throw abi::Error::BadHandle;
318		-	}
319		-	let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
320		-	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	376	+	let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	377	+	&mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Compiled(compiled))
321	378		catch error { sync::leave(publicationGuard); throw error; };
322	379		sync::leave(publicationGuard);
323	380		return *result;
324	381		}
325	382
326	383		/// Execute a call with short metadata transactions and private bulk initialization.
327	384		/// Context ownership and pending calls retain storage across dispatch boundaries.
328	385		export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
329	386		assert arguments.len == 4;
330		-	if operation == 21 or operation == 50 { return try runtimeReady(owner, operation, arguments, now); }
	387	+	if operation == 21 or operation == 50 { return try runtimeReady(owner, operation, arguments); }
331	388		let guard = sync::enter();
332	389		let mut domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
333	390		if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
	391	+	try allocationAuthority(&domain.memory.table, operation, abi::Handle(arguments[0])) catch error {
	392	+	sync::leave(guard); throw error;
	393	+	};
	394	+	if operation == 22 or operation == 43 or operation == 49 {
	395	+	let stopped = try termination(owner, operation, arguments) catch error { sync::leave(guard); throw error; };
	396	+	dispatch::reschedule(stopped);
	397	+	sync::leave(guard); return 0;
	398	+	}
334	399		let target = try budgetHart(&domain.memory.table, operation, arguments) catch error { sync::leave(guard); throw error; };
335	400		if let destination = target {
336	401		let source = dispatch::hart();
337	402		if destination <> source {
338	403		let call = remote::Call { owner, context: abi::reference(dispatch::current().context), operation,

349	414		let handle = abi::Handle(arguments[0]);
350	415		match operation {
351	416		case 20 => {
352	417		let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
353	418		&mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error { sync::leave(guard); throw error; };
	419	+	let context = abi::reference(dispatch::current().context);
	420	+	let lease = lifecycle::holdDomain(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
354	421		sync::leave(guard);
355		-	try domains::prepare(&registry::STORE, &pending) catch error {
	422	+	try lifecycle::prepareDomain(&lifecycle::CALLS, &registry::STORE, &lease) catch error {
356	423		let guard = sync::enter();
357		-	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	424	+	let failed = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
	425	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, failed);
358	426		sync::leave(guard); throw error;
359	427		};
360	428		let guard = sync::enter();
	429	+	let ready = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
361	430		if not live(owner) {
362		-	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	431	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready);
363	432		sync::leave(guard); throw abi::Error::BadHandle;
364	433		}
365		-	let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
	434	+	let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready)
366	435		catch error { sync::leave(guard); throw error; };
367	436		sync::leave(guard); return *result;
368	437		},
369	438		case 30 => {
370	439		let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
371	440		catch error { sync::leave(guard); throw error; };
	441	+	let context = abi::reference(dispatch::current().context);
	442	+	let lease = lifecycle::holdPage(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
372	443		sync::leave(guard);
373		-	pages::clear(&pending);
	444	+	lifecycle::clearPage(&lifecycle::CALLS, &lease);
374	445		let guard = sync::enter();
	446	+	let ready = lifecycle::takePage(&mut lifecycle::CALLS, &mut domains::STORE, lease);
375	447		if not live(owner) {
376		-	pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
	448	+	pages::cancel(&mut pages::STORE, &mut domain.memory.table, ready);
377	449		sync::leave(guard); throw abi::Error::BadHandle;
378	450		}
379		-	let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
	451	+	let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, ready);
380	452		sync::leave(guard); return *result;
381	453		},
382	454		case 51 => return try runtimeLoad(&mut domain, loader::Request {
383	455		authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
384	456		}, guard),

391	463		let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
392	464		let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
393	465		&domain.memory.table, handle, start) catch error { sync::leave(guard); throw error; };
394	466		let mut target: abi::Ref = undefined;
395	467		match &pending { case domains::ContextReservation::Held(allocation) => { set target = allocation.owner; }, }
	468	+	let context = abi::reference(dispatch::current().context);
	469	+	let lease = lifecycle::holdContext(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
396	470		sync::leave(guard);
397		-	domains::contextClear(&pending);
	471	+	lifecycle::clearContext(&lifecycle::CALLS, &lease);
398	472		let guard = sync::enter();
	473	+	let ready = lifecycle::takeContext(&mut lifecycle::CALLS, &mut domains::STORE, lease);
399	474		if not live(owner) or not live(target) {
400		-	domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
	475	+	domains::contextCancel(&mut domains::STORE, &mut pages::STORE, ready);
401	476		sync::leave(guard); throw abi::Error::BadHandle;
402	477		}
403		-	let result = domains::contextPublish(&mut domains::STORE, pending);
	478	+	let result = domains::contextPublish(&mut domains::STORE, ready);
404	479		sync::leave(guard); return abi::id(result);
405	480		},
406	481		else => {
407		-	let result = try invoke(owner, operation, arguments, now) catch error { sync::leave(guard); throw error; };
	482	+	let result = try invoke(owner, operation, arguments, dispatch::clock()) catch error { sync::leave(guard); throw error; };
408	483		sync::leave(guard); return result;
409	484		},
410	485		}
411	486		}
412	487
413	488		/// Reserve remote dispatch capacity before activation or event publication.
414		-	unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
	489	+	unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u64 throws (abi::Error) {
415	490		if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) { throw abi::Error::InvalidArg; }
416	491		let guard = sync::enter();
417	492		let domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
418	493		if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
419	494		let handle = abi::Handle(arguments[0]);

421	496		let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
422	497		sync::leave(guard); throw error;
423	498		};
424	499		let source = dispatch::hart();
425	500		let context = abi::reference(dispatch::current().context);
	501	+	let now = dispatch::clock();
426	502		let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
427	503		let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error { sync::leave(guard); throw error; };
428	504		let result = try invoke(owner, operation, arguments, now) catch error {
429	505		remote::cancelReservation(&mut remote::STORE, pending);
430	506		sync::leave(guard); throw error;

442	518		{
443	519		assert trap::fromUser(frame) and frame.cause == 8;
444	520		assert trap::advanceCall(frame);
445	521		let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
446	522		let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
	523	+	let operation = frame.registers[17];
	524	+	if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
	525	+	try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
	526	+	assert stopped == abi::Error::BadHandle;
	527	+	};
	528	+	}
447	529		set frame.registers[10] = (0 as u64) - error as u64;
448	530		return;
449	531		};
450	532		set frame.registers[10] = result;
451	533		}

42	49		record Runtime: Copy {
43	50		/// Current context and retained idle frame.
44	51		state: State,
45	52		/// Validated timer registers for this hart.
46	53		timer: Timer,
	54	+	/// Minimum ticks between scheduled maintenance passes.
	55	+	maintenance: u64,
	56	+	/// Earliest tick for another background maintenance pass.
	57	+	nextMaintenance: u64,
47	58		}
48	59
49	60		/// Runtime slots initialized before each hart enables timer interrupts.
50	61		unsafe static HARTS: [Runtime; limits::HARTS] = undefined;
51	62

65	76		let device = try timer(machine, id);
66	77		let stack = machine.stacks[id];
67	78		let mut idle = trap::Frame { registers: [0; 32], pc: idleAddress(), status: 0x1880, cause: 0, value: 0 };
68	79		set idle.registers[2] = stack.end;
69	80		set idle.registers[3] = kernelGp;
	81	+	let mut maintenance = machine.timebase as u64 / 100;
	82	+	if maintenance == 0 { set maintenance = 1; }
70	83		set HARTS[id] = Runtime {
71		-	state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
	84	+	state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0,
72	85		};
73	86		arm(device, 0xffffffffffffffff);
74	87		}
75	88
76	89		/// Tail-enter dispatch on the firmware stack and resume the selected frame.

103	116		set deliveries[count] = delivery;
104	117		set count += 1;
105	118		}
106	119		let clock = now(HARTS[id].timer);
107	120		let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
108		-	let notified = timeout.harts \| plic::service(clock);
	121	+	let notified = timeout.harts \| plic::service(clock) \| lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
109	122		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
110	123		let mut deadline = choice.deadline;
111	124		if timeout.deadline < deadline { set deadline = timeout.deadline; }
112		-	for target in 0..limits::HARTS {
113		-	if target <> id and (notified & (1 << target)) <> 0 { signal(target); }
114		-	}
115		-	arm(HARTS[id].timer, deadline);
	125	+	reschedule(notified);
116	126		if previous <> choice.context {
117	127		if let context = previous { signalContext(context, clock); }
118	128		}
119	129		for delivery in &deliveries[..count] {
120	130		assert remote::acknowledge(&mut remote::STORE, id, delivery.id);
121	131		if delivery.request.source <> id { signal(delivery.request.source); }
122	132		if let case remote::Action::Budget(call) = delivery.request.action { signalContext(call.context, clock); }
123	133		if let case remote::Action::Ready(context) = delivery.request.action { signalContext(context, clock); }
124	134		}
	135	+	lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE);
	136	+	let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE);
	137	+	let maintenanceClock = now(HARTS[id].timer);
	138	+	let mut maintenance = cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
	139	+	if maintenance and maintenanceClock >= HARTS[id].nextMaintenance {
	140	+	let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE);
	141	+	let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256);
	142	+	set maintenance = cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0;
	143	+	let after = now(HARTS[id].timer);
	144	+	let interval = HARTS[id].maintenance;
	145	+	set HARTS[id].nextMaintenance = 0xffffffffffffffff;
	146	+	if interval < 0xffffffffffffffff - after { set HARTS[id].nextMaintenance = after + interval; }
	147	+	}
	148	+	if maintenance and HARTS[id].nextMaintenance < deadline { set deadline = HARTS[id].nextMaintenance; }
	149	+	arm(HARTS[id].timer, deadline);
125	150		sync::leave(guard);
126	151		}
127	152
128	153		/// Signal an initialized online hart while request metadata is serialized.
129	154		unsafe fn signal(target: u32) { writeSoftware(HARTS[target].timer.software, 1); }
130	155
	156	+	/// Signal other harts that retain stopped contexts under the metadata lock.
	157	+	export unsafe fn reschedule(harts: u32) {
	158	+	let source = hart();
	159	+	for target in 0..limits::HARTS {
	160	+	if target <> source and (harts & (1 << target)) <> 0 { signal(target); }
	161	+	}
	162	+	}
	163	+
131	164		/// Notify the current owner or the eligible hart after a context is released.
132	165		unsafe fn signalContext(context: abi::Ref, now: u64) {
133	166		if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) { return; }
134	167		if let owner = domains::STORE.contexts[context.index].hart { signal(owner); return; }
135	168		for i in 0..limits::BUDGETS {

215	248		};
216	249		releaseRequest(source, request);
217	250		sync::leave(guard); return result;
218	251		}
219	252
	253	+	/// Read mtime through the executing hart's validated timer mapping.
	254	+	export unsafe fn clock() -> u64 { return now(HARTS[hart()].timer); }
	255	+
220	256		/// Read the running context's identity and domain-relative CPU authority.
221	257		export unsafe fn current() -> abi::CurrentContextInfo {
222	258		let id = hart();
223	259		assert id < limits::HARTS;
224	260		let context = HARTS[id].state.current else panic "no current context";

235	271		let guard = sync::enter();
236	272		assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
237	273		let owner = domains::STORE.contexts[context.index].owner;
238	274		if frame.registers[17] == abi::Operation::ContextReturn as u64 {
239	275		assert trap::advanceCall(frame);
240		-	try! domains::returned(&mut domains::STORE, owner, context);
	276	+	let stopped = try! domains::returned(&mut domains::STORE, owner, context);
	277	+	reschedule(stopped);
241	278		sync::leave(guard);
242	279		} else {
243	280		sync::leave(guard);
	281	+	sync::restoreInterrupts(8);
244	282		calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
	283	+	let interrupts = sync::maskInterrupts();
245	284		}
246	285		interrupt(frame, &mut *localAnchor());
247	286		}
248	287
	288	+	/// Stop a faulting user domain and resume another authorized context.
	289	+	export unsafe fn fault(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
	290	+	assert trap::fromUser(frame);
	291	+	let guard = sync::enter();
	292	+	let context = HARTS[hart()].state.current else panic "fault without current context";
	293	+	assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
	294	+	let owner = domains::STORE.contexts[context.index].owner;
	295	+	let stopped = try! domains::terminate(&mut domains::STORE, owner, events::FAULT, frame.cause as u32);
	296	+	reschedule(stopped);
	297	+	sync::leave(guard);
	298	+	interrupt(frame, &mut *localAnchor());
	299	+	}
	300	+
249	301		/// Enter the first authorized context or the hart's retained idle frame.
250	302		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
251	303		let id = hart();
252	304		assert id < limits::HARTS;
253	305		let mut frame = HARTS[id].state.idle;

18	18		export constant KERNEL_STACK_PAGES: u32 = 64;
19	19
20	20		/// Management rights returned to a domain's creator.
21	21		export constant MANAGEMENT_RIGHTS: u16 = abi::DESTROY \| abi::EXECUTE \| abi::GRANT \| abi::TRANSFER \| abi::WAKE;
22	22
	23	+	/// Words needed to retain one ancestry bit per domain slot.
	24	+	constant ANCESTRY_WORDS: u32 = (limits::DOMAINS + 31) / 32;
	25	+
23	26		/// Protection-domain lifetime independent of execution budgets.
24	27		export union Lifecycle: Copy {
25	28		/// Resources can be installed before one-time activation.
26	29		Pending,
27	30		/// The domain can execute when a context has CPU authority.
28	31		Active,
29		-	/// Execution has stopped and the identity can remain as a tombstone.
	32	+	/// Contexts cannot be selected; hart release and resource cleanup can remain.
30	33		Dead,
31	34		}
32	35
33	36		/// Private domain metadata and its shared event ring.
34	37		export record Memory: Copy {

58	61		state: ContextState,
59	62		/// Hart retaining this context as its current execution owner.
60	63		hart: ?u32,
61	64		/// Remote call whose retained continuation requires this context's storage.
62	65		pending: ?abi::Ref,
	66	+	/// A kernel call retains unpublished resources and requires this context's stack.
	67	+	reservation: bool,
63	68		/// Physical frames retained until this context stops.
64	69		kernelFrames: frames::Run,
65	70		/// Exclusive stack bounds used by trap entry and suspended M-mode calls.
66	71		kernelStack: range::Range,
67	72		/// Exclusive user stack extent; empty until initial activation.

96	101		export record Store: Copy {
97	102		/// Domain generations, including retained dead identities.
98	103		slots: [slots::Slot; limits::DOMAINS],
99	104		/// Payload for each published domain.
100	105		records: [Domain; limits::DOMAINS],
	106	+	/// True after private storage is retired, while terminal identity can remain.
	107	+	reclaimed: [bool; limits::DOMAINS],
	108	+	/// True after quiescence admits a domain to resource teardown.
	109	+	reaping: [bool; limits::DOMAINS],
	110	+	/// Number of dead domain identities retained by the store.
	111	+	dead: u32,
	112	+	/// First domain slot considered by the next reaper pass.
	113	+	reapCursor: u32,
	114	+	/// Creation ancestors by domain slot. Clear a column before its ancestor slot is reused.
	115	+	ancestors: [[u32; ANCESTRY_WORDS]; limits::DOMAINS],
101	116		/// Context generations shared by initial and additional contexts.
102	117		contextSlots: [slots::Slot; limits::CONTEXTS],
103	118		/// Payload for each published execution context.
104	119		contexts: [Context; limits::CONTEXTS],
105	120		/// Trusted event producers and terminal records.