radiance · commit

kernel: Complete domain teardown and parenting

60847643637a9cab44b538d7f5323f74207a64cd5dafe0c1c66c190d1b2ef6ca

Alexis Sellier committed 13 days ago 1 parent 7aab0c13

kernel/kernel.rad +1 -0

export mod backing;
export mod pages;
export mod loader;
export mod instances;
export mod domains;
export mod lifecycle;
export mod budgets;
export mod dispatch;
export mod timers;
export mod remote;
export mod calls;

kernel/kernel/abi.rad +2 -0

export constant DESTROY: u16 = 256;
/// All defined right bits.
export constant RIGHTS_MASK: u16 = 511;
/// Cascade destruction through creation ancestry.
export constant CASCADE: u64 = 1;
/// Fault code for explicit user Abort, separate from CPU exception codes.
export constant FAULT_ABORT: u32 = 256;

/// Object kind stored in the high four handle bits.
export union Kind: Copy {
    /// Unoccupied table entry.
    Empty = 0,

kernel/kernel/backing.rad +52 -10

    pool: frames::Pool,
    /// Backing-record lifetimes.
    slots: [slots::Slot; limits::ALLOCATIONS],
    /// Payloads for live backing records.
    records: [Allocation; limits::ALLOCATIONS],
    /// Exclusive upper bound of backing slots that have been published.
    used: u32,
    /// Live domain generation, or zero after all of that domain's exposures end.
    domains: [u32; limits::DOMAINS],
    /// Zero while open; otherwise one plus the next exposure slot to inspect.
    ending: [u32; limits::DOMAINS],
}

/// Initialize fresh backing and exposure-domain tables.
export fn initialize(store: &mut Store) {
    set store.used = 0;
    set store.pool.count = 0;
    set store.pool.retiredHead = 0;
    set store.pool.retiredCount = 0;
    slots::initialize(&mut store.slots[..]);
    for i in 0..limits::DOMAINS {
        set store.domains[i] = 0;
        set store.ending[i] = 0;
    }
}

/// Register a domain before it can receive allocation exposure.
export fn registerDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {

    set store.domains[domain.index] = domain.generation;
}

/// Test exposure-domain generation without reading beyond the fixed table.
export fn domainLive(store: &Store, domain: abi::Ref) -> bool {
    return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation;
    return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation and store.ending[domain.index] == 0;
}

/// Publish reserved metadata with one page object and its live origin exposure.
/// The caller owns the committed run and has validated the origin domain.
export unsafe fn publish(store: &mut Store, reservation: slots::Reservation, run: frames::Run, origin: abi::Ref) -> abi::Ref {
    assert domainLive(store, origin);
    let object = slots::reference(&reservation);
    assert slots::matches(&store.slots[..], object, slots::State::Reserved);
    if object.index >= store.used {
        set store.used = object.index + 1;
    }
    set store.records[object.index].run = run;
    set store.records[object.index].pages = 1;
    set store.records[object.index].exposed = 1;
    let mut bits = bitset::init(&mut store.records[object.index].exposures[..]);
    bitset::put(&mut bits, origin.index);

    bitset::put(&mut bits, domain.index);
    set store.records[object.index].exposed += 1;
}

/// Reclaim only when no page object and no domain exposure remains.
fn reclaim(store: &mut Store, object: abi::Ref) {
fn reclaim(store: &mut Store, object: abi::Ref, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) {
    if store.records[object.index].pages <> 0 or store.records[object.index].exposed <> 0 {
        return;
    }
    let run = store.records[object.index].run;
    try! frames::release(&mut store.pool, run);
    try! release(&mut store.pool, run);
    try! slots::release(&mut store.slots[..], object);
}

/// Release one page object after its final handle disappears.
export fn releasePage(store: &mut Store, object: abi::Ref) throws (abi::Error) {
    try require(store, object);
    if store.records[object.index].pages == 0 {
        throw abi::Error::BadHandle;
    }
    set store.records[object.index].pages -= 1;
    reclaim(store, object);
    reclaim(store, object, frames::release);
}

/// End all exposure for one domain before its slot can be registered again.
/// Callers serialize this bounded scan with allocation and capability changes.
/// End all exposure while the caller has exclusive access to metadata.
export unsafe fn endDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
    if not domainLive(store, domain) {
        throw abi::Error::BadHandle;
    assert try end(store, domain, limits::ALLOCATIONS, frames::release);
}

/// Inspect at most limit backing slots and retire allocations whose lifetimes end.
/// Returns true after all exposure is clear and the domain slot can be registered again.
export unsafe fn endDomainStep(store: &mut Store, domain: abi::Ref, limit: u32) -> bool throws (abi::Error) {
    return try end(store, domain, limit, frames::retire);
}

/// Close exposure admission and advance one generation's retained scan cursor.
unsafe fn end(store: &mut Store, domain: abi::Ref, limit: u32,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) -> bool throws (abi::Error)
{
    if domain.index >= limits::DOMAINS or domain.generation == 0
        or store.domains[domain.index] <> domain.generation {
            throw abi::Error::BadHandle;
        }
    if limit == 0 {
        throw abi::Error::InvalidArg;
    }
    if store.ending[domain.index] == 0 {
        set store.ending[domain.index] = 1;
    }
    for i in 0..limits::ALLOCATIONS {
    let first = store.ending[domain.index] - 1;
    let mut count = store.used - first;
    if count > limit {
        set count = limit;
    }
    let next = first + count;
    for i in first..next {
        if store.slots[i].state <> slots::State::Live {
            continue;
        }
        let mut bits = bitset::new(&mut store.records[i].exposures[..]);
        if not bitset::contains(&bits, domain.index) {
            continue;
        }
        bitset::clear(&mut bits, domain.index);
        set store.records[i].exposed -= 1;
        reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation });
        reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation }, release);
    }
    if next < store.used {
        set store.ending[domain.index] = next + 1;
        return false;
    }
    set store.domains[domain.index] = 0;
    set store.ending[domain.index] = 0;
    return true;
}

kernel/kernel/boot.rad +6 -3

use super::sync;
use super::trap;
use super::pages;
use super::registry;
use super::domains;
use super::lifecycle;
use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use super::interrupts;

/// Number of harts that validated their machine stack.
static ARRIVED: u64 = 0;
/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and user calls, and stop on unsupported traps.
/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    let cause = trap::classify(frame.cause);
    if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
        dispatch::interrupt(frame, hart);
        return;
    }
    if cause == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);
        return;
    }
    if trap::fromUser(frame) {
        dispatch::fault(frame, hart);
    }
    print("kernel: unexpected trap\n");
    trap::halt();
}


        let treeRange = range::new(treeAddress, size as u64) else panic "FDT range";
        assert platform::inRam(&PLATFORM, treeRange);
        try! platform::protect(&mut PLATFORM, treeRange);
        try! pages::initialize(&mut pages::STORE, &PLATFORM);
        domains::initialize(&mut domains::STORE);
        lifecycle::initialize(&mut lifecycle::CALLS);
        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
        try! plic::initialize(&PLATFORM);

kernel/kernel/budgets.rad +4 -3

    if window.hart as u64 <> current.hart or window.context <> abi::reference(current.context)
        or now < window.start or now >= window.end {
            throw abi::Error::Busy;
        }
    let permit = try capability::lookup(table, target, abi::Kind::Domain, abi::Rights(abi::WAKE));
    let receiver = try domains::get(domainStore, permit.object) catch {
        throw abi::Error::Busy;
    };
    let receiver = try domains::get(domainStore, permit.object);
    if receiver.state == domains::Lifecycle::Dead {
        throw abi::Error::BadHandle;
    }
    if receiver.state <> domains::Lifecycle::Active or abi::id(receiver.initial) == current.context {
        throw abi::Error::Busy;
    }
    let context = try domains::context(domainStore, permit.object, receiver.initial) catch {
        throw abi::Error::Busy;

kernel/kernel/calls.rad +143 -31

use super::timers;
use super::sync;
use super::remote;
use super::interrupts;
use super::plic;
use super::lifecycle;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {
    match operation {
        case 11, 12, 70, 71, 73 => {

/// Validate resident object generations for capabilities without page ownership.
unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => {
            let object = try domains::get(&domains::STORE, entry.object);
            if object.state == domains::Lifecycle::Dead {
                throw abi::Error::BadHandle;
            }
        },
        case abi::Kind::Image => {
            let object = try registry::get(&registry::STORE, entry.object);
        },
        else => throw abi::Error::BadHandle,
    }
}

/// Reject expired domain authority before an allocation can reserve resources.
unsafe fn allocationAuthority(table: &capability::Table, operation: u64, handle: abi::Handle) throws (abi::Error) {
    if operation <> 20 and operation <> 30 and operation <> 51 {
        return;
    }
    let rights = abi::CREATE if operation == 20 else abi::ALLOCATE;
    try resident(try capability::authority(table, handle, abi::Rights(rights)));
}

/// Grant or transfer authority while both domain tables remain serialized.
unsafe fn delegate(domain: &mut domains::Domain, handle: abi::Handle, target: abi::Handle, rights: u64, moving: bool, now: u64)
    -> abi::Handle throws (abi::Error)
{
    let owner = domain.memory.table.owner;

    for i in 0..words.len {
        set output[i] = words[i];
    }
}

/// Validate a termination call and return harts that must release stopped contexts.
export unsafe fn termination(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u32 throws (abi::Error) {
    assert arguments.len == 4;
    let domain = try domains::get(&domains::STORE, owner);
    if domain.state <> domains::Lifecycle::Active {
        throw abi::Error::BadHandle;
    }
    if arguments[2] <> 0 or arguments[3] <> 0 {
        throw abi::Error::InvalidArg;
    }
    match operation {
        case 49 => {
            if arguments[0] > 0xffffffff or arguments[1] <> 0 {
                throw abi::Error::InvalidArg;
            }
            return try domains::terminate(&mut domains::STORE, owner, events::CHILD_EXIT, arguments[0] as u32);
        },
        case 43 => {
            if arguments[0] <> 0 or arguments[1] <> 0 {
                throw abi::Error::InvalidArg;
            }
            return try domains::terminate(&mut domains::STORE, owner, events::FAULT, abi::FAULT_ABORT);
        },
        case 22 => {
            let flags = arguments[1];
            if (flags & ~abi::CASCADE) <> 0 or (arguments[0] == 0 and flags <> 0) {
                throw abi::Error::InvalidArg;
            }
            let mut target = owner;
            if arguments[0] <> 0 {
                let entry = try capability::lookup(&domain.memory.table, abi::Handle(arguments[0]), abi::Kind::Domain, abi::Rights(abi::DESTROY));
                set target = entry.object;
                if (try domains::get(&domains::STORE, target)).state == domains::Lifecycle::Dead {
                    throw abi::Error::BadHandle;
                }
            }
            if flags == abi::CASCADE {
                return try domains::terminateTree(&mut domains::STORE, target, events::CHILD_EXIT, 0);
            }
            return try domains::terminate(&mut domains::STORE, target, events::CHILD_EXIT, 0);
        },
        else => throw abi::Error::InvalidArg,
    }
}

/// Execute one operation with domain and resource metadata serialized.
export unsafe fn invoke(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    let mut domain = try domains::get(&domains::STORE, owner);
    let handle = abi::Handle(arguments[0]);
    try allocationAuthority(&domain.memory.table, operation, handle);
    match operation {
        case 22, 43, 49 => {
            let stopped = try termination(owner, operation, arguments);
            return 0;
        },
        case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
        case 12 => {
            try drop(&mut domain, handle);
            return 0;
        },

        case 21 => {
            try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,
                handle, arguments[1], arguments[2], arguments[3]);
            return 0;
        },
        case 23 => {
            if arguments[2] <> 0 or arguments[3] <> 0 {
                throw abi::Error::InvalidArg;
            }
            try domains::reparent(&mut domains::STORE, &domain.memory.table, handle, abi::Handle(arguments[1]));
            return 0;
        },
        case 30 => return *try pages::allocate(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
        case 31 => return *try pages::split(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
        case 40 => {
            if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 {
                throw abi::Error::InvalidArg;

            return 0;
        },
        case 47 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
            let target = try domains::get(&domains::STORE, entry.object);
            if target.state == domains::Lifecycle::Dead {
                throw abi::Error::BadHandle;
            }
            let mut parent: u64 = 0;
            if let object = target.parent {
                set parent = abi::id(object);
            }
            try reply(&domain, arguments, &[abi::id(entry.object), abi::id(target.creator), parent, target.state as u64]);
            return 0;
        },
        case 48 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Events, abi::Rights(0));
            let target = try domains::get(&domains::STORE, entry.object);
            if target.state == domains::Lifecycle::Dead {
                throw abi::Error::BadHandle;
            }
            try reply(&domain, arguments, &[events::CAPACITY as u64]);
            return 0;
        },
        case 51 => return *try loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut domain.memory.table,
            loader::Request { authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3] }),

            return 0;
        },
        case 62 => {
            let permission = try capability::authority(&domain.memory.table, handle, abi::Rights(0));
            let target = try domains::get(&domains::STORE, permission.object);
            if target.state == domains::Lifecycle::Dead {
                throw abi::Error::BadHandle;
            }
            let mut object = target.initial;
            if arguments[1] <> 0 {
                set object = abi::reference(arguments[1]);
            }
            let context = try domains::context(&domains::STORE, permission.object, object);

    let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
        catch error {
            sync::leave(guard);
            throw error;
        };
    let context = abi::reference(dispatch::current().context);
    let lease = lifecycle::holdLoad(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
    let retained = lifecycle::load(&lifecycle::CALLS, &lease);
    sync::leave(guard);
    let input = try loader::decode(&pending) catch error {
    let input = try loader::decodeInput(&retained.input) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(guard); throw error;
    };
    let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
    let existing = try loader::identifyInput(&registry::STORE, &retained.input, &input) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(guard); throw error;
    };
    let allocationGuard = sync::enter();
    if not live(owner) {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(allocationGuard); throw abi::Error::BadHandle;
    }
    if let object = existing {
        let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
        let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Existing(object))
            catch error {
                sync::leave(allocationGuard);
                throw error;
            };
        sync::leave(allocationGuard); return *result;
    }
    let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
    try lifecycle::reserveOutput(&mut lifecycle::CALLS, &lease, &mut pages::STORE, &mut registry::STORE) catch error {
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(allocationGuard); throw error;
    };
    let compiling = lifecycle::load(&lifecycle::CALLS, &lease);
    let output = compiling.output else panic "missing loader output";
    sync::leave(allocationGuard);
    let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
    let compiled = try loader::compileInput(&registry::STORE, &compiling.input, &input, &output) catch error {
        let guard = sync::enter();
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
            &mut loader::STATE, &mut registry::STORE, lease);
        sync::leave(guard); throw error;
    };
    let publicationGuard = sync::enter();
    if not live(owner) {
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(publicationGuard); throw abi::Error::BadHandle;
    }
    let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
    let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
    let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
        &mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Compiled(compiled))
        catch error {
            sync::leave(publicationGuard);
            throw error;
        };
    sync::leave(publicationGuard);

/// Execute a call with short metadata transactions and private bulk initialization.
/// Context ownership and pending calls retain storage across dispatch boundaries.
export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    if operation == 21 or operation == 50 {
        return try runtimeReady(owner, operation, arguments, now);
        return try runtimeReady(owner, operation, arguments);
    }
    let guard = sync::enter();
    let mut domain = try domains::get(&domains::STORE, owner) catch error {
        sync::leave(guard);
        throw error;
    };
    if domain.state <> domains::Lifecycle::Active {
        sync::leave(guard);
        throw abi::Error::BadHandle;
    }
    try allocationAuthority(&domain.memory.table, operation, abi::Handle(arguments[0])) catch error {
        sync::leave(guard); throw error;
    };
    if operation == 22 or operation == 43 or operation == 49 {
        let stopped = try termination(owner, operation, arguments) catch error {
            sync::leave(guard);
            throw error;
        };
        dispatch::reschedule(stopped);
        sync::leave(guard); return 0;
    }
    let target = try budgetHart(&domain.memory.table, operation, arguments) catch error {
        sync::leave(guard);
        throw error;
    };
    if let destination = target {

            let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
                &mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error {
                    sync::leave(guard);
                    throw error;
                };
            let context = abi::reference(dispatch::current().context);
            let lease = lifecycle::holdDomain(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
            sync::leave(guard);
            try domains::prepare(&registry::STORE, &pending) catch error {
            try lifecycle::prepareDomain(&lifecycle::CALLS, &registry::STORE, &lease) catch error {
                let guard = sync::enter();
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                let failed = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, failed);
                sync::leave(guard); throw error;
            };
            let guard = sync::enter();
            let ready = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
            if not live(owner) {
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
            let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready)
                catch error {
                    sync::leave(guard);
                    throw error;
                };
            sync::leave(guard); return *result;

            let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
                catch error {
                    sync::leave(guard);
                    throw error;
                };
            let context = abi::reference(dispatch::current().context);
            let lease = lifecycle::holdPage(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
            sync::leave(guard);
            pages::clear(&pending);
            lifecycle::clearPage(&lifecycle::CALLS, &lease);
            let guard = sync::enter();
            let ready = lifecycle::takePage(&mut lifecycle::CALLS, &mut domains::STORE, lease);
            if not live(owner) {
                pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
                pages::cancel(&mut pages::STORE, &mut domain.memory.table, ready);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
            let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, ready);
            sync::leave(guard); return *result;
        },
        case 51 => return try runtimeLoad(&mut domain, loader::Request {
            authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
        }, guard),

            match &pending {
                case domains::ContextReservation::Held(allocation) => {
                    set target = allocation.owner;
                },
            }
            let context = abi::reference(dispatch::current().context);
            let lease = lifecycle::holdContext(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
            sync::leave(guard);
            domains::contextClear(&pending);
            lifecycle::clearContext(&lifecycle::CALLS, &lease);
            let guard = sync::enter();
            let ready = lifecycle::takeContext(&mut lifecycle::CALLS, &mut domains::STORE, lease);
            if not live(owner) or not live(target) {
                domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
                domains::contextCancel(&mut domains::STORE, &mut pages::STORE, ready);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = domains::contextPublish(&mut domains::STORE, pending);
            let result = domains::contextPublish(&mut domains::STORE, ready);
            sync::leave(guard); return abi::id(result);
        },
        else => {
            let result = try invoke(owner, operation, arguments, now) catch error {
            let result = try invoke(owner, operation, arguments, dispatch::clock()) catch error {
                sync::leave(guard);
                throw error;
            };
            sync::leave(guard); return result;
        },
    }
}

/// Reserve remote dispatch capacity before activation or event publication.
unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u64 throws (abi::Error) {
    if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) {
        throw abi::Error::InvalidArg;
    }
    let guard = sync::enter();
    let domain = try domains::get(&domains::STORE, owner) catch error {

    let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
        sync::leave(guard); throw error;
    };
    let source = dispatch::hart();
    let context = abi::reference(dispatch::current().context);
    let now = dispatch::clock();
    let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
    let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error {
        sync::leave(guard);
        throw error;
    };

{
    assert trap::fromUser(frame) and frame.cause == 8;
    assert trap::advanceCall(frame);
    let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
    let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
        let operation = frame.registers[17];
        if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
            try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
                assert stopped == abi::Error::BadHandle;
            };
        }
        set frame.registers[10] = (0 as u64) - error as u64;
        return;
    };
    set frame.registers[10] = result;
}

kernel/kernel/dispatch.rad +67 -9

use super::abi;
use super::limits;
use super::slots;
use super::budgets;
use super::domains;
use super::lifecycle;
use super::loader;
use super::registry;
use super::frames;
use super::interrupts;
use super::events;
use super::pages;
use super::platform;
use super::trap;
use super::range;
use super::calls;
use super::timers;

record Runtime: Copy {
    /// Current context and retained idle frame.
    state: State,
    /// Validated timer registers for this hart.
    timer: Timer,
    /// Minimum ticks between scheduled maintenance passes.
    maintenance: u64,
    /// Earliest tick for another background maintenance pass.
    nextMaintenance: u64,
}

/// Runtime slots initialized before each hart enables timer interrupts.
unsafe static HARTS: [Runtime; limits::HARTS] = undefined;


    let device = try timer(machine, id);
    let stack = machine.stacks[id];
    let mut idle = trap::Frame { registers: [0; 32], pc: idleAddress(), status: 0x1880, cause: 0, value: 0 };
    set idle.registers[2] = stack.end;
    set idle.registers[3] = kernelGp;
    let mut maintenance = machine.timebase as u64 / 100;
    if maintenance == 0 {
        set maintenance = 1;
    }
    set HARTS[id] = Runtime {
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0,
    };
    arm(device, 0xffffffffffffffff);
}

/// Tail-enter dispatch on the firmware stack and resume the selected frame.

        set deliveries[count] = delivery;
        set count += 1;
    }
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
    let notified = timeout.harts | plic::service(clock);
    let notified = timeout.harts | plic::service(clock) | lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout.deadline < deadline {
        set deadline = timeout.deadline;
    }
    for target in 0..limits::HARTS {
        if target <> id and (notified & (1 << target)) <> 0 {
            signal(target);
        }
    }
    arm(HARTS[id].timer, deadline);
    reschedule(notified);
    if previous <> choice.context {
        if let context = previous {
            signalContext(context, clock);
        }
    }

        }
        if let case remote::Action::Ready(context) = delivery.request.action {
            signalContext(context, clock);
        }
    }
    lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE);
    let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE);
    let maintenanceClock = now(HARTS[id].timer);
    let mut maintenance = cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
    if maintenance and maintenanceClock >= HARTS[id].nextMaintenance {
        let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE);
        let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256);
        set maintenance = cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0;
        let after = now(HARTS[id].timer);
        let interval = HARTS[id].maintenance;
        set HARTS[id].nextMaintenance = 0xffffffffffffffff;
        if interval < 0xffffffffffffffff - after {
            set HARTS[id].nextMaintenance = after + interval;
        }
    }
    if maintenance and HARTS[id].nextMaintenance < deadline {
        set deadline = HARTS[id].nextMaintenance;
    }
    arm(HARTS[id].timer, deadline);
    sync::leave(guard);
}

/// Signal an initialized online hart while request metadata is serialized.
unsafe fn signal(target: u32) {
    writeSoftware(HARTS[target].timer.software, 1);
}

/// Signal other harts that retain stopped contexts under the metadata lock.
export unsafe fn reschedule(harts: u32) {
    let source = hart();
    for target in 0..limits::HARTS {
        if target <> source and (harts & (1 << target)) <> 0 {
            signal(target);
        }
    }
}

/// Notify the current owner or the eligible hart after a context is released.
unsafe fn signalContext(context: abi::Ref, now: u64) {
    if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) {
        return;
    }

    };
    releaseRequest(source, request);
    sync::leave(guard); return result;
}

/// Read mtime through the executing hart's validated timer mapping.
export unsafe fn clock() -> u64 { return now(HARTS[hart()].timer); }

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else panic "no current context";

    let guard = sync::enter();
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    let owner = domains::STORE.contexts[context.index].owner;
    if frame.registers[17] == abi::Operation::ContextReturn as u64 {
        assert trap::advanceCall(frame);
        try! domains::returned(&mut domains::STORE, owner, context);
        let stopped = try! domains::returned(&mut domains::STORE, owner, context);
        reschedule(stopped);
        sync::leave(guard);
    } else {
        sync::leave(guard);
        sync::restoreInterrupts(8);
        calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
        let interrupts = sync::maskInterrupts();
    }
    interrupt(frame, &mut *localAnchor());
}

/// Stop a faulting user domain and resume another authorized context.
export unsafe fn fault(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
    assert trap::fromUser(frame);
    let guard = sync::enter();
    let context = HARTS[hart()].state.current else panic "fault without current context";
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    let owner = domains::STORE.contexts[context.index].owner;
    let stopped = try! domains::terminate(&mut domains::STORE, owner, events::FAULT, frame.cause as u32);
    reschedule(stopped);
    sync::leave(guard);
    interrupt(frame, &mut *localAnchor());
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let mut frame = HARTS[id].state.idle;

kernel/kernel/domains.rad +251 -30

export constant KERNEL_STACK_PAGES: u32 = 64;

/// Management rights returned to a domain's creator.
export constant MANAGEMENT_RIGHTS: u16 = abi::DESTROY | abi::EXECUTE | abi::GRANT | abi::TRANSFER | abi::WAKE;

/// Words needed to retain one ancestry bit per domain slot.
constant ANCESTRY_WORDS: u32 = (limits::DOMAINS + 31) / 32;

/// Protection-domain lifetime independent of execution budgets.
export union Lifecycle: Copy {
    /// Resources can be installed before one-time activation.
    Pending,
    /// The domain can execute when a context has CPU authority.
    Active,
    /// Execution has stopped and the identity can remain as a tombstone.
    /// Contexts cannot be selected; hart release and resource cleanup can remain.
    Dead,
}

/// Private domain metadata and its shared event ring.
export record Memory: Copy {

    state: ContextState,
    /// Hart retaining this context as its current execution owner.
    hart: ?u32,
    /// Remote call whose retained continuation requires this context's storage.
    pending: ?abi::Ref,
    /// A kernel call retains unpublished resources and requires this context's stack.
    reservation: bool,
    /// Physical frames retained until this context stops.
    kernelFrames: frames::Run,
    /// Exclusive stack bounds used by trap entry and suspended M-mode calls.
    kernelStack: range::Range,
    /// Exclusive user stack extent; empty until initial activation.

export record Store: Copy {
    /// Domain generations, including retained dead identities.
    slots: [slots::Slot; limits::DOMAINS],
    /// Payload for each published domain.
    records: [Domain; limits::DOMAINS],
    /// True after private storage is retired, while terminal identity can remain.
    reclaimed: [bool; limits::DOMAINS],
    /// True after quiescence admits a domain to resource teardown.
    reaping: [bool; limits::DOMAINS],
    /// Number of dead domain identities retained by the store.
    dead: u32,
    /// First domain slot considered by the next reaper pass.
    reapCursor: u32,
    /// Creation ancestors by domain slot. Clear a column before its ancestor slot is reused.
    ancestors: [[u32; ANCESTRY_WORDS]; limits::DOMAINS],
    /// Context generations shared by initial and additional contexts.
    contextSlots: [slots::Slot; limits::CONTEXTS],
    /// Payload for each published execution context.
    contexts: [Context; limits::CONTEXTS],
    /// Trusted event producers and terminal records.

/// Address of the U-mode normal-return call sequence.
fn returnAddress() -> u64;

/// Initialize fresh metadata before boot-domain publication.
export fn initialize(store: &mut Store) {
    set store.dead = 0;
    set store.reapCursor = 0;
    slots::initialize(&mut store.slots[..]);
    slots::initialize(&mut store.contextSlots[..]);
    for i in 0..limits::CONTEXTS {
        set store.contexts[i].reservation = false;
    }
    for i in 0..limits::DOMAINS {
        set store.ancestors[i] = [0;ANCESTRY_WORDS];
        set store.reclaimed[i] = false;
        set store.reaping[i] = false;
    }
    events::initialize(&mut store.events);
}

/// Read a domain record after validating its retained generation.
export fn get(store: &Store, object: abi::Ref) -> Domain throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) or store.reclaimed[object.index] {
        throw abi::Error::BadHandle;
    }
    return store.records[object.index];
}

/// Test whether all published and unpublished execution ownership has ended.
export fn quiescent(store: &Store, owner: abi::Ref) -> bool throws (abi::Error) {
    let domain = try get(store, owner);
    if domain.state <> Lifecycle::Dead {
        return false;
    }
    for i in 0..limits::CONTEXTS {
        let state = store.contextSlots[i].state;
        if state <> slots::State::Live and state <> slots::State::Reserved {
            continue;
        }
        if store.contexts[i].owner <> owner {
            continue;
        }
        if state == slots::State::Reserved or store.contexts[i].state <> ContextState::Stopped
            or store.contexts[i].hart <> nil or store.contexts[i].pending <> nil or store.contexts[i].reservation {
                return false;
            }
    }
    return true;
}

/// Retire at most one stopped context after all domain execution ownership ends.
export fn retireContext(store: &mut Store, pool: &mut frames::Pool, owner: abi::Ref) -> bool throws (abi::Error) {
    if not try quiescent(store, owner) {
        throw abi::Error::Busy;
    }
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state <> slots::State::Live or store.contexts[i].owner <> owner {
            continue;
        }
        let object = abi::Ref { index: i, generation: store.contextSlots[i].generation };
        try! frames::retire(pool, store.contexts[i].kernelFrames);
        try! slots::release(&mut store.contextSlots[..], object);
        return true;
    }
    return false;
}

/// Retire private domain storage after contexts, handles, and exposures are gone.
export unsafe fn retireStorage(store: &mut Store, backings: &mut backing::Store, owner: abi::Ref) throws (abi::Error) {
    let domain = try get(store, owner);
    if not try quiescent(store, owner) {
        throw abi::Error::Busy;
    }
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state == slots::State::Live and store.contexts[i].owner == owner {
            throw abi::Error::Busy;
        }
    }
    for i in 0..limits::HANDLES {
        let state = domain.memory.table.slots[i].state;
        if state == slots::State::Live or state == slots::State::Reserved {
            throw abi::Error::Busy;
        }
    }
    if backings.domains[owner.index] <> 0 or store.events.queues[owner.index].generation <> 0 {
        throw abi::Error::Busy;
    }
    try! frames::retire(&mut backings.pool, domain.graph.frames);
    try! frames::retire(&mut backings.pool, domain.allocation);
    set store.reclaimed[owner.index] = true;
}

/// Release an acknowledged terminal identity and clear its creation-ancestry column.
export fn releaseIdentity(store: &mut Store, owner: abi::Ref) throws (abi::Error) {
    if not slots::matches(&store.slots[..], owner, slots::State::Live) {
        throw abi::Error::BadHandle;
    }
    if not store.reclaimed[owner.index] or not try events::acknowledged(&store.events, owner) {
        throw abi::Error::Busy;
    }
    let word = owner.index / 32;
    let bit = 1 << (owner.index % 32);
    for i in 0..limits::DOMAINS {
        set store.ancestors[i][word] &= ~bit;
    }
    try! events::release(&mut store.events, owner);
    try! slots::release(&mut store.slots[..], owner);
    assert store.dead > 0;
    set store.dead -= 1;
}

/// Read a live context after checking its generation and owning domain.
export fn context(store: &Store, owner: abi::Ref, object: abi::Ref) -> Context throws (abi::Error) {
    let domain = try get(store, owner);
    if not slots::matches(&store.contextSlots[..], object, slots::State::Live)
        or store.contexts[object.index].owner <> owner {
            throw abi::Error::BadHandle;
        }
    return store.contexts[object.index];
}

/// Change the live lifecycle receiver under shared metadata serialization.
export fn reparent(store: &mut Store, table: &capability::Table, child: abi::Handle, parent: abi::Handle)
    throws (abi::Error)
{
    let target = try capability::lookup(table, child, abi::Kind::Domain, abi::Rights(abi::DESTROY));
    let receiver = try capability::lookup(table, parent, abi::Kind::Domain, abi::Rights(abi::WAKE));
    let domain = try get(store, target.object);
    let destination = try get(store, receiver.object);
    if domain.state == Lifecycle::Dead or destination.state == Lifecycle::Dead {
        throw abi::Error::BadHandle;
    }
    set store.records[target.object.index].parent = receiver.object;
}

/// Retain a terminal event, stop execution, and report harts that must release contexts.
export fn terminate(store: &mut Store, owner: abi::Ref, kind: u16, code: u32) -> u32 throws (abi::Error) {
    return try stop(store, owner, false, kind, code);
}

/// Stop a domain and its creation descendants without changing creation ancestry.
export fn terminateTree(store: &mut Store, owner: abi::Ref, kind: u16, code: u32) -> u32 throws (abi::Error) {
    return try stop(store, owner, true, kind, code);
}

/// Commit terminal state for a selected set before releasing the metadata lock.
fn stop(store: &mut Store, owner: abi::Ref, tree: bool, kind: u16, code: u32) -> u32 throws (abi::Error) {
    if kind <> events::CHILD_EXIT and kind <> events::FAULT {
        throw abi::Error::InvalidArg;
    }
    let domain = try get(store, owner);
    let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
    for i in 0..limits::DOMAINS {
        if store.slots[i].state <> slots::State::Live or store.records[i].state == Lifecycle::Dead {
            continue;
        }
        if i <> owner.index and (not tree or (store.ancestors[i][owner.index / 32] & (1 << (owner.index % 32))) == 0) {
            continue;
        }
        if store.events.terminals[i].state <> events::State::Free {
            throw abi::Error::Busy;
        }
        if store.events.queues[i].generation <> store.slots[i].generation {
            throw abi::Error::BadHandle;
        }
        set generations[i] = store.slots[i].generation;
    }
    for i in 0..limits::DOMAINS {
        if generations[i] == 0 {
            continue;
        }
        let object = abi::Ref { index: i, generation: generations[i] };
        let terminal = try! events::reserve(&mut store.events, object);
        let mut receiver = abi::Ref { index: 0, generation: 0 };
        if let parent = store.records[i].parent {
            set receiver = parent;
        }
        events::finish(&mut store.events, terminal, receiver, kind, code);
        set store.records[i].state = Lifecycle::Dead;
        set store.dead += 1;
    }
    try! events::closeSet(&mut store.events, &generations[..]);
    let mut harts: u32 = 0;
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state <> slots::State::Live {
            continue;
        }
        let target = store.contexts[i].owner;
        if generations[target.index] == target.generation {
            set store.contexts[i].state = ContextState::Stopped;
            if let hart = store.contexts[i].hart {
                set harts |= 1 << hart;
            }
        }
    }
    for i in 0..limits::DOMAINS {
        if store.slots[i].state <> slots::State::Live {
            continue;
        }
        if let parent = store.records[i].parent {
            if generations[parent.index] == parent.generation {
                set store.records[i].parent = nil;
            }
        }
    }
    return harts;
}

/// Stop a returned context; initial return ends execution of its whole domain.
export fn returned(store: &mut Store, owner: abi::Ref, object: abi::Ref) throws (abi::Error) {
export fn returned(store: &mut Store, owner: abi::Ref, object: abi::Ref) -> u32 throws (abi::Error) {
    let current = try context(store, owner, object);
    if store.records[owner.index].initial == object {
        set store.records[owner.index].state = Lifecycle::Dead;
        for i in 0..limits::CONTEXTS {
            if store.contextSlots[i].state == slots::State::Live and store.contexts[i].owner == owner {
                set store.contexts[i].state = ContextState::Stopped;
            }
        }
    } else {
        set store.contexts[object.index].state = ContextState::Stopped;
        return try terminate(store, owner, events::CHILD_EXIT, 0);
    }
    set store.contexts[object.index].state = ContextState::Stopped;
    return 0;
}

/// Atomically check notifications and suspend the caller if its queue is empty.
/// The caller serializes this check with notification publication.
export unsafe fn wait(store: &mut Store, owner: abi::Ref, object: abi::Ref) throws (abi::Error) {

export fn contextDestroy(store: &mut Store, memory: &mut pages::Store, table: &capability::Table,
    authority: abi::Handle, object: abi::Ref) throws (abi::Error)
{
    let permit = try capability::authority(table, authority, abi::Rights(abi::DESTROY));
    let target = try context(store, permit.object, object);
    if store.records[permit.object.index].state == Lifecycle::Dead {
        throw abi::Error::BadHandle;
    }
    if store.records[permit.object.index].initial == object {
        throw abi::Error::Denied;
    }
    if target.hart <> nil or target.pending <> nil {
    if target.hart <> nil or target.pending <> nil or target.reservation {
        throw abi::Error::Busy;
    }
    try! frames::release(&mut memory.backings.pool, target.kernelFrames);
    try! slots::release(&mut store.contextSlots[..], object);
}

export unsafe fn activate(store: &mut Store, memory: &pages::Store, table: &capability::Table,
    handle: abi::Handle, stack: u64, args: u64, size: u64) throws (abi::Error)
{
    let entry = try capability::lookup(table, handle, abi::Kind::Domain, abi::Rights(abi::EXECUTE));
    let mut domain = try get(store, entry.object);
    if domain.state == Lifecycle::Dead {
        throw abi::Error::BadHandle;
    }
    if domain.state <> Lifecycle::Pending {
        throw abi::Error::NotPending;
    }
    let userStack = try stackRange(memory, &domain.memory.table, stack);
    if size <> 0 and not pages::accessible(memory, &domain.memory.table, args, size, abi::Rights(abi::READ)) {

    set frame.registers[3] = domain.graph.table.ptr as u64;
    set frame.registers[10] = start.args;
    set frame.registers[11] = start.size;
    let object = slots::reference(&slot);
    set store.contexts[object.index] = Context {
        owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames, kernelStack, userStack, frame,
        owner: permission.object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames, kernelStack, userStack, frame,
    };
    match slot {
        case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
            object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
        }),
    }
}

/// Clear an unpublished context's private kernel stack.
export fn contextClear(reservation: &ContextReservation) {
    match reservation { case ContextReservation::Held(allocation) => zero(allocation.base, allocation.frames.count), }
    match reservation { case ContextReservation::Held(allocation) => contextClearAllocation(allocation), }
}

/// Clear retained private stack frames outside the metadata lock.
export fn contextClearAllocation(allocation: &ContextAllocation) {
    zero(allocation.base, allocation.frames.count);
}

/// Check the reserved context and its retained physical allocation.
fn contextRequired(store: &Store, allocation: &ContextAllocation) {
    assert slots::matches(&store.contextSlots[..], allocation.object, slots::State::Reserved);

    }
}

/// Return an unpublished context's stack and slot under metadata serialization.
export fn contextCancel(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation) {
    contextCancelUsing(store, memory, reservation, frames::release);
}

/// Cancel a context slot and transfer its private stack to the release operation.
export fn contextCancelUsing(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case ContextReservation::Held(allocation) => {
            contextRequired(store, &allocation);
            try! frames::release(&mut memory.backings.pool, allocation.frames);
            try! release(&mut memory.backings.pool, allocation.frames);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
        },
    }
}


        },
    }
}

/// Return storage that has never been exposed to a published domain.
fn discard(pool: &mut frames::Pool, prepared: Prepared) {
    try! frames::release(pool, prepared.kernelFrames);
    try! frames::release(pool, prepared.graph.frames);
    try! frames::release(pool, prepared.allocation);
fn discard(pool: &mut frames::Pool, prepared: Prepared, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) {
    try! release(pool, prepared.kernelFrames);
    try! release(pool, prepared.graph.frames);
    try! release(pool, prepared.allocation);
}

/// Unpublished domain metadata retained during private initialization.
export record Creation: Copy {
    /// Calling domain kept live until publication or cancellation.

    let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch err {
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let initialSlot = slots::reference(&contextSlot);
    set store.contexts[initialSlot.index].owner = slots::reference(&domainSlot);
    set store.contexts[initialSlot.index].userStack = range::Range { start: 0, end: 0 };
    let prepared = try allocate(&mut backings.pool, packages, root) catch err {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;

    }
}

/// Initialize private storage from resident packages while metadata can change.
export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
    match reservation {
        case Reservation::Held(creation) => {
            let mut storage = creation.storage;
            zero(storage.memory as u64, storage.allocation.count);
            zero(storage.kernelStack.start, storage.kernelFrames.count);
            try instances::fill(packages, &storage.graph.graph, storage.graph.base);
            capability::initialize(&mut storage.memory.table, creation.domain);
        },
    }
    match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
}

/// Initialize retained domain storage outside the metadata lock.
export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
    let mut storage = creation.storage;
    zero(storage.memory as u64, storage.allocation.count);
    zero(storage.kernelStack.start, storage.kernelFrames.count);
    try instances::fill(packages, &storage.graph.graph, storage.graph.base);
    capability::initialize(&mut storage.memory.table, creation.domain);
}

/// Check the unpublished capacity retained by one creation transaction.
fn require(store: &Store, table: &capability::Table, creation: &Creation) {
    assert table.owner == creation.owner;

    assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
}

/// Return private domain storage and reserved slots under metadata serialization.
export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
    cancelUsing(store, backings, table, reservation, frames::release);
}

/// Cancel private domain slots and transfer their storage to the release operation.
export fn cancelUsing(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            discard(&mut backings.pool, creation.storage);
            discard(&mut backings.pool, creation.storage, release);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
        },
    }

            // MPIE enables interrupts after the first return into U-mode.
            let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
            set frame.registers[1] = returnAddress();
            set frame.registers[3] = graph.table.ptr as u64;
            set store.contexts[initial.index] = Context {
                owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.reclaimed[object.index] = false;
            set store.reaping[object.index] = false;
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
            };
            set store.ancestors[object.index] = store.ancestors[creation.owner.index];
            set store.ancestors[object.index][creation.owner.index / 32] |= 1 << (creation.owner.index % 32);
            let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
            let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
            return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
                kind: abi::Kind::Domain, object: domain, rights: creation.rights,
            });

kernel/kernel/events.rad +60 -6

    state: State,
}

/// Fixed event metadata. Callers serialize all kernel-side mutations.
export record Store: Copy {
    /// Terminal events retained for publication into receiver rings.
    pending: u32,
    /// Producer metadata indexed by receiver domain.
    queues: [Queue; limits::DOMAINS],
    /// Terminal storage indexed by the terminating domain.
    terminals: [Terminal; limits::DOMAINS],
}

    return ((value as u64 + 1) & 0xffffffff) as u32;
}

/// Initialize metadata before any queue or terminal record exists.
export fn initialize(store: &mut Store) {
    set store.pending = 0;
    for i in 0..limits::DOMAINS {
        set store.queues[i].generation = 0;
        set store.terminals[i].state = State::Free;
    }
}

            assert terminal.subject == subject and terminal.state == State::Reserved;
            set terminal.receiver = receiver;
            set terminal.event = Event { kind, reserved: 0, code, value: abi::id(subject) };
            if active {
                set terminal.state = State::Pending;
                set store.pending += 1;
            }
            else {
                set terminal.state = State::Acknowledged;
            }
        },
    }
}

/// Publish one retained terminal event when its receiver has ring capacity.
export fn deliver(store: &mut Store, subject: abi::Ref, ring: &mut Ring) -> bool throws (abi::Error) {
    if subject.index >= limits::DOMAINS or store.terminals[subject.index].subject <> subject
        or store.terminals[subject.index].state == State::Free {
            throw abi::Error::BadHandle;
        }
    let terminal = store.terminals[subject.index];
    if terminal.state <> State::Pending {
        return false;
    }
    try refresh(store, terminal.receiver, ring);
    let queue = &mut store.queues[terminal.receiver.index];
    if distance(queue.tail, queue.head) == CAPACITY {
        return false;
    }
    set store.terminals[subject.index].state = State::Queued;
    assert store.pending > 0;
    set store.pending -= 1;
    publish(queue, ring, terminal.event, subject);
    return true;
}

/// Publish retained terminal events in domain-index order as ring capacity permits.
export fn flush(store: &mut Store, receiver: abi::Ref, ring: &mut Ring) -> u32 throws (abi::Error) {
    try refresh(store, receiver, ring);
    let mut count: u32 = 0;
    for i in 0..limits::DOMAINS {

            break;
        }
        if store.terminals[i].state <> State::Pending or store.terminals[i].receiver <> receiver {
            continue;
        }
        set store.terminals[i].state = State::Queued;
        let event = store.terminals[i].event;
        let subject = store.terminals[i].subject;
        publish(&mut store.queues[receiver.index], ring, event, subject);
        set count += 1;
        if try deliver(store, subject, ring) {
            set count += 1;
        }
    }
    return count;
}

/// Report whether a retained terminal identity can be released.

/// Close a stopped domain's queue and end all terminal references held by that receiver.
export fn close(store: &mut Store, receiver: abi::Ref) throws (abi::Error) {
    if not live(store, receiver) {
        throw abi::Error::BadHandle;
    }
    set store.queues[receiver.index].generation = 0;
    let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
    set generations[receiver.index] = receiver.generation;
    try! closeSet(store, &generations[..]);
}

/// Close selected receiver generations and acknowledge their retained terminal events.
/// Zero entries leave a receiver open. Validate all selected generations before mutation.
export fn closeSet(store: &mut Store, generations: &[u32]) throws (abi::Error) {
    if generations.len <> limits::DOMAINS {
        throw abi::Error::InvalidArg;
    }
    for i in 0..limits::DOMAINS {
        if generations[i] <> 0 and store.queues[i].generation <> generations[i] {
            throw abi::Error::BadHandle;
        }
    }
    for i in 0..limits::DOMAINS {
        if (store.terminals[i].state == State::Pending or store.terminals[i].state == State::Queued) and store.terminals[i].receiver == receiver {
        if generations[i] <> 0 {
            set store.queues[i].generation = 0;
        }
    }
    for i in 0..limits::DOMAINS {
        let terminal = store.terminals[i];
        if terminal.state <> State::Pending and terminal.state <> State::Queued {
            continue;
        }
        let receiver = terminal.receiver;
        if generations[receiver.index] == receiver.generation {
            if store.terminals[i].state == State::Pending {
                assert store.pending > 0;
                set store.pending -= 1;
            }
            set store.terminals[i].state = State::Acknowledged;
        }
    }
}

kernel/kernel/frames.rad +39 -0

    addresses: [u64; limits::FRAMES],
    /// True when the frame is available for reservation.
    free: [bool; limits::FRAMES],
    /// Number of valid frame-map entries.
    count: u32,
    /// Retired runs whose frames remain unavailable until reclamation.
    retired: [Run; limits::FRAMES],
    /// Index of the next retired run to reclaim.
    retiredHead: u32,
    /// Number of occupied retirement entries.
    retiredCount: u32,
}

/// Insert one range into an address-sorted prefix of fixed storage.
fn insert(ranges: &mut [range::Range], count: u32, value: range::Range) {
    let mut at = count;


/// Build a fresh frame map from complete RAM pages with no reserved byte.
/// The pool has zero valid entries if initialization fails.
export unsafe fn initialize(pool: &mut Pool, machine: &platform::Platform) throws (abi::Error) {
    set pool.count = 0;
    set pool.retiredHead = 0;
    set pool.retiredCount = 0;
    if machine.ramCount > platform::RAM_BANKS or machine.reservedCount > platform::RESERVATIONS {
        throw abi::Error::InvalidArg;
    }
    let mut reserved: [range::Range; platform::RESERVATIONS] = undefined;
    for i in 0..machine.reservedCount {

    for i in run.first..run.first + run.count {
        set pool.free[i] = true;
    }
}

/// Transfer exclusive ownership of a committed run to bounded reclamation.
/// The caller must not release or retire this run again.
export fn retire(pool: &mut Pool, run: Run) throws (abi::Error) {
    let memory = try extent(pool, run);
    assert pool.retiredCount < pool.count;
    let tail = (pool.retiredHead + pool.retiredCount) % limits::FRAMES;
    set pool.retired[tail] = run;
    set pool.retiredCount += 1;
}

/// Return at most limit retired frames while metadata is serialized.
export fn reclaim(pool: &mut Pool, limit: u32) -> u32 {
    let mut returned: u32 = 0;
    while pool.retiredCount > 0 and returned < limit {
        let mut count = pool.retired[pool.retiredHead].count;
        if count > limit - returned {
            set count = limit - returned;
        }
        let batch = Run { first: pool.retired[pool.retiredHead].first, count };
        try! release(pool, batch);
        set pool.retired[pool.retiredHead].first += count;
        set pool.retired[pool.retiredHead].count -= count;
        set returned += count;
        if pool.retired[pool.retiredHead].count == 0 {
            set pool.retiredHead = (pool.retiredHead + 1) % limits::FRAMES;
            set pool.retiredCount -= 1;
        }
    }
    return returned;
}

/// Cancel unpublished frame ownership and restore its capacity.
export fn cancel(pool: &mut Pool, reservation: Reservation) throws (abi::Error) {
    match reservation { case Reservation::Held(run) => try release(pool, run), }
}

kernel/kernel/lifecycle.rad added +558 -0

1	+	//! Terminal notification delivery and execution eligibility.
2	+	use super::limits;
3	+	use super::slots;
4	+	use super::events;
5	+	use super::domains;
6	+	use super::budgets;
7	+	use super::abi;
8	+	use super::remote;
9	+	use super::pages;
10	+	use super::registry;
11	+	use super::loader;
12	+	use super::capability;
13	+	use super::frames;
14	+	use super::backing;
15	+	use super::interrupts;
16	+	use super::plic;
17	+
18	+	/// Words needed to track request ownership for every context slot.
19	+	constant CONTEXT_WORDS: u32 = (limits::CONTEXTS + 31) / 32;
20	+
21	+	/// Private input and optional native output retained by a loader call.
22	+	export record Loading: Copy {
23	+	/// Workspace and reserved caller handle.
24	+	input: loader::Input,
25	+	/// Output frames and package slot, when compilation has been admitted.
26	+	output: ?loader::Output,
27	+	}
28	+
29	+	/// Successful loader work ready for publication under the metadata lock.
30	+	export union LoadResult: Copy {
31	+	/// Matching immutable package already present in the registry.
32	+	Existing(abi::Ref),
33	+	/// Generated code and metadata in the retained output allocation.
34	+	Compiled(loader::Compiled),
35	+	}
36	+
37	+	/// Private storage retained by an unfinished syscall.
38	+	export union Allocation: Copy {
39	+	/// Page frames and unpublished capability metadata.
40	+	Page(pages::Allocation),
41	+	/// Additional context slot and private kernel stack.
42	+	Context(domains::ContextAllocation),
43	+	/// Reserved domain generation with its retained creation payload.
44	+	Domain(abi::Ref),
45	+	/// Private compiler workspace and unpublished output.
46	+	Load(Loading),
47	+	}
48	+
49	+	/// One unfinished allocation retained by a caller generation.
50	+	export record HeldAllocation: Copy {
51	+	/// Context whose continuation owns this allocation.
52	+	context: abi::Ref,
53	+	/// Calling domain, which can differ from the allocation target.
54	+	owner: abi::Ref,
55	+	/// Frames and metadata slots excluded from allocation.
56	+	allocation: Allocation,
57	+	}
58	+
59	+	/// Allocations retained while their contexts initialize private frames.
60	+	export record Calls: Copy {
61	+	/// One validity bit per occupied context slot.
62	+	active: [u32; CONTEXT_WORDS],
63	+	/// Reservation payloads indexed by context slot.
64	+	records: [HeldAllocation; limits::CONTEXTS],
65	+	/// Creation payloads valid while their domain slots are reserved.
66	+	creations: [domains::Creation; limits::DOMAINS],
67	+	}
68	+
69	+	/// A continuation's exclusive claim to its retained reservation.
70	+	export union Lease: Once {
71	+	/// Context generation that must take back the reservation before completion.
72	+	Held(abi::Ref),
73	+	}
74	+
75	+	/// Reservations serialized with context and physical-memory metadata.
76	+	export unsafe static CALLS: Calls = undefined;
77	+
78	+	/// Clear allocation ownership before contexts start.
79	+	export fn initialize(calls: &mut Calls) {
80	+	for i in 0..CONTEXT_WORDS {
81	+	set calls.active[i] = 0;
82	+	}
83	+	}
84	+
85	+	/// Retain private storage and pin the caller's kernel stack under the lock.
86	+	fn hold(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, allocation: Allocation) -> Lease {
87	+	assert context.index < limits::CONTEXTS and context.generation <> 0;
88	+	assert slots::matches(&store.contextSlots[..], context, slots::State::Live);
89	+	assert not store.contexts[context.index].reservation;
90	+	let bit = 1 << (context.index % 32);
91	+	assert (calls.active[context.index / 32] & bit) == 0;
92	+	set calls.records[context.index] = HeldAllocation { context, owner: store.contexts[context.index].owner, allocation };
93	+	set calls.active[context.index / 32] \|= bit;
94	+	set store.contexts[context.index].reservation = true;
95	+	return Lease::Held(context);
96	+	}
97	+
98	+	/// Transfer a page reservation into context-owned storage under the metadata lock.
99	+	export fn holdPage(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: pages::Reservation) -> Lease {
100	+	match reservation {
101	+	case pages::Reservation::Held(allocation) => {
102	+	assert store.contexts[context.index].owner == allocation.owner;
103	+	return hold(calls, store, context, Allocation::Page(allocation));
104	+	},
105	+	}
106	+	}
107	+
108	+	/// Transfer an unpublished context into its caller's retained storage.
109	+	export fn holdContext(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: domains::ContextReservation) -> Lease {
110	+	match reservation {
111	+	case domains::ContextReservation::Held(allocation) => return hold(calls, store, context, Allocation::Context(allocation)),
112	+	}
113	+	}
114	+
115	+	/// Retain unpublished domain storage under its reserved domain generation.
116	+	export fn holdDomain(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: domains::Reservation) -> Lease {
117	+	match reservation {
118	+	case domains::Reservation::Held(creation) => {
119	+	assert slots::matches(&store.slots[..], creation.domain, slots::State::Reserved);
120	+	assert store.contexts[context.index].owner == creation.owner;
121	+	set calls.creations[creation.domain.index] = creation;
122	+	return hold(calls, store, context, Allocation::Domain(creation.domain));
123	+	},
124	+	}
125	+	}
126	+
127	+	/// Read the private payload for an exact reserved domain generation.
128	+	fn creation(calls: &Calls, object: abi::Ref) -> domains::Creation {
129	+	assert object.index < limits::DOMAINS;
130	+	assert calls.creations[object.index].domain == object;
131	+	return calls.creations[object.index];
132	+	}
133	+
134	+	/// Initialize domain storage while the caller retains execution ownership.
135	+	export unsafe fn prepareDomain(calls: &Calls, packages: &registry::Store, lease: &Lease) throws (abi::Error) {
136	+	match lease {
137	+	case Lease::Held(context) => match held(calls, *context) {
138	+	case Allocation::Domain(object) => {
139	+	let retained = creation(calls, object);
140	+	try domains::prepareCreation(packages, &retained);
141	+	},
142	+	else => panic "domain reservation required",
143	+	},
144	+	}
145	+	}
146	+
147	+	/// Retain private loader input and pin its caller's kernel stack.
148	+	export fn holdLoad(calls: &mut Calls, store: &mut domains::Store, context: abi::Ref, reservation: loader::Reservation) -> Lease {
149	+	match reservation {
150	+	case loader::Reservation::Held(input) => {
151	+	assert store.contexts[context.index].owner == input.owner;
152	+	return hold(calls, store, context, Allocation::Load(Loading { input, output: nil }));
153	+	},
154	+	}
155	+	}
156	+
157	+	/// Read the private loader allocations through their caller's lease.
158	+	export fn load(calls: &Calls, lease: &Lease) -> Loading {
159	+	match lease {
160	+	case Lease::Held(context) => match held(calls, *context) {
161	+	case Allocation::Load(loading) => return loading,
162	+	else => panic "loader reservation required",
163	+	},
164	+	}
165	+	}
166	+
167	+	/// Attach reserved native output to its caller before compilation can start.
168	+	export fn reserveOutput(calls: &mut Calls, lease: &Lease, memory: &mut pages::Store, packages: &mut registry::Store) throws (abi::Error) {
169	+	let mut loading = load(calls, lease);
170	+	assert loading.output == nil;
171	+	let pending = try loader::reserveOutput(&mut memory.backings.pool, packages);
172	+	match pending {
173	+	case loader::OutputReservation::Held(output) => {
174	+	set loading.output = output;
175	+	},
176	+	}
177	+	match lease {
178	+	case Lease::Held(context) => {
179	+	set calls.records[context.index].allocation = Allocation::Load(loading);
180	+	},
181	+	}
182	+	}
183	+
184	+	/// Return all unpublished loader capacity under the metadata lock.
185	+	fn discardLoad(memory: &mut pages::Store, state: &mut loader::State, packages: &mut registry::Store,
186	+	table: &mut capability::Table, loading: Loading, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
187	+	{
188	+	if let output = loading.output {
189	+	loader::cancelOutputUsing(&mut memory.backings.pool, packages, loader::OutputReservation::Held(output), release);
190	+	}
191	+	loader::cancelUsing(state, memory, table, loader::Reservation::Held(loading.input), release);
192	+	}
193	+
194	+	/// Cancel a live continuation's load and release its stack pin under the lock.
195	+	export unsafe fn cancelLoad(calls: &mut Calls, store: &mut domains::Store, memory: &mut pages::Store,
196	+	state: &mut loader::State, packages: &mut registry::Store, lease: Lease)
197	+	{
198	+	match take(calls, store, lease) {
199	+	case Allocation::Load(loading) => {
200	+	let mut owner = try! domains::get(store, loading.input.owner);
201	+	discardLoad(memory, state, packages, &mut owner.memory.table, loading, frames::release);
202	+	},
203	+	else => panic "loader reservation required",
204	+	}
205	+	}
206	+
207	+	/// Publish successful loader work and release caller-owned temporary storage.
208	+	export unsafe fn finishLoad(calls: &mut Calls, store: &mut domains::Store, memory: &mut pages::Store,
209	+	state: &mut loader::State, packages: &mut registry::Store, lease: Lease, result: LoadResult) -> abi::Handle throws (abi::Error)
210	+	{
211	+	match take(calls, store, lease) {
212	+	case Allocation::Load(loading) => {
213	+	let mut owner = try! domains::get(store, loading.input.owner);
214	+	if owner.state == domains::Lifecycle::Dead {
215	+	discardLoad(memory, state, packages, &mut owner.memory.table, loading, frames::release);
216	+	throw abi::Error::BadHandle;
217	+	}
218	+	let mut object: abi::Ref = undefined;
219	+	match result {
220	+	case LoadResult::Existing(existing) => {
221	+	assert loading.output == nil;
222	+	set object = existing;
223	+	},
224	+	case LoadResult::Compiled(compiled) => {
225	+	let output = loading.output else panic "missing loader output";
226	+	set object = loader::publishOutput(state, &mut memory.backings.pool, packages,
227	+	loader::OutputReservation::Held(output), compiled);
228	+	},
229	+	}
230	+	return try loader::finish(state, memory, &mut owner.memory.table, loader::Reservation::Held(loading.input), object);
231	+	},
232	+	else => panic "loader reservation required",
233	+	}
234	+	}
235	+
236	+	/// Read retained storage through its exact caller generation.
237	+	fn held(calls: &Calls, context: abi::Ref) -> Allocation {
238	+	assert context.index < limits::CONTEXTS;
239	+	assert (calls.active[context.index / 32] & (1 << (context.index % 32))) <> 0;
240	+	assert calls.records[context.index].context == context;
241	+	return calls.records[context.index].allocation;
242	+	}
243	+
244	+	/// Clear private page frames while the executing context retains hart ownership.
245	+	export fn clearPage(calls: &Calls, lease: &Lease) {
246	+	match lease {
247	+	case Lease::Held(context) => match held(calls, *context) {
248	+	case Allocation::Page(allocation) => pages::clearAllocation(&allocation),
249	+	else => panic "page reservation required",
250	+	},
251	+	}
252	+	}
253	+
254	+	/// Clear a private kernel stack while the caller retains hart ownership.
255	+	export fn clearContext(calls: &Calls, lease: &Lease) {
256	+	match lease {
257	+	case Lease::Held(context) => match held(calls, *context) {
258	+	case Allocation::Context(allocation) => domains::contextClearAllocation(&allocation),
259	+	else => panic "context reservation required",
260	+	},
261	+	}
262	+	}
263	+
264	+	/// Remove retained ownership before serialized publication or cancellation.
265	+	fn take(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> Allocation {
266	+	match lease {
267	+	case Lease::Held(context) => {
268	+	let allocation = held(calls, context);
269	+	set calls.active[context.index / 32] &= ~(1 << (context.index % 32));
270	+	assert slots::matches(&store.contextSlots[..], context, slots::State::Live);
271	+	set store.contexts[context.index].reservation = false;
272	+	return allocation;
273	+	},
274	+	}
275	+	}
276	+
277	+	/// Return a page reservation for publication or cancellation under the lock.
278	+	export fn takePage(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> pages::Reservation {
279	+	match take(calls, store, lease) {
280	+	case Allocation::Page(allocation) => return pages::Reservation::Held(allocation),
281	+	else => panic "page reservation required",
282	+	}
283	+	}
284	+
285	+	/// Return a context reservation for publication or cancellation under the lock.
286	+	export fn takeContext(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> domains::ContextReservation {
287	+	match take(calls, store, lease) {
288	+	case Allocation::Context(allocation) => return domains::ContextReservation::Held(allocation),
289	+	else => panic "context reservation required",
290	+	}
291	+	}
292	+
293	+	/// Return domain storage for publication or cancellation under the lock.
294	+	export fn takeDomain(calls: &mut Calls, store: &mut domains::Store, lease: Lease) -> domains::Reservation {
295	+	match take(calls, store, lease) {
296	+	case Allocation::Domain(object) => return domains::Reservation::Held(creation(calls, object)),
297	+	else => panic "domain reservation required",
298	+	}
299	+	}
300	+
301	+	/// Retire one dead caller's private storage after execution ownership ends.
302	+	export unsafe fn cancelAllocations(calls: &mut Calls, store: &mut domains::Store, memory: &mut pages::Store,
303	+	state: &mut loader::State, packages: &mut registry::Store) -> bool
304	+	{
305	+	for word in 0..CONTEXT_WORDS {
306	+	if calls.active[word] == 0 {
307	+	continue;
308	+	}
309	+	for offset in 0..32 as u32 {
310	+	let bit: u32 = 1 << offset;
311	+	if (calls.active[word] & bit) == 0 {
312	+	continue;
313	+	}
314	+	let index = word * 32 + offset;
315	+	let retained = calls.records[index];
316	+	let context = try domains::context(store, retained.owner, retained.context) catch {
317	+	continue;
318	+	};
319	+	if context.hart <> nil or context.state <> domains::ContextState::Stopped {
320	+	continue;
321	+	}
322	+	let mut owner = try! domains::get(store, retained.owner);
323	+	if owner.state <> domains::Lifecycle::Dead {
324	+	continue;
325	+	}
326	+	set calls.active[word] &= ~bit;
327	+	match retained.allocation {
328	+	case Allocation::Load(loading) => discardLoad(memory, state, packages, &mut owner.memory.table, loading, frames::retire),
329	+	case Allocation::Page(allocation) => pages::cancelUsing(memory, &mut owner.memory.table, pages::Reservation::Held(allocation), frames::retire),
330	+	case Allocation::Context(allocation) => domains::contextCancelUsing(store, memory, domains::ContextReservation::Held(allocation), frames::retire),
331	+	case Allocation::Domain(object) => domains::cancelUsing(store, &mut memory.backings,
332	+	&mut owner.memory.table, domains::Reservation::Held(creation(calls, object)), frames::retire),
333	+	}
334	+	set store.contexts[index].reservation = false;
335	+	return true;
336	+	}
337	+	}
338	+	return false;
339	+	}
340	+
341	+	/// Release stopped callers' remote work after hart ownership ends.
342	+	/// Taken requests keep the context pinned until the target acknowledges them.
343	+	export fn cancelRequests(store: &mut domains::Store, requests: &mut remote::Store) {
344	+	let mut touched: [u32; CONTEXT_WORDS] = [0; CONTEXT_WORDS];
345	+	let mut retained: [u32; CONTEXT_WORDS] = [0; CONTEXT_WORDS];
346	+	let mut cancelled = false;
347	+	for i in 0..limits::REMOTE_REQUESTS {
348	+	if requests.slots[i].state <> slots::State::Live {
349	+	continue;
350	+	}
351	+	let request = requests.requests[i];
352	+	let mut caller: ?abi::Ref = nil;
353	+	match request.action {
354	+	case remote::Action::Budget(call) => {
355	+	set caller = call.context;
356	+	},
357	+	case remote::Action::Ready(context) => {
358	+	set caller = context;
359	+	},
360	+	else => continue,
361	+	}
362	+	let object = caller else {
363	+	continue;
364	+	};
365	+	if not slots::matches(&store.contextSlots[..], object, slots::State::Live) {
366	+	continue;
367	+	}
368	+	if store.contexts[object.index].hart <> nil or store.contexts[object.index].state <> domains::ContextState::Stopped {
369	+	continue;
370	+	}
371	+	if not slots::matches(&store.slots[..], store.contexts[object.index].owner, slots::State::Live)
372	+	or store.records[store.contexts[object.index].owner.index].state <> domains::Lifecycle::Dead {
373	+	continue;
374	+	}
375	+	let word = object.index / 32;
376	+	let bit = 1 << (object.index % 32);
377	+	set touched[word] \|= bit;
378	+	if request.state == remote::State::Delivered {
379	+	set retained[word] \|= bit;
380	+	set store.contexts[object.index].pending = abi::Ref { index: i, generation: requests.slots[i].generation };
381	+	continue;
382	+	}
383	+	try! remote::cancel(requests, request.source, abi::Ref { index: i, generation: requests.slots[i].generation });
384	+	set cancelled = true;
385	+	}
386	+	if not cancelled {
387	+	return;
388	+	}
389	+	for i in 0..limits::CONTEXTS {
390	+	let bit = 1 << (i % 32);
391	+	if (touched[i / 32] & bit) <> 0 and (retained[i / 32] & bit) == 0 {
392	+	set store.contexts[i].pending = nil;
393	+	}
394	+	}
395	+	}
396	+
397	+	/// Advance one eligible domain from a rotating slot cursor.
398	+	export unsafe fn reapNext(store: &mut domains::Store, memory: &mut pages::Store, windows: &mut budgets::Store,
399	+	irqs: &mut interrupts::Store) -> bool
400	+	{
401	+	if store.dead == 0 {
402	+	return false;
403	+	}
404	+	let mut checked = false;
405	+	let mut blocked: [bool; limits::DOMAINS] = [false; limits::DOMAINS];
406	+	for offset in 0..limits::DOMAINS {
407	+	let index = (store.reapCursor + offset) % limits::DOMAINS;
408	+	if store.slots[index].state <> slots::State::Live or store.records[index].state <> domains::Lifecycle::Dead {
409	+	continue;
410	+	}
411	+	let owner = abi::Ref { index, generation: store.slots[index].generation };
412	+	if store.reclaimed[index] {
413	+	if not try! events::acknowledged(&store.events, owner) {
414	+	continue;
415	+	}
416	+	} else if not store.reaping[index] {
417	+	if not checked {
418	+	for i in 0..limits::CONTEXTS {
419	+	let state = store.contextSlots[i].state;
420	+	if state <> slots::State::Live and state <> slots::State::Reserved {
421	+	continue;
422	+	}
423	+	if state == slots::State::Reserved or store.contexts[i].state <> domains::ContextState::Stopped
424	+	or store.contexts[i].hart <> nil or store.contexts[i].pending <> nil or store.contexts[i].reservation {
425	+	set blocked[store.contexts[i].owner.index] = true;
426	+	}
427	+	}
428	+	set checked = true;
429	+	}
430	+	if blocked[index] {
431	+	continue;
432	+	}
433	+	}
434	+	if try! reap(store, memory, windows, irqs, owner) {
435	+	set store.reapCursor = (index + 1) % limits::DOMAINS;
436	+	return true;
437	+	}
438	+	}
439	+	return false;
440	+	}
441	+
442	+	/// Advance one dead domain's resource teardown under the metadata lock.
443	+	/// Returns false while execution ownership or terminal acknowledgement is pending.
444	+	export unsafe fn reap(store: &mut domains::Store, memory: &mut pages::Store, windows: &mut budgets::Store,
445	+	irqs: &mut interrupts::Store, owner: abi::Ref) -> bool throws (abi::Error)
446	+	{
447	+	if not slots::matches(&store.slots[..], owner, slots::State::Live) {
448	+	throw abi::Error::BadHandle;
449	+	}
450	+	if store.reclaimed[owner.index] {
451	+	if not try events::acknowledged(&store.events, owner) {
452	+	return false;
453	+	}
454	+	try! domains::releaseIdentity(store, owner);
455	+	return true;
456	+	}
457	+	if not store.reaping[owner.index] {
458	+	if not try domains::quiescent(store, owner) {
459	+	return false;
460	+	}
461	+	set store.reaping[owner.index] = true;
462	+	}
463	+	let mut domain = try! domains::get(store, owner);
464	+	for i in 0..limits::HANDLES {
465	+	let slot = domain.memory.table.slots[i];
466	+	if slot.state == slots::State::Reserved {
467	+	return false;
468	+	}
469	+	if slot.state <> slots::State::Live {
470	+	continue;
471	+	}
472	+	let entry = domain.memory.table.entries[i];
473	+	let handle = try! abi::handle(entry.kind, abi::Ref { index: i, generation: slot.generation });
474	+	match entry.kind {
475	+	case abi::Kind::Page => try! pages::drop(memory, &mut domain.memory.table, handle),
476	+	case abi::Kind::Budget => try! budgets::drop(windows, &mut domain.memory.table, handle),
477	+	case abi::Kind::Interrupt => {
478	+	let number = try! interrupts::query(irqs, &domain.memory.table, handle);
479	+	plic::mask(number);
480	+	try! interrupts::drop(irqs, &mut domain.memory.table, handle);
481	+	},
482	+	else => {
483	+	let removed = try! capability::invalidate(&mut domain.memory.table, handle);
484	+	},
485	+	}
486	+	return true;
487	+	}
488	+	if memory.backings.domains[owner.index] <> 0 {
489	+	let complete = try! backing::endDomainStep(&mut memory.backings, owner, 64);
490	+	return true;
491	+	}
492	+	if try! domains::retireContext(store, &mut memory.backings.pool, owner) {
493	+	return true;
494	+	}
495	+	try! domains::retireStorage(store, &mut memory.backings, owner);
496	+	return true;
497	+	}
498	+
499	+	/// Publish retained terminal events and report receiver harts under the metadata lock.
500	+	export unsafe fn service(store: &mut domains::Store, windows: &budgets::Store, now: u64) -> u32 {
501	+	if store.events.pending == 0 {
502	+	return 0;
503	+	}
504	+	let mut notified: [bool; limits::DOMAINS] = [false; limits::DOMAINS];
505	+	let mut changed = false;
506	+	for i in 0..limits::DOMAINS {
507	+	let terminal = store.events.terminals[i];
508	+	if terminal.state <> events::State::Pending {
509	+	continue;
510	+	}
511	+	let mut receiver = try domains::get(store, terminal.receiver) catch {
512	+	continue;
513	+	};
514	+	if receiver.state == domains::Lifecycle::Dead {
515	+	continue;
516	+	}
517	+	let delivered = try events::deliver(&mut store.events, terminal.subject, &mut receiver.memory.ring) catch {
518	+	false
519	+	};
520	+	if delivered {
521	+	set notified[terminal.receiver.index] = true;
522	+	set changed = true;
523	+	}
524	+	}
525	+	if not changed {
526	+	return 0;
527	+	}
528	+	for i in 0..limits::CONTEXTS {
529	+	if store.contextSlots[i].state <> slots::State::Live {
530	+	continue;
531	+	}
532	+	if notified[store.contexts[i].owner.index] and store.contexts[i].state == domains::ContextState::Waiting {
533	+	set store.contexts[i].state = domains::ContextState::Ready;
534	+	}
535	+	}
536	+	let mut harts: u32 = 0;
537	+	for i in 0..limits::BUDGETS {
538	+	if windows.slots[i].state <> slots::State::Live {
539	+	continue;
540	+	}
541	+	let window = windows.windows[i];
542	+	if not slots::matches(&store.slots[..], window.owner, slots::State::Live)
543	+	or not notified[window.owner.index] or window.end <= now {
544	+	continue;
545	+	}
546	+	let context = window.context else {
547	+	continue;
548	+	};
549	+	if not slots::matches(&store.contextSlots[..], context, slots::State::Live) {
550	+	continue;
551	+	}
552	+	if store.contexts[context.index].owner == window.owner
553	+	and store.contexts[context.index].state <> domains::ContextState::Stopped {
554	+	set harts \|= 1 << window.hart;
555	+	}
556	+	}
557	+	return harts;
558	+	}

kernel/kernel/loader.rad +54 -20

    return Lease::Held(frames::commit(reservation));
}

/// Return temporary frames and release the compiler workspace under metadata serialization.
export fn release(state: &mut State, pool: &mut frames::Pool, lease: Lease) {
    releaseUsing(state, pool, lease, frames::release);
}

/// Transfer temporary frames to their release operation and clear workspace ownership.
fn releaseUsing(state: &mut State, pool: &mut frames::Pool, lease: Lease,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match lease {
        case Lease::Held(run) => {
            try! frames::release(pool, run);
            try! release(pool, run);
        },
    }
    set state.owner = abi::Ref { index: 0, generation: 0 };
    set state.busy = false;
}

}

/// Snapshot and decode trusted input into the held private workspace.
/// The decoded package stays valid until this reservation is completed or cancelled.
export unsafe fn decode(reservation: &Reservation) -> binary::Package throws (abi::Error) {
    match reservation {
        case Reservation::Held(input) => {
            let mut work = workspace(input.work);
            try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
            let mut decoder = alloc::new(&mut work.decoded[..]);
            return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
                if err == binary::Error::Storage {
                    throw abi::Error::Exhausted;
                }
                throw abi::Error::VerifyFailed;
            };
        },
    }
    match reservation { case Reservation::Held(input) => return try decodeInput(input), }
}

/// Snapshot retained input and decode it in its private workspace.
export unsafe fn decodeInput(input: &Input) -> binary::Package throws (abi::Error) {
    let mut work = workspace(input.work);
    try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
    let mut decoder = alloc::new(&mut work.decoded[..]);
    return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
        if err == binary::Error::Storage {
            throw abi::Error::Exhausted;
        }
        throw abi::Error::VerifyFailed;
    };
}

/// Reserved native output and its immutable physical extents.
export record Output: Copy {
    /// Reserved package slot selected for generated state references.

    -> Compiled throws (abi::Error)
{
    match reservation {
        case Reservation::Held(source) => match output {
            case OutputReservation::Held(target) => {
                let work = workspace(source.work);
                return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
                return try compileInput(packages, source, input, target);
            },
        },
    }
}

/// Generate native code into retained output while the caller owns its workspace.
export unsafe fn compileInput(packages: &registry::Store, source: &Input, input: &binary::Package, target: &Output)
    -> Compiled throws (abi::Error)
{
    let work = workspace(source.work);
    return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
}

/// Return unpublished output frames and its package slot under serialization.
export fn cancelOutput(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation) {
    cancelOutputUsing(pool, packages, reservation, frames::release);
}

/// Cancel a package slot and transfer private output to the release operation.
export fn cancelOutputUsing(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case OutputReservation::Held(output) => {
            assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
            try! frames::release(pool, output.metadata);
            try! frames::release(pool, output.code);
            try! release(pool, output.metadata);
            try! release(pool, output.code);
            registry::cancel(packages, slots::Reservation::Held(output.object));
        },
    }
}


}

/// Compare the snapshot with immutable resident content while the workspace is held.
export unsafe fn identify(packages: &registry::Store, reservation: &Reservation, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
    match reservation {
        case Reservation::Held(source) => return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]),
        case Reservation::Held(source) => return try identifyInput(packages, source, input),
    }
}

/// Compare retained source bytes with immutable resident package content.
export unsafe fn identifyInput(packages: &registry::Store, source: &Input, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
    return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]);
}

/// Reserve an Image handle and exclusive workspace under metadata serialization.
export unsafe fn reserve(state: &mut State, store: &mut pages::Store,
    table: &mut capability::Table, request: Request) -> Reservation throws (abi::Error)
{
    let address = try source(store, table, request);

    assert slots::matches(&table.slots[..], input.handle, slots::State::Reserved);
}

/// Return the workspace and reserved handle under metadata serialization.
export fn cancel(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation) {
    cancelUsing(state, store, table, reservation, frames::release);
}

/// Cancel a reserved Image handle and transfer its workspace to the release operation.
export fn cancelUsing(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(input) => {
            require(state, table, &input);
            release(state, &mut store.backings.pool, Lease::Held(input.frames));
            releaseUsing(state, &mut store.backings.pool, Lease::Held(input.frames), release);
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(input.handle));
        },
    }
}


kernel/kernel/pages.rad +14 -2

    }
}

/// Clear private frames while shared metadata can be used by other harts.
export fn clear(reservation: &Reservation) {
    match reservation { case Reservation::Held(allocation) => zero(allocation.base, allocation.run.count), }
    match reservation { case Reservation::Held(allocation) => clearAllocation(allocation), }
}

/// Clear frames held by an unpublished allocation's exclusive owner.
export fn clearAllocation(allocation: &Allocation) {
    zero(allocation.base, allocation.run.count);
}

/// Check reserved generations before publishing or cancelling an allocation.
fn require(store: &Store, table: &capability::Table, allocation: &Allocation) {
    assert table.owner == allocation.owner;

    }
}

/// Return private allocation capacity under metadata serialization.
export fn cancel(store: &mut Store, table: &mut capability::Table, reservation: Reservation) {
    cancelUsing(store, table, reservation, frames::release);
}

/// Cancel unpublished slots and transfer their frames to the selected release operation.
export fn cancelUsing(store: &mut Store, table: &mut capability::Table, reservation: Reservation,
    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(allocation) => {
            require(store, table, &allocation);
            try! frames::release(&mut store.backings.pool, allocation.run);
            try! release(&mut store.backings.pool, allocation.run);
            try! slots::cancel(&mut store.backings.slots[..], slots::Reservation::Held(allocation.backing));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(allocation.page));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
        },
    }

kernel/kernel/sys.rad +22 -0

/// Copy authority to a domain with optional rights narrowing.
export fn capabilityGrant(handle: abi::Handle, target: abi::Handle, rights: u64) -> abi::Handle throws (abi::Error) {
    return abi::Handle(try result(ecall(abi::Operation::CapabilityGrant as u32, *handle as i64, *target as i64, rights as i64, 0)));
}

/// Set a child's lifecycle receiver with Destroy and Wake authority.
export fn domainReparent(domain: abi::Handle, parent: abi::Handle) throws (abi::Error) {
    try result(ecall(abi::Operation::DomainReparent as u32, *domain as i64, *parent as i64, 0, 0));
}

/// Load a trusted binary package from a readable Page range.
export fn imageLoad(authority: abi::Handle, source: abi::Handle, offset: u64, length: u64) -> abi::Handle throws (abi::Error) {
    return abi::Handle(try result(ecall(abi::Operation::ImageLoad as u32, *authority as i64, *source as i64, offset as i64, length as i64)));
}

/// Destroy the named domain or the calling domain through the zero sentinel.
export fn domainDestroy(handle: abi::Handle, flags: u64) throws (abi::Error) {
    try result(ecall(abi::Operation::DomainDestroy as u32, *handle as i64, flags as i64, 0, 0));
}

/// End the calling domain and report its normal exit status to its parent.
export fn exit(status: u32) -> ! {
    ecall(abi::Operation::Exit as u32, status as i64, 0, 0, 0);
    panic "exit returned";
}

/// End the calling domain and report an explicit-abort fault to its parent.
export fn abort() -> ! {
    ecall(abi::Operation::Abort as u32, 0, 0, 0, 0);
    panic "abort returned";
}

/// Read resident package metadata through a readable Image capability.
export fn queryImage(handle: abi::Handle) -> abi::ImageInfo throws (abi::Error) {
    unsafe {
        let mut info: abi::ImageInfo = undefined;
        try result(ecall(abi::Operation::QueryImage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::ImageInfo) as i64, 0));

kernel/kernel/tests.rad +1 -0

export mod interrupts;
export mod registry;
export mod loader;
export mod instances;
export mod domains;
export mod lifecycle;
export mod budgets;
export mod dispatch;
export mod calls;
export mod timers;
export mod remote;

kernel/kernel/tests/backing.rad +60 -0

    let reservation = try! frames::reserve(&mut STORE.pool, 4);
    let run = frames::commit(reservation);
    return backing::publish(&mut STORE, slot, run, abi::Ref { index: 0, generation: 1 });
}

/// Exposure teardown advances in bounded steps and retains shared allocations.
@test unsafe fn boundedEnd() throws (testing::TestError) {
    initialize();
    let owner = abi::Ref { index: 0, generation: 1 };
    let receiver = abi::Ref { index: 1, generation: 1 };
    let mut objects: [abi::Ref; 3] = undefined;
    for i in 0..3 {
        let slot = try! slots::reserve(&mut STORE.slots[..]);
        let run = frames::commit(try! frames::reserve(&mut STORE.pool, 1));
        set objects[i] = backing::publish(&mut STORE, slot, run, owner);
    }
    assert STORE.used == 3;
    try! backing::expose(&mut STORE, objects[1], receiver);
    try! backing::releasePage(&mut STORE, objects[0]);
    try! backing::releasePage(&mut STORE, objects[1]);
    let mut invalid = false;
    try backing::endDomainStep(&mut STORE, owner, 0) catch error {
        assert error == abi::Error::InvalidArg; set invalid = true;
    };
    assert invalid and backing::domainLive(&STORE, owner);
    assert not try! backing::endDomainStep(&mut STORE, owner, 1);
    assert not backing::domainLive(&STORE, owner);
    assert STORE.slots[objects[0].index].state == slots::State::Free;
    assert STORE.pool.retiredCount == 1 and not STORE.pool.free[0];
    assert STORE.records[objects[1].index].exposed == 2;
    let mut busy = false;
    try backing::registerDomain(&mut STORE, abi::Ref { index: 0, generation: 2 }) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy;
    let mut denied = false;
    try backing::expose(&mut STORE, objects[2], owner) catch error {
        assert error == abi::Error::BadHandle; set denied = true;
    };
    assert denied;
    assert not try! backing::endDomainStep(&mut STORE, owner, 1);
    assert STORE.records[objects[1].index].exposed == 1;
    assert STORE.records[objects[2].index].exposed == 1;
    while not try! backing::endDomainStep(&mut STORE, owner, 64) {
    }
    assert STORE.records[objects[2].index].exposed == 0 and STORE.records[objects[2].index].pages == 1;
    assert STORE.pool.retiredCount == 1;
    assert frames::reclaim(&mut STORE.pool, 1) == 1 and STORE.pool.free[0];
    assert not STORE.pool.free[1] and not STORE.pool.free[2];
    let next = abi::Ref { index: 0, generation: 2 };
    try! backing::registerDomain(&mut STORE, next);
    assert backing::domainLive(&STORE, next);
    let mut stale = false;
    try backing::endDomainStep(&mut STORE, owner, 64) catch error {
        assert error == abi::Error::BadHandle; set stale = true;
    };
    assert stale and backing::domainLive(&STORE, next);
    try! backing::releasePage(&mut STORE, objects[2]);
    assert STORE.pool.free[2];
    while not try! backing::endDomainStep(&mut STORE, receiver, 64) {
    }
    assert STORE.pool.retiredCount == 1 and not STORE.pool.free[1];
    assert frames::reclaim(&mut STORE.pool, 1) == 1 and STORE.pool.free[1];
}

/// Dropping all page handles preserves capacity while an exposed domain lives.
@test unsafe fn exposures() throws (testing::TestError) {
    initialize();
    let object = allocate();
    let receiver = abi::Ref { index: 1, generation: 1 };

kernel/kernel/tests/budgets.rad +19 -2

    let source = abi::Ref { index: 7, generation: 1 };
    set STORE.windows[entry.object.index].context = source;
    let current = abi::CurrentContextInfo { context: abi::id(source), hart: 0, budget: *budget };
    let before = STORE.windows[entry.object.index];
    let mut rejected: u32 = 0;
    set DOMAINS.records[permission.object.index].state = domains::Lifecycle::Dead;
    for phase in 0..3 {
        if phase == 1 {
            set DOMAINS.reclaimed[permission.object.index] = true;
        }
        if phase == 2 {
            set DOMAINS.slots[permission.object.index].state = slots::State::Free;
        }
        try budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30) catch error {
            assert error == abi::Error::BadHandle; set rejected += 1;
        };
        assert STORE.windows[entry.object.index] == before;
        assert try! capability::get(&TABLE, budget) == entry;
    }
    set DOMAINS.slots[permission.object.index].state = slots::State::Live;
    set DOMAINS.reclaimed[permission.object.index] = false;
    set DOMAINS.records[permission.object.index].state = domains::Lifecycle::Pending;
    try budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30) catch error {
        assert (error == abi::Error::Busy); set rejected += 1;
    };
    set DOMAINS.records[permission.object.index].state = domains::Lifecycle::Active;
    set DOMAINS.contexts[initial.index].state = domains::ContextState::Waiting;

        set MEMORY.table.slots[i].state = slots::State::Retired;
    }
    try budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30) catch error {
        assert (error == abi::Error::Busy); set rejected += 1;
    };
    assert (rejected == 7 and STORE.windows[entry.object.index] == before);
    assert (rejected == 10 and STORE.windows[entry.object.index] == before);
    assert (try! capability::get(&TABLE, budget) == entry);
    capability::initialize(&mut MEMORY.table, permission.object);
    try! budgets::handoff(&mut STORE, &DOMAINS, &mut TABLE, current, target, 30);
    let after = STORE.windows[entry.object.index];
    assert (after.start == 30 and after.end == 100 and after.hart == 0);

    let child = slots::reference(&childSlot);
    capability::initialize(&mut MEMORY.table, child);
    let contextSlot = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
    let context = slots::reference(&contextSlot);
    set DOMAINS.contexts[context.index] = domains::Context {
        owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil,
        owner: child, state: domains::ContextState::Ready, hart: nil, pending: nil, reservation: false,
        kernelFrames: frames::Run { first: 0, count: 0 }, kernelStack: range::Range { start: 0, end: 0 },
        userStack: range::Range { start: 0, end: 0 },
        frame: trap::Frame { registers: [0; 32], pc: 4, status: 0x80, cause: 0, value: 0 },
    };
    set DOMAINS.records[child.index] = domains::Domain {

kernel/kernel/tests/calls.rad +212 -0

use kernel::limits;
use kernel::trap;
use kernel::budgets;
use kernel::remote;
use kernel::interrupts;
use kernel::events;

/// Event storage for administrative destruction targets.
unsafe static TARGETS: [domains::Memory; 2] = undefined;

/// Retained dead-domain records cannot be queried or delegated through handles.
@test unsafe fn deadHandles() throws (testing::TestError) {
    let owner = initialize();
    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
    let object = try! slots::commit(&mut domains::STORE.slots[..], pending);
    try! backing::registerDomain(&mut pages::STORE.backings, object);
    set domains::STORE.records[object.index].state = domains::Lifecycle::Dead;
    set domains::STORE.records[object.index].creator = owner;
    set domains::STORE.records[object.index].parent = nil;
    set domains::STORE.records[object.index].initial = object;
    set domains::STORE.contextSlots[object.index] = slots::Slot { generation: object.generation, state: slots::State::Live };
    set domains::STORE.contexts[object.index].owner = object;
    set domains::STORE.contexts[object.index].state = domains::ContextState::Stopped;
    set domains::STORE.contexts[object.index].hart = nil;
    set domains::STORE.contexts[object.index].userStack = { start: 0, end: 0 };
    for operation in [47 as u64, 48, 62, 10, 12, 21, 61, 20, 30, 51] {
        let mut kind = abi::Kind::Domain;
        if operation == 48 {
            set kind = abi::Kind::Events;
        }
        let handle = try! capability::install(&mut MEMORY.table, capability::Entry {
            kind: kind, object: object,
            rights: abi::Rights(abi::GRANT | abi::EXECUTE | abi::DESTROY | abi::CREATE | abi::ALLOCATE),
        });
        let mut rejected = false;
        let mut argument: u64 = 0;
        if operation == 61 {
            set argument = abi::id(object);
        }
        try calls::invoke(owner, operation, &[*handle, argument, 0, 0], 0) catch error {
            assert error == abi::Error::BadHandle;
            set rejected = true;
        };
        assert rejected;
        assert (try! capability::get(&MEMORY.table, handle)).object == object;
    }
}

/// Administrative destruction checks authority and follows creation ancestry only on request.
@test unsafe fn administrativeTermination() throws (testing::TestError) {
    for flags in [0 as u64, abi::CASCADE] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let mut objects: [abi::Ref; 2] = undefined;
        for i in 0..2 {
            let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
            let object = try! slots::commit(&mut domains::STORE.slots[..], pending);
            set objects[i] = object;
            set domains::STORE.records[object.index].state = domains::Lifecycle::Pending;
            set domains::STORE.records[object.index].creator = owner;
            set domains::STORE.records[object.index].parent = owner;
            set domains::STORE.records[object.index].memory = &mut TARGETS[i];
            try! events::open(&mut domains::STORE.events, object, &mut TARGETS[i].ring);
            set domains::STORE.contextSlots[object.index] = slots::Slot { generation: 1, state: slots::State::Live };
            set domains::STORE.contexts[object.index].owner = object;
            set domains::STORE.contexts[object.index].state = domains::ContextState::Ready;
            set domains::STORE.contexts[object.index].hart = 1 + 2 * i;
        }
        set domains::STORE.records[objects[1].index].creator = objects[0];
        set domains::STORE.records[objects[1].index].parent = objects[0];
        set domains::STORE.ancestors[objects[1].index][0] = 1 << objects[0].index;
        let handle = try! capability::install(&mut MEMORY.table, capability::Entry {
            kind: abi::Kind::Domain, object: objects[0], rights: abi::Rights(0),
        });
        let mut denied = false;
        try calls::termination(owner, 22, &[*handle, flags, 0, 0]) catch error {
            assert error == abi::Error::Denied; set denied = true;
        };
        assert denied and domains::STORE.dead == 0;
        let decoded = try! abi::decode(handle);
        set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::DESTROY);
        let mut invalid = false;
        try calls::termination(owner, 22, &[*handle, 2, 0, 0]) catch error {
            assert error == abi::Error::InvalidArg; set invalid = true;
        };
        assert invalid and domains::STORE.dead == 0;
        let stopped = try! calls::termination(owner, 22, &[*handle, flags, 0, 0]);
        assert domains::STORE.events.terminals[objects[0].index].event.code == 0;
        if flags == 0 {
            assert stopped == 2 and domains::STORE.records[objects[1].index].state == domains::Lifecycle::Pending;
            assert domains::STORE.records[objects[1].index].parent == nil;
        } else {
            assert stopped == 10 and domains::STORE.records[objects[1].index].state == domains::Lifecycle::Dead;
        }
        let mut dead = false;
        try calls::termination(owner, 22, &[*handle, 0, 0, 0]) catch error {
            assert error == abi::Error::BadHandle; set dead = true;
        };
        assert dead;
    }
}

/// Self termination validates arguments and reports the requested terminal kind.
@test unsafe fn terminationCalls() throws (testing::TestError) {
    for operation in [22 as u64, 43, 49] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let pending = try! slots::reserve(&mut domains::STORE.contextSlots[..]);
        let context = try! slots::commit(&mut domains::STORE.contextSlots[..], pending);
        set domains::STORE.contexts[context.index].owner = owner;
        set domains::STORE.contexts[context.index].state = domains::ContextState::Ready;
        set domains::STORE.contexts[context.index].hart = 2;
        let mut invalid = false;
        try calls::termination(owner, operation, &[0, 0, 1, 0]) catch error {
            assert error == abi::Error::InvalidArg; set invalid = true;
        };
        assert invalid and domains::STORE.records[owner.index].state == domains::Lifecycle::Active;
        let mut code: u64 = 0;
        if operation == 49 {
            set code = 0xffffffff;
        }
        assert try! calls::termination(owner, operation, &[code, 0, 0, 0]) == 4;
        let terminal = domains::STORE.events.terminals[owner.index];
        assert terminal.subject == owner;
        if operation == 43 {
            assert terminal.event.kind == events::FAULT and terminal.event.code == abi::FAULT_ABORT;
        } else {
            assert terminal.event.kind == events::CHILD_EXIT and terminal.event.code == code as u32;
        }
        assert domains::STORE.contexts[context.index].state == domains::ContextState::Stopped;
        assert domains::STORE.contexts[context.index].hart == 2;
    }
    let owner = initialize();
    for arguments in &[[0 as u64, 1, 0, 0], [0, 2, 0, 0]] {
        let mut invalid = false;
        try calls::termination(owner, 22, &arguments[..]) catch error {
            assert error == abi::Error::InvalidArg; set invalid = true;
        };
        assert invalid;
    }
    let mut overflow = false;
    try calls::termination(owner, 49, &[0x100000000, 0, 0, 0]) catch error {
        assert error == abi::Error::InvalidArg; set overflow = true;
    };
    assert overflow;
}

/// Parenting requires both capabilities and preserves creation ancestry.
@test unsafe fn reparent() throws (testing::TestError) {
    let owner = initialize();
    let childSlot = try! slots::reserve(&mut domains::STORE.slots[..]);
    let child = try! slots::commit(&mut domains::STORE.slots[..], childSlot);
    let parentSlot = try! slots::reserve(&mut domains::STORE.slots[..]);
    let parent = try! slots::commit(&mut domains::STORE.slots[..], parentSlot);
    set domains::STORE.records[child.index].state = domains::Lifecycle::Pending;
    set domains::STORE.records[child.index].creator = owner;
    set domains::STORE.records[child.index].parent = owner;
    set domains::STORE.records[parent.index].state = domains::Lifecycle::Active;
    let childHandle = try! capability::install(&mut MEMORY.table, capability::Entry {
        kind: abi::Kind::Domain, object: child, rights: abi::Rights(abi::DESTROY),
    });
    let parentHandle = try! capability::install(&mut MEMORY.table, capability::Entry {
        kind: abi::Kind::Domain, object: parent, rights: abi::Rights(abi::WAKE),
    });
    let changed = try! calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0);
    assert changed == 0 and domains::STORE.records[child.index].parent == parent;
    assert domains::STORE.records[child.index].creator == owner;
    let repeated = try! calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0);
    assert repeated == 0;
    let childCap = try! abi::decode(childHandle);
    let parentCap = try! abi::decode(parentHandle);
    let mut denied: u32 = 0;
    for index in &[childCap.object.index, parentCap.object.index] {
        let rights = MEMORY.table.entries[index].rights;
        set MEMORY.table.entries[index].rights = abi::Rights(0);
        try calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0) catch error {
            assert error == abi::Error::Denied; set denied += 1;
        };
        set MEMORY.table.entries[index].rights = rights;
    }
    assert denied == 2 and domains::STORE.records[child.index].parent == parent;
    let mut invalid: u32 = 0;
    for arguments in &[[*childHandle, *parentHandle, 1, 0], [*childHandle, *parentHandle, 0, 1]] {
        try calls::invoke(owner, 23, &arguments[..], 0) catch error {
            assert error == abi::Error::InvalidArg; set invalid += 1;
        };
    }
    assert invalid == 2;
    let mut dead: u32 = 0;
    for object in &[child, parent] {
        set domains::STORE.records[object.index].state = domains::Lifecycle::Dead;
        try calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0) catch error {
            assert error == abi::Error::BadHandle; set dead += 1;
        };
        set domains::STORE.records[object.index].state = domains::Lifecycle::Active;
    }
    assert dead == 2 and domains::STORE.records[child.index].parent == parent;
    try! slots::release(&mut domains::STORE.slots[..], parent);
    try calls::invoke(owner, 23, &[*childHandle, *parentHandle, 0, 0], 0) catch error {
        assert error == abi::Error::BadHandle; set dead += 1;
    };
    assert dead == 3 and domains::STORE.records[child.index].creator == owner;
}

/// Interrupt queries write exactly four bytes at four-byte-aligned addresses.
@test unsafe fn interruptRecord() throws (testing::TestError) {
    let owner = initialize();
    interrupts::initialize(&mut interrupts::STORE, 8);

        try testing::expect(error == abi::Error::BadHandle); set rejected += 1;
    };
    try testing::expect(rejected == 6 and domains::STORE.contextSlots[0].state == slots::State::Free);
}

/// Invalid infallible queries terminate the caller at the syscall boundary.
@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
    for operation in [44 as u64, 46, 47, 48] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let mut frame = trap::Frame { registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
        set frame.registers[17] = operation;
        calls::handle(owner, &mut frame, 0, calls::invoke);
        assert domains::STORE.records[owner.index].state == domains::Lifecycle::Dead;
        assert domains::STORE.events.terminals[owner.index].event.code == abi::FAULT_ABORT;
        assert domains::STORE.events.terminals[owner.index].event.kind == events::FAULT;
    }
}

/// An unknown user operation advances PC and returns only a negative a0 error.
@test unsafe fn trapReply() throws (testing::TestError) {
    let owner = initialize();
    let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
    set frame.registers[17] = 0xffffffffffffffff;

kernel/kernel/tests/domains.rad +374 -4

use kernel::slots;
use kernel::backing;
use kernel::capability;
use kernel::registry;
use kernel::domains;
use kernel::lifecycle;
use kernel::loader;
use kernel::frames;
use kernel::budgets;
use kernel::interrupts;
use kernel::transactions;
use kernel::pages;
use kernel::events;

/// Exclusive resources released by the domain reaper.
unsafe static WINDOWS: budgets::Store = undefined;
/// Interrupt ownership released after hardware masking.
unsafe static IRQS: interrupts::Store = undefined;

/// Caller-owned storage used while a context stack is initialized.
unsafe static CALLS: lifecycle::Calls = undefined;

/// Mapped physical memory for child metadata and package instances.
static RAM: [u64; 131072] = [0; 131072];
/// Domain and context metadata under test.
unsafe static DOMAINS: domains::Store = undefined;
/// Physical allocation and exposure state.

        set BACKINGS.pool.addresses[i] = aligned + i as u64 * 4096;
        set BACKINGS.pool.free[i] = true;
    }
    let pending = try! slots::reserve(&mut DOMAINS.slots[..]);
    let owner = try! slots::commit(&mut DOMAINS.slots[..], pending);
    set DOMAINS.records[owner.index].state = domains::Lifecycle::Active;
    set DOMAINS.records[owner.index].parent = nil;
    set DOMAINS.records[owner.index].creator = owner;
    try! backing::registerDomain(&mut BACKINGS, owner);
    capability::initialize(&mut TABLE, owner);
    let authority = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(rights),
    });

        template: &[7], memory: 8, alignment: 8, relocations: &[],
    });
    set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
}

/// Interrupted creation returns all private storage after its caller releases its hart.
@test unsafe fn cancelledCreation() throws (testing::TestError) {
    initialize(abi::CREATE);
    lifecycle::initialize(&mut CALLS);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let permit = try! capability::get(&TABLE, handle);
    let mut caller = try! domains::get(&DOMAINS, permit.object);
    set DOMAINS.records[permit.object.index].state = domains::Lifecycle::Active;
    let authority = try! capability::install(&mut caller.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: permit.object, rights: abi::Rights(abi::CREATE),
    });
    let root = caller.image;
    let image = try! registry::install(&PACKAGES, &mut caller.memory.table, root);
    set PAGES.backings = BACKINGS;
    let pending = try! domains::reserve(&mut DOMAINS, &mut PAGES.backings, &PACKAGES, &mut caller.memory.table, authority, image);
    let mut creation: domains::Creation = undefined;
    match &pending {
        case domains::Reservation::Held(value) => {
            set creation = *value;
        },
    }
    assert DOMAINS.contexts[creation.context.index].owner == creation.domain;
    let lease = lifecycle::holdDomain(&mut CALLS, &mut DOMAINS, caller.initial, pending);
    try! lifecycle::prepareDomain(&CALLS, &PACKAGES, &lease);
    lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut loader::STATE, &mut registry::STORE);
    assert DOMAINS.slots[creation.domain.index].state == slots::State::Reserved;
    set DOMAINS.records[permit.object.index].state = domains::Lifecycle::Dead;
    set DOMAINS.contexts[caller.initial.index].state = domains::ContextState::Stopped;
    set DOMAINS.contexts[caller.initial.index].hart = 1;
    lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut loader::STATE, &mut registry::STORE);
    assert DOMAINS.contexts[caller.initial.index].reservation;
    assert not PAGES.backings.pool.free[creation.storage.allocation.first];
    match lease {
        case lifecycle::Lease::Held(context) => {
            assert context == caller.initial;
        },
    }
    set DOMAINS.contexts[caller.initial.index].hart = nil;
    lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut loader::STATE, &mut registry::STORE);
    assert not DOMAINS.contexts[caller.initial.index].reservation;
    assert DOMAINS.slots[creation.domain.index].state == slots::State::Free;
    assert DOMAINS.contextSlots[creation.context.index].state == slots::State::Free;
    assert caller.memory.table.slots[creation.handle.index].state == slots::State::Free;
    assert PAGES.backings.pool.retiredCount == 3;
    while PAGES.backings.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    for run in [creation.storage.allocation, creation.storage.kernelFrames, creation.storage.graph.frames] {
        for i in run.first..run.first + run.count {
            assert PAGES.backings.pool.free[i];
        }
    }
    set DOMAINS.records[permit.object.index].state = domains::Lifecycle::Active;
    let next = try! domains::reserve(&mut DOMAINS, &mut PAGES.backings, &PACKAGES, &mut caller.memory.table, authority, image);
    let active = lifecycle::holdDomain(&mut CALLS, &mut DOMAINS, caller.initial, next);
    try! lifecycle::prepareDomain(&CALLS, &PACKAGES, &active);
    let ready = lifecycle::takeDomain(&mut CALLS, &mut DOMAINS, active);
    let published = try! domains::publish(&mut DOMAINS, &mut PAGES.backings, &mut caller.memory.table, ready);
    let entry = try! capability::get(&caller.memory.table, published);
    assert entry.object.index == creation.domain.index and entry.object.generation <> creation.domain.generation;
    assert (try! domains::get(&DOMAINS, entry.object)).state == domains::Lifecycle::Pending;
    assert not DOMAINS.contexts[caller.initial.index].reservation;
}

/// Private storage and terminal identity have separate reclamation boundaries.
@test unsafe fn reclaimDomain() throws (testing::TestError) {
    initialize(abi::CREATE);
    let parentHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let entry = try! capability::get(&TABLE, handle);
    let child = try! domains::get(&DOMAINS, entry.object);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, parentHandle);
    let mut private = child.memory;
    let authority = try! capability::install(&mut private.table, capability::Entry {
        kind: abi::Kind::Domain, object: entry.object, rights: abi::Rights(abi::CREATE),
    });
    let image = try! registry::install(&PACKAGES, &mut private.table, child.image);
    let descendantHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut private.table, authority, image);
    let descendant = try! capability::get(&private.table, descendantHandle);
    let word = entry.object.index / 32;
    let bit = 1 << (entry.object.index % 32);
    assert (DOMAINS.ancestors[descendant.object.index][word] & bit) <> 0;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].hart = 1;
    assert try! domains::terminate(&mut DOMAINS, entry.object, events::CHILD_EXIT, 17) == 2;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].hart = nil;
    set DOMAINS.contexts[child.initial.index].pending = abi::Ref { index: 0, generation: 1 };
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].pending = nil;
    set DOMAINS.contexts[child.initial.index].reservation = true;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    set DOMAINS.contexts[child.initial.index].reservation = false;
    let reserved = try! slots::reserve(&mut DOMAINS.contextSlots[..]);
    let context = slots::reference(&reserved);
    set DOMAINS.contexts[context.index].owner = entry.object;
    assert not try! domains::quiescent(&DOMAINS, entry.object);
    let mut busy = false;
    try domains::retireContext(&mut DOMAINS, &mut BACKINGS.pool, entry.object) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy and BACKINGS.pool.retiredCount == 0;
    try! slots::cancel(&mut DOMAINS.contextSlots[..], reserved);
    assert try! domains::quiescent(&DOMAINS, entry.object);
    assert try! domains::retireContext(&mut DOMAINS, &mut BACKINGS.pool, entry.object);
    assert not try! domains::retireContext(&mut DOMAINS, &mut BACKINGS.pool, entry.object);
    let mut handles = false;
    try domains::retireStorage(&mut DOMAINS, &mut BACKINGS, entry.object) catch error {
        assert error == abi::Error::Busy; set handles = true;
    };
    assert handles;
    let mut memory = child.memory;
    for cap in [child.events, authority, image, descendantHandle] {
        let removed = try! capability::invalidate(&mut memory.table, cap);
    }
    let mut exposed = false;
    try domains::retireStorage(&mut DOMAINS, &mut BACKINGS, entry.object) catch error {
        assert error == abi::Error::Busy; set exposed = true;
    };
    assert exposed;
    while not try! backing::endDomainStep(&mut BACKINGS, entry.object, 64) {
    }
    try! domains::retireStorage(&mut DOMAINS, &mut BACKINGS, entry.object);
    assert BACKINGS.pool.retiredCount == 3;
    let mut stale = false;
    try domains::get(&DOMAINS, entry.object) catch error {
        assert error == abi::Error::BadHandle; set stale = true;
    };
    assert stale and DOMAINS.slots[entry.object.index].state == slots::State::Live;
    while BACKINGS.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut BACKINGS.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    assert not try! events::acknowledged(&DOMAINS.events, entry.object);
    let mut retained = false;
    try domains::releaseIdentity(&mut DOMAINS, entry.object) catch error {
        assert error == abi::Error::Busy; set retained = true;
    };
    assert retained;
    let parentEntry = try! capability::get(&TABLE, parentHandle);
    let mut parent = try! domains::get(&DOMAINS, parentEntry.object);
    assert try! events::flush(&mut DOMAINS.events, parentEntry.object, &mut parent.memory.ring) == 1;
    let event = try! events::pop(&mut parent.memory.ring) else panic "missing terminal event";
    assert event.value == abi::id(entry.object) and event.code == 17;
    try! events::refresh(&mut DOMAINS.events, parentEntry.object, &parent.memory.ring);
    try! domains::releaseIdentity(&mut DOMAINS, entry.object);
    assert DOMAINS.slots[entry.object.index].state == slots::State::Free;
    assert (DOMAINS.ancestors[descendant.object.index][word] & bit) == 0;
    let next = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let replacement = try! capability::get(&TABLE, next);
    assert replacement.object.index == entry.object.index and replacement.object.generation <> entry.object.generation;
    assert (try! domains::get(&DOMAINS, replacement.object)).state == domains::Lifecycle::Pending;
    assert try! domains::terminateTree(&mut DOMAINS, replacement.object, events::CHILD_EXIT, 0) == 0;
    assert (try! domains::get(&DOMAINS, descendant.object)).state == domains::Lifecycle::Pending;
}

/// Reaping releases private authority while surviving shared pages retain their frames.
@test unsafe fn reaper() throws (testing::TestError) {
    initialize(abi::CREATE | abi::ALLOCATE);
    budgets::initialize(&mut WINDOWS);
    interrupts::initialize(&mut IRQS, 8);
    let parentHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let target = (try! capability::get(&TABLE, handle)).object;
    let receiver = (try! capability::get(&TABLE, parentHandle)).object;
    let mut child = try! domains::get(&DOMAINS, target);
    let mut parent = try! domains::get(&DOMAINS, receiver);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, parentHandle);
    set PAGES.backings = BACKINGS;
    slots::initialize(&mut PAGES.slots[..]);
    let authority = try! capability::install(&mut child.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: target, rights: abi::Rights(abi::ALLOCATE),
    });
    let page = try! pages::allocate(&mut PAGES, &mut child.memory.table, authority, 1);
    let shared = try! pages::grant(&mut PAGES, &child.memory.table, &mut parent.memory.table, page, abi::READ as u64);
    let allocation = (try! pages::get(&PAGES, (try! capability::get(&child.memory.table, page)).object)).backing;
    let run = PAGES.backings.records[allocation.index].run;
    let budget = try! budgets::seed(&mut WINDOWS, &mut child.memory.table, 0, 0, 1000);
    let window = (try! capability::get(&child.memory.table, budget)).object;
    let irq = try! interrupts::seed(&mut IRQS, &mut child.memory.table, 7);
    let dangling = try! capability::install(&mut child.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: abi::Ref { index: 511, generation: 99 }, rights: abi::Rights(0),
    });
    set DOMAINS.contexts[child.initial.index].hart = 0;
    assert try! domains::terminate(&mut DOMAINS, target, events::CHILD_EXIT, 19) == 1;
    assert not lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    assert (try! capability::get(&child.memory.table, page)).kind == abi::Kind::Page;
    set DOMAINS.contexts[child.initial.index].hart = nil;
    assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    assert child.memory.table.slots[0].state == slots::State::Free;
    assert (try! capability::get(&child.memory.table, page)).kind == abi::Kind::Page;
    for step in 0..2000 {
        if DOMAINS.reclaimed[target.index] {
            break;
        }
        assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    }
    assert DOMAINS.reclaimed[target.index];
    assert WINDOWS.slots[window.index].state == slots::State::Free;
    assert IRQS.slots[6].state == slots::State::Free;
    assert not lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    while PAGES.backings.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    assert not PAGES.backings.pool.free[run.first];
    assert PAGES.backings.records[allocation.index].exposed == 1;
    assert (try! pages::get(&PAGES, (try! capability::get(&parent.memory.table, shared)).object)).count == 1;
    assert try! events::flush(&mut DOMAINS.events, receiver, &mut parent.memory.ring) == 1;
    let event = try! events::pop(&mut parent.memory.ring) else panic "missing reaped child event";
    assert event.value == abi::id(target) and event.code == 19;
    try! events::refresh(&mut DOMAINS.events, receiver, &parent.memory.ring);
    assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    assert DOMAINS.slots[target.index].state == slots::State::Free;
    assert try! domains::terminate(&mut DOMAINS, receiver, events::CHILD_EXIT, 0) == 0;
    for step in 0..2000 {
        if DOMAINS.slots[receiver.index].state == slots::State::Free {
            break;
        }
        assert lifecycle::reapNext(&mut DOMAINS, &mut PAGES, &mut WINDOWS, &mut IRQS);
    }
    assert DOMAINS.slots[receiver.index].state == slots::State::Free;
    while PAGES.backings.pool.retiredCount > 0 {
        let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
        assert reclaimed > 0 and reclaimed <= 256;
    }
    for i in 0..PAGES.backings.pool.count {
        assert PAGES.backings.pool.free[i];
    }
}

/// Initial return reports to the current parent and retains its terminal identity.
@test unsafe fn terminalParent() throws (testing::TestError) {
    initialize(abi::CREATE);
    let first = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let second = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let target = try! capability::get(&TABLE, handle);
    let destination = try! capability::get(&TABLE, second);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, first);
    try! domains::reparent(&mut DOMAINS, &TABLE, handle, second);
    let child = try! domains::get(&DOMAINS, target.object);
    set DOMAINS.contexts[child.initial.index].hart = 3;
    set DOMAINS.contexts[child.initial.index].pending = abi::Ref { index: 7, generation: 1 };
    let stopped = try! domains::returned(&mut DOMAINS, target.object, child.initial);
    assert stopped == 8 and DOMAINS.contexts[child.initial.index].hart == 3;
    assert DOMAINS.contexts[child.initial.index].pending <> nil;
    let repeated = try! domains::terminate(&mut DOMAINS, target.object, events::FAULT, 99);
    assert repeated == 0;
    assert DOMAINS.records[target.object.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.events.terminals[target.object.index].state == events::State::Pending;
    assert DOMAINS.events.terminals[target.object.index].receiver == destination.object;
    assert DOMAINS.events.queues[target.object.index].generation == 0;
    let mut rejected = false;
    try domains::reparent(&mut DOMAINS, &TABLE, handle, first) catch error {
        assert error == abi::Error::BadHandle; set rejected = true;
    };
    assert rejected and DOMAINS.records[target.object.index].creator == TABLE.owner;
    let mut parent = try! domains::get(&DOMAINS, destination.object);
    assert try! events::flush(&mut DOMAINS.events, destination.object, &mut parent.memory.ring) == 1;
    assert not try! events::acknowledged(&DOMAINS.events, target.object);
    let event = try! events::pop(&mut parent.memory.ring) else panic "missing child exit";
    assert event.kind == events::CHILD_EXIT and event.code == 0 and event.value == abi::id(target.object);
    try! events::refresh(&mut DOMAINS.events, destination.object, &parent.memory.ring);
    assert try! events::acknowledged(&DOMAINS.events, target.object);
}

/// Parent exit leaves surviving children parentless without changing their creator.
@test unsafe fn parentExit() throws (testing::TestError) {
    initialize(abi::CREATE);
    let parentHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let childHandle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let parentCap = try! capability::get(&TABLE, parentHandle);
    let childCap = try! capability::get(&TABLE, childHandle);
    try! domains::reparent(&mut DOMAINS, &TABLE, childHandle, parentHandle);
    let parent = try! domains::get(&DOMAINS, parentCap.object);
    try! domains::returned(&mut DOMAINS, parentCap.object, parent.initial);
    let child = try! domains::get(&DOMAINS, childCap.object);
    assert child.state == domains::Lifecycle::Pending and child.parent == nil and child.creator == TABLE.owner;
    try! domains::returned(&mut DOMAINS, childCap.object, child.initial);
    assert try! events::acknowledged(&DOMAINS.events, childCap.object);
}

/// Create a child through its creator's own authority and resident image.
unsafe fn descendant(table: &mut capability::Table, image: abi::Ref) -> abi::Ref {
    let owner = table.owner;
    let authority = try! capability::install(table, capability::Entry {
        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE),
    });
    let executable = try! registry::install(&PACKAGES, table, image);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, table, authority, executable);
    return (try! capability::get(table, handle)).object;
}

/// Cascade retains creation ancestry after reparenting and an intermediate exit.
@test unsafe fn cascadeAncestry() throws (testing::TestError) {
    initialize(abi::CREATE);
    let package = try! capability::get(&TABLE, IMAGE);
    let first = descendant(&mut TABLE, package.object);
    let mut ancestor = try! domains::get(&DOMAINS, first);
    let second = descendant(&mut ancestor.memory.table, package.object);
    let mut middle = try! domains::get(&DOMAINS, second);
    let third = descendant(&mut middle.memory.table, package.object);
    let child = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: third, rights: abi::Rights(abi::DESTROY),
    });
    let root = TABLE.owner;
    let receiver = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: root, rights: abi::Rights(abi::WAKE),
    });
    try! domains::reparent(&mut DOMAINS, &TABLE, child, receiver);
    let reserved = try! events::reserve(&mut DOMAINS.events, third);
    let mut busy = false;
    try domains::terminateTree(&mut DOMAINS, first, events::CHILD_EXIT, 7) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy and DOMAINS.records[first.index].state == domains::Lifecycle::Pending;
    assert DOMAINS.records[second.index].state == domains::Lifecycle::Pending;
    assert DOMAINS.events.terminals[first.index].state == events::State::Free;
    events::cancel(&mut DOMAINS.events, reserved);
    let stopped = try! domains::terminate(&mut DOMAINS, second, events::CHILD_EXIT, 12);
    assert stopped == 0 and DOMAINS.records[third.index].state == domains::Lifecycle::Pending;
    assert DOMAINS.records[third.index].parent == TABLE.owner;
    set DOMAINS.contexts[ancestor.initial.index].hart = 1;
    let last = try! domains::get(&DOMAINS, third);
    set DOMAINS.contexts[last.initial.index].hart = 3;
    let harts = try! domains::terminateTree(&mut DOMAINS, first, events::CHILD_EXIT, 7);
    assert harts == 10;
    assert DOMAINS.records[first.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.records[second.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.records[third.index].state == domains::Lifecycle::Dead;
    assert DOMAINS.records[TABLE.owner.index].state == domains::Lifecycle::Active;
    assert DOMAINS.events.terminals[second.index].event.code == 12;
    assert DOMAINS.events.terminals[third.index].event.code == 7;
    assert DOMAINS.records[third.index].creator == second;
    assert DOMAINS.contexts[last.initial.index].hart == 3;
}

/// Activation validates child authority before publishing its initial state.
@test unsafe fn activation() throws (testing::TestError) {
    initialize(abi::CREATE);
    slots::initialize(&mut PAGES.slots[..]);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);

    let stackBase = DOMAINS.contexts[index].kernelStack.start;
    let base = (&RAM[0]) as u64;
    let firstWord = ((stackBase - base) / 8) as u32;
    let lastWord = firstWord + domains::KERNEL_STACK_PAGES * 512 - 1;
    set RAM[firstWord] = 123; set RAM[lastWord] = 456;
    domains::contextClear(&pendingContext);
    lifecycle::initialize(&mut CALLS);
    let lease = lifecycle::holdContext(&mut CALLS, &mut DOMAINS, child.initial, pendingContext);
    assert DOMAINS.contexts[child.initial.index].reservation;
    lifecycle::clearContext(&CALLS, &lease);
    assert RAM[firstWord] == 0 and RAM[lastWord] == 0;
    let context = domains::contextPublish(&mut DOMAINS, pendingContext);
    let ready = lifecycle::takeContext(&mut CALLS, &mut DOMAINS, lease);
    assert not DOMAINS.contexts[child.initial.index].reservation;
    let context = domains::contextPublish(&mut DOMAINS, ready);
    let created = try! domains::context(&DOMAINS, authority.object, context);
    let initial = DOMAINS.contexts[child.initial.index];
    try testing::expect(context <> child.initial and created.owner == authority.object);
    try testing::expect(created.frame.registers[3] == initial.frame.registers[3]);
    try testing::expect(created.kernelStack.start >= initial.kernelStack.end or created.kernelStack.end <= initial.kernelStack.start);

    assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        assert not PAGES.backings.pool.free[i];
    }
    set DOMAINS.contexts[context.index].pending = nil;
    set DOMAINS.contexts[context.index].reservation = true;
    set retained = false;
    try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context) catch error {
        assert error == abi::Error::Busy; set retained = true;
    };
    assert retained and DOMAINS.contextSlots[context.index].state == slots::State::Live;
    set DOMAINS.contexts[context.index].reservation = false;
    try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, context);
    for i in created.kernelFrames.first..created.kernelFrames.first + created.kernelFrames.count {
        try testing::expect(PAGES.backings.pool.free[i]);
    }
    try domains::context(&DOMAINS, authority.object, context) catch error {

    let next = try! domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    try testing::expect(next.index == context.index and next.generation <> context.generation);
    try! domains::returned(&mut DOMAINS, authority.object, child.initial);
    try testing::expect(DOMAINS.records[authority.object.index].state == domains::Lifecycle::Dead);
    try testing::expect(DOMAINS.contexts[next.index].state == domains::ContextState::Stopped);
    try! domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, next);
    try testing::expect(rejected == 11);
    try domains::contextDestroy(&mut DOMAINS, &mut PAGES, &TABLE, handle, next) catch error {
        assert error == abi::Error::BadHandle; set rejected += 1;
    };
    assert DOMAINS.contextSlots[next.index].state == slots::State::Live;
    assert rejected == 12;
}

/// Both serialized wait/wakeup orders preserve notification and runnable state.
@test unsafe fn waitWakeup() throws (testing::TestError) {
    initialize(abi::CREATE);

kernel/kernel/tests/frames.rad +30 -0

//! Contiguous frame reservation and reserved-memory exclusion.

use std::testing;
use kernel::abi;
use kernel::frames;
use kernel::limits;
use kernel::platform;
use kernel::range;

/// Frame-map workspace is static because its size exceeds a small kernel stack.
unsafe static POOL: frames::Pool = undefined;

    set machine.reservedCount = 1;
    set machine.reserved[0] = range::Range { start: 0x80003000, end: 0x80005000 };
    try! frames::initialize(&mut POOL, &machine);
}

/// Retired frames stay unavailable until bounded reclamation returns them.
@test unsafe fn retirement() throws (testing::TestError) {
    initialize();
    set POOL.retiredHead = limits::FRAMES - 1;
    let left = frames::commit(try! frames::reserve(&mut POOL, 3));
    let right = frames::commit(try! frames::reserve(&mut POOL, 5));
    try! frames::retire(&mut POOL, left);
    try! frames::retire(&mut POOL, right);
    assert POOL.retiredCount == 2;
    for i in 0..8 {
        assert not POOL.free[i];
    }
    assert frames::reclaim(&mut POOL, 0) == 0;
    assert frames::reclaim(&mut POOL, 2) == 2;
    assert POOL.free[0] and POOL.free[1] and not POOL.free[2];
    let reused = frames::commit(try! frames::reserve(&mut POOL, 2));
    assert reused.first == 0;
    assert frames::reclaim(&mut POOL, 3) == 3;
    assert POOL.retiredCount == 1 and not POOL.free[5];
    assert not POOL.free[0] and not POOL.free[1];
    try! frames::retire(&mut POOL, reused);
    assert frames::reclaim(&mut POOL, 20) == 5;
    assert POOL.retiredCount == 0;
    for i in 0..8 {
        assert POOL.free[i];
    }
    assert frames::reclaim(&mut POOL, 20) == 0;
}

/// Adjacent physical frames form runs; a reserved hole cannot be crossed.
@test unsafe fn fragmentation() throws (testing::TestError) {
    initialize();
    try testing::expect(POOL.count == 8);
    let left = try! frames::reserve(&mut POOL, 3);

kernel/kernel/tests/lifecycle.rad added +329 -0

1	+	//! Terminal delivery, receiver eligibility, and retained full-queue events.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::domains;
6	+	use kernel::budgets;
7	+	use kernel::events;
8	+	use kernel::lifecycle;
9	+	use kernel::registry;
10	+	use kernel::loader;
11	+	use kernel::limits;
12	+	use kernel::remote;
13	+	use kernel::pages;
14	+	use kernel::backing;
15	+	use kernel::capability;
16	+	use kernel::frames;
17	+
18	+	/// Receiver generations, contexts, and terminal events.
19	+	unsafe static STORE: domains::Store = undefined;
20	+	/// CPU authority retained by receivers.
21	+	unsafe static WINDOWS: budgets::Store = undefined;
22	+	/// Queued calls and acknowledgements retained by stopped contexts.
23	+	unsafe static REQUESTS: remote::Store = undefined;
24	+	/// Physical allocation metadata used for interrupted page calls.
25	+	unsafe static MEMORY: pages::Store = undefined;
26	+	/// Mapped frames available to interrupted page calls.
27	+	static RAM: [u64; 4096] = [0; 4096];
28	+	/// Reservations retained across private initialization.
29	+	unsafe static CALLS: lifecycle::Calls = undefined;
30	+	/// Shared queue storage for the first receiver.
31	+	unsafe static FIRST: domains::Memory = undefined;
32	+	/// Shared queue storage for the second receiver.
33	+	unsafe static SECOND: domains::Memory = undefined;
34	+
35	+	/// Open one receiver with a waiting initial context.
36	+	unsafe fn open(index: u32, memory: &mut domains::Memory) {
37	+	let owner = abi::Ref { index, generation: 1 };
38	+	set STORE.slots[index] = slots::Slot { generation: 1, state: slots::State::Live };
39	+	set STORE.records[index].state = domains::Lifecycle::Active;
40	+	set STORE.records[index].memory = (&mut memory) as unsafe mut domains::Memory;
41	+	set STORE.contextSlots[index] = slots::Slot { generation: 1, state: slots::State::Live };
42	+	set STORE.contexts[index].owner = owner;
43	+	set STORE.contexts[index].state = domains::ContextState::Waiting;
44	+	set STORE.contexts[index].hart = nil;
45	+	set STORE.contexts[index].pending = nil;
46	+	try! events::open(&mut STORE.events, owner, &mut memory.ring);
47	+	}
48	+
49	+	/// Initialize two independent receivers and one future window on hart one.
50	+	unsafe fn initialize() {
51	+	domains::initialize(&mut STORE);
52	+	budgets::initialize(&mut WINDOWS);
53	+	remote::initialize(&mut REQUESTS, 3);
54	+	open(0, &mut FIRST);
55	+	open(1, &mut SECOND);
56	+	let owner = abi::Ref { index: 0, generation: 1 };
57	+	set WINDOWS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
58	+	set WINDOWS.windows[0] = budgets::Window {
59	+	hart: 1, start: 30, end: 100, owner, handle: abi::Handle(0), context: owner,
60	+	};
61	+	}
62	+
63	+	/// Retain a normal exit for a chosen receiver.
64	+	unsafe fn retain(index: u32, receiver: u32) {
65	+	let pending = try! events::reserve(&mut STORE.events, abi::Ref { index, generation: 1 });
66	+	events::finish(&mut STORE.events, pending, abi::Ref { index: receiver, generation: 1 }, events::CHILD_EXIT, index);
67	+	}
68	+
69	+	/// Dead page callers release unpublished capacity only after hart ownership ends.
70	+	@test unsafe fn cancelPage() throws (testing::TestError) {
71	+	initialize();
72	+	lifecycle::initialize(&mut CALLS);
73	+	backing::initialize(&mut MEMORY.backings);
74	+	slots::initialize(&mut MEMORY.slots[..]);
75	+	let owner = abi::Ref { index: 0, generation: 1 };
76	+	try! backing::registerDomain(&mut MEMORY.backings, owner);
77	+	capability::initialize(&mut FIRST.table, owner);
78	+	let authority = try! capability::install(&mut FIRST.table, capability::Entry {
79	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
80	+	});
81	+	let base = ((&RAM[0]) as u64 + 4095) & ~4095;
82	+	set MEMORY.backings.pool.count = 2;
83	+	for i in 0..2 {
84	+	set MEMORY.backings.pool.addresses[i] = base + i as u64 * 4096;
85	+	set MEMORY.backings.pool.free[i] = true;
86	+	}
87	+	let reservation = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 2);
88	+	let held = lifecycle::holdPage(&mut CALLS, &mut STORE, owner, reservation);
89	+	assert STORE.contexts[0].reservation;
90	+	lifecycle::clearPage(&CALLS, &held);
91	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
92	+	assert not MEMORY.backings.pool.free[0] and not MEMORY.backings.pool.free[1];
93	+	set STORE.records[0].state = domains::Lifecycle::Dead;
94	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
95	+	set STORE.contexts[0].hart = 0;
96	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
97	+	assert not MEMORY.backings.pool.free[0];
98	+	// Model the end of the stopped continuation before hart release.
99	+	match held {
100	+	case lifecycle::Lease::Held(context) => {
101	+	assert context == owner;
102	+	},
103	+	}
104	+	set STORE.contexts[0].hart = nil;
105	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
106	+	assert MEMORY.backings.pool.retiredCount == 1 and not MEMORY.backings.pool.free[0];
107	+	assert frames::reclaim(&mut MEMORY.backings.pool, 256) == 2;
108	+	assert MEMORY.backings.pool.free[0] and MEMORY.backings.pool.free[1];
109	+	assert MEMORY.slots[0].state == slots::State::Free;
110	+	assert MEMORY.backings.slots[0].state == slots::State::Free;
111	+	assert FIRST.table.slots[1].state == slots::State::Free and CALLS.active[0] == 0;
112	+	assert not STORE.contexts[0].reservation;
113	+	set STORE.records[0].state = domains::Lifecycle::Active;
114	+	let next = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 1);
115	+	let lease = lifecycle::holdPage(&mut CALLS, &mut STORE, owner, next);
116	+	lifecycle::clearPage(&CALLS, &lease);
117	+	let ready = lifecycle::takePage(&mut CALLS, &mut STORE, lease);
118	+	let handle = pages::publish(&mut MEMORY, &mut FIRST.table, ready);
119	+	assert CALLS.active[0] == 0;
120	+	let entry = try! capability::get(&FIRST.table, handle);
121	+	assert (try! pages::get(&MEMORY, entry.object)).count == 1;
122	+	}
123	+
124	+	/// One cancellation pass retires at most one caller's private allocation.
125	+	@test unsafe fn cancellationBatch() throws (testing::TestError) {
126	+	initialize();
127	+	lifecycle::initialize(&mut CALLS);
128	+	backing::initialize(&mut MEMORY.backings);
129	+	slots::initialize(&mut MEMORY.slots[..]);
130	+	let owner = abi::Ref { index: 0, generation: 1 };
131	+	let second = abi::Ref { index: 1, generation: 1 };
132	+	set STORE.contexts[1].owner = owner;
133	+	try! backing::registerDomain(&mut MEMORY.backings, owner);
134	+	capability::initialize(&mut FIRST.table, owner);
135	+	let authority = try! capability::install(&mut FIRST.table, capability::Entry {
136	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
137	+	});
138	+	let base = ((&RAM[0]) as u64 + 4095) & ~4095;
139	+	set MEMORY.backings.pool.count = 2;
140	+	for i in 0..2 {
141	+	set MEMORY.backings.pool.addresses[i] = base + i as u64 * 4096;
142	+	set MEMORY.backings.pool.free[i] = true;
143	+	}
144	+	let firstPage = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 1);
145	+	let first = lifecycle::holdPage(&mut CALLS, &mut STORE, owner, firstPage);
146	+	let secondPage = try! pages::reserve(&mut MEMORY, &mut FIRST.table, authority, 1);
147	+	let other = lifecycle::holdPage(&mut CALLS, &mut STORE, second, secondPage);
148	+	match first {
149	+	case lifecycle::Lease::Held(context) => {
150	+	assert context == owner;
151	+	},
152	+	}
153	+	match other {
154	+	case lifecycle::Lease::Held(context) => {
155	+	assert context == second;
156	+	},
157	+	}
158	+	set STORE.records[0].state = domains::Lifecycle::Dead;
159	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
160	+	set STORE.contexts[1].state = domains::ContextState::Stopped;
161	+	assert lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
162	+	assert CALLS.active[0] == 2 and MEMORY.backings.pool.retiredCount == 1;
163	+	assert not STORE.contexts[0].reservation and STORE.contexts[1].reservation;
164	+	assert lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
165	+	assert CALLS.active[0] == 0 and MEMORY.backings.pool.retiredCount == 2;
166	+	assert not lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
167	+	assert not MEMORY.backings.pool.free[0] and not MEMORY.backings.pool.free[1];
168	+	assert frames::reclaim(&mut MEMORY.backings.pool, 1) == 1;
169	+	assert frames::reclaim(&mut MEMORY.backings.pool, 1) == 1;
170	+	assert MEMORY.backings.pool.retiredCount == 0;
171	+	}
172	+
173	+	/// A context call belongs to its caller even when its target domain dies.
174	+	@test unsafe fn cancelContext() throws (testing::TestError) {
175	+	initialize();
176	+	lifecycle::initialize(&mut CALLS);
177	+	backing::initialize(&mut MEMORY.backings);
178	+	let caller = abi::Ref { index: 0, generation: 1 };
179	+	let target = abi::Ref { index: 1, generation: 1 };
180	+	let base = ((&RAM[0]) as u64 + 4095) & ~4095;
181	+	set MEMORY.backings.pool.count = 1;
182	+	set MEMORY.backings.pool.addresses[0] = base;
183	+	set MEMORY.backings.pool.free[0] = true;
184	+	let slot = try! slots::reserve(&mut STORE.contextSlots[..]);
185	+	let object = slots::reference(&slot);
186	+	let run = frames::commit(try! frames::reserve(&mut MEMORY.backings.pool, 1));
187	+	set STORE.contexts[object.index].owner = target;
188	+	set STORE.contexts[object.index].kernelFrames = run;
189	+	set STORE.contexts[object.index].hart = nil;
190	+	match slot {
191	+	case slots::Reservation::Held(id) => {
192	+	assert id == object;
193	+	},
194	+	}
195	+	let pending = domains::ContextReservation::Held(domains::ContextAllocation {
196	+	object, owner: target, frames: run, base,
197	+	});
198	+	let lease = lifecycle::holdContext(&mut CALLS, &mut STORE, caller, pending);
199	+	lifecycle::clearContext(&CALLS, &lease);
200	+	set STORE.records[1].state = domains::Lifecycle::Dead;
201	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
202	+	assert STORE.contexts[0].reservation and not MEMORY.backings.pool.free[0];
203	+	set STORE.records[0].state = domains::Lifecycle::Dead;
204	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
205	+	set STORE.contexts[0].hart = 1;
206	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
207	+	assert STORE.contextSlots[object.index].state == slots::State::Reserved;
208	+	match lease {
209	+	case lifecycle::Lease::Held(id) => {
210	+	assert id == caller;
211	+	},
212	+	}
213	+	set STORE.contexts[0].hart = nil;
214	+	lifecycle::cancelAllocations(&mut CALLS, &mut STORE, &mut MEMORY, &mut loader::STATE, &mut registry::STORE);
215	+	assert MEMORY.backings.pool.retiredCount == 1 and not MEMORY.backings.pool.free[0];
216	+	assert frames::reclaim(&mut MEMORY.backings.pool, 256) == 1;
217	+	assert MEMORY.backings.pool.free[0];
218	+	assert STORE.contextSlots[object.index].state == slots::State::Free;
219	+	assert not STORE.contexts[0].reservation and CALLS.active[0] == 0;
220	+	}
221	+
222	+	/// Dead contexts retain requests until hart release and completion of taken work.
223	+	@test unsafe fn cancelRequests() throws (testing::TestError) {
224	+	initialize();
225	+	let context = abi::Ref { index: 0, generation: 1 };
226	+	let pending = try! remote::send(&mut REQUESTS, 0, 1, remote::Action::Ready(context));
227	+	let other = try! remote::send(&mut REQUESTS, 0, 1, remote::Action::Ready(context));
228	+	let delivered = try! remote::take(&mut REQUESTS, 1) else panic "missing readiness request";
229	+	assert delivered.id == pending;
230	+	set STORE.contexts[0].pending = other;
231	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
232	+	assert REQUESTS.slots[other.index].state == slots::State::Live;
233	+	set STORE.records[0].state = domains::Lifecycle::Dead;
234	+	set STORE.contexts[0].state = domains::ContextState::Stopped;
235	+	set STORE.contexts[0].hart = 0;
236	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
237	+	assert REQUESTS.slots[other.index].state == slots::State::Live;
238	+	set STORE.contexts[0].hart = nil;
239	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
240	+	assert REQUESTS.slots[other.index].state == slots::State::Free;
241	+	assert REQUESTS.slots[pending.index].state == slots::State::Live;
242	+	assert STORE.contexts[0].pending == pending;
243	+	assert remote::acknowledge(&mut REQUESTS, 1, pending);
244	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
245	+	assert STORE.contexts[0].pending == nil and REQUESTS.slots[pending.index].state == slots::State::Free;
246	+	assert try! remote::take(&mut REQUESTS, 1) == nil;
247	+	assert REQUESTS.queues[1].count == 0;
248	+	let budget = try! remote::send(&mut REQUESTS, 0, 1, remote::Action::Budget(remote::Call {
249	+	owner: context, context, operation: 70, arguments: [0; 4],
250	+	}));
251	+	set STORE.contexts[0].pending = budget;
252	+	let work = try! remote::take(&mut REQUESTS, 1) else panic "missing budget request";
253	+	remote::respond(&mut REQUESTS, 1, work.id, remote::Reply::Error(abi::Error::BadHandle));
254	+	assert remote::acknowledge(&mut REQUESTS, 1, work.id);
255	+	lifecycle::cancelRequests(&mut STORE, &mut REQUESTS);
256	+	assert STORE.contexts[0].pending == nil and REQUESTS.slots[budget.index].state == slots::State::Free;
257	+	}
258	+
259	+	/// Closing multiple receivers validates all generations before changing any queue.
260	+	@test unsafe fn closeReceivers() throws (testing::TestError) {
261	+	initialize();
262	+	retain(2, 0);
263	+	retain(3, 1);
264	+	let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
265	+	set generations[0] = 1;
266	+	set generations[1] = 2;
267	+	let mut rejected = false;
268	+	try events::closeSet(&mut STORE.events, &generations[..]) catch error {
269	+	assert error == abi::Error::BadHandle; set rejected = true;
270	+	};
271	+	assert rejected and STORE.events.queues[0].generation == 1 and STORE.events.queues[1].generation == 1;
272	+	assert STORE.events.pending == 2;
273	+	set generations[1] = 1;
274	+	try! events::closeSet(&mut STORE.events, &generations[..]);
275	+	assert STORE.events.pending == 0;
276	+	assert STORE.events.queues[0].generation == 0 and STORE.events.queues[1].generation == 0;
277	+	assert try! events::acknowledged(&STORE.events, abi::Ref { index: 2, generation: 1 });
278	+	assert try! events::acknowledged(&STORE.events, abi::Ref { index: 3, generation: 1 });
279	+	}
280	+
281	+	/// Events ready both receivers but signal only existing CPU authority.
282	+	@test unsafe fn receivers() throws (testing::TestError) {
283	+	initialize();
284	+	retain(2, 0);
285	+	retain(3, 1);
286	+	assert STORE.events.pending == 2;
287	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2;
288	+	assert STORE.events.pending == 0;
289	+	assert STORE.contexts[0].state == domains::ContextState::Ready;
290	+	assert STORE.contexts[1].state == domains::ContextState::Ready;
291	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 0;
292	+	assert STORE.events.queues[0].tail == 1 and STORE.events.queues[1].tail == 1;
293	+	let first = try! events::pop(&mut FIRST.ring) else panic "first exit";
294	+	let second = try! events::pop(&mut SECOND.ring) else panic "second exit";
295	+	assert first.kind == events::CHILD_EXIT and first.code == 2;
296	+	assert second.kind == events::CHILD_EXIT and second.code == 3;
297	+	retain(4, 0);
298	+	assert lifecycle::service(&mut STORE, &WINDOWS, 100) == 0;
299	+	assert STORE.events.terminals[4].state == events::State::Queued;
300	+	}
301	+
302	+	/// Full receivers retain terminal events while other receivers make progress.
303	+	@test unsafe fn saturation() throws (testing::TestError) {
304	+	initialize();
305	+	let receiver = abi::Ref { index: 0, generation: 1 };
306	+	for i in 0..events::ORDINARY {
307	+	assert try! events::send(&mut STORE.events, receiver, &mut FIRST.ring,
308	+	events::Event { kind: events::WAKEUP, reserved: 0, code: i, value: 0 });
309	+	}
310	+	for i in 64..128 {
311	+	retain(i, 0);
312	+	}
313	+	assert try! events::flush(&mut STORE.events, receiver, &mut FIRST.ring) == 64;
314	+	retain(2, 0);
315	+	retain(3, 1);
316	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 0;
317	+	assert STORE.events.terminals[2].state == events::State::Pending;
318	+	assert STORE.events.pending == 1;
319	+	assert STORE.events.terminals[3].state == events::State::Queued;
320	+	let consumed = try! events::pop(&mut FIRST.ring) else panic "full queue";
321	+	assert consumed.kind == events::WAKEUP;
322	+	assert lifecycle::service(&mut STORE, &WINDOWS, 20) == 2;
323	+	assert STORE.events.terminals[2].state == events::State::Queued;
324	+	assert STORE.events.queues[0].tail == events::CAPACITY + 1;
325	+	retain(4, 0);
326	+	assert STORE.events.pending == 1;
327	+	try! events::close(&mut STORE.events, receiver);
328	+	assert STORE.events.pending == 0;
329	+	}

kernel/kernel/tests/loader.rad +85 -0

use kernel::frames;
use kernel::backing;
use kernel::capability;
use kernel::registry;
use kernel::loader;
use kernel::domains;
use kernel::lifecycle;

/// Caller metadata used for interrupted loader calls.
unsafe static DOMAINS: domains::Store = undefined;
/// Caller-owned private reservations.
unsafe static CALLS: lifecycle::Calls = undefined;
/// Capability storage retained until the stopped caller is reclaimed.
unsafe static MEMORY: domains::Memory = undefined;

/// Mapped frames for temporary compiler storage and persistent native output.
static RAM: [u64; 18874368] = [0; 18874368];
/// Physical frame and page metadata.
unsafe static PAGES: pages::Store = undefined;

        }
    }
    return count;
}

/// Dead loaders retain private resources until hart release, then permit a retry.
@test unsafe fn cancelledCall() throws (testing::TestError) {
    for stage in 0..3 {
        initialize();
        domains::initialize(&mut DOMAINS);
        lifecycle::initialize(&mut CALLS);
        let request = input(43, false);
        let before = free();
        let owner = TABLE.owner;
        set MEMORY.table = TABLE;
        set DOMAINS.slots[0] = slots::Slot { generation: 1, state: slots::State::Live };
        set DOMAINS.records[0].state = domains::Lifecycle::Active;
        set DOMAINS.records[0].memory = &mut MEMORY;
        set DOMAINS.contextSlots[0] = slots::Slot { generation: 1, state: slots::State::Live };
        set DOMAINS.contexts[0].owner = owner;
        set DOMAINS.contexts[0].state = domains::ContextState::Ready;
        set DOMAINS.contexts[0].hart = nil;
        let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
        let lease = lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, pending);
        if stage > 0 {
            try! lifecycle::reserveOutput(&mut CALLS, &lease, &mut PAGES, &mut PACKAGES);
        }
        if stage == 2 {
            let retained = lifecycle::load(&CALLS, &lease);
            let decoded = try! loader::decodeInput(&retained.input);
            let output = retained.output else panic "missing output";
            let compiled = try! loader::compileInput(&PACKAGES, &retained.input, &decoded, &output);
            assert compiled.code > 0;
        }
        lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER, &mut PACKAGES);
        assert LOADER.busy and DOMAINS.contexts[0].reservation;
        set DOMAINS.records[0].state = domains::Lifecycle::Dead;
        set DOMAINS.contexts[0].state = domains::ContextState::Stopped;
        set DOMAINS.contexts[0].hart = 1;
        lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER, &mut PACKAGES);
        assert LOADER.busy and free() < before;
        match lease {
            case lifecycle::Lease::Held(context) => {
                assert context == owner;
            },
        }
        set DOMAINS.contexts[0].hart = nil;
        lifecycle::cancelAllocations(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER, &mut PACKAGES);
        assert not LOADER.busy and not DOMAINS.contexts[0].reservation;
        assert free() < before and PAGES.backings.pool.retiredCount > 0;
        while PAGES.backings.pool.retiredCount > 0 {
            let reclaimed = frames::reclaim(&mut PAGES.backings.pool, 256);
            assert reclaimed > 0 and reclaimed <= 256;
        }
        assert free() == before and MEMORY.table.slots[2].state == slots::State::Free;
        assert PACKAGES.slots[0].state == slots::State::Free and not LOADER.live[0];
        set DOMAINS.records[0].state = domains::Lifecycle::Active;
        let retry = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
        let active = lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, retry);
        try! lifecycle::reserveOutput(&mut CALLS, &active, &mut PAGES, &mut PACKAGES);
        let retained = lifecycle::load(&CALLS, &active);
        let decoded = try! loader::decodeInput(&retained.input);
        let output = retained.output else panic "missing retry output";
        let compiled = try! loader::compileInput(&PACKAGES, &retained.input, &decoded, &output);
        let handle = try! lifecycle::finishLoad(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER,
            &mut PACKAGES, active, lifecycle::LoadResult::Compiled(compiled));
        let object = try! registry::image(&PACKAGES, &MEMORY.table, handle, abi::Rights(0));
        assert object.index == 0 and not DOMAINS.contexts[0].reservation;
        let available = free();
        let duplicate = try! loader::reserve(&mut LOADER, &mut PAGES, &mut MEMORY.table, request);
        let reuse = lifecycle::holdLoad(&mut CALLS, &mut DOMAINS, owner, duplicate);
        let source = lifecycle::load(&CALLS, &reuse);
        let program = try! loader::decodeInput(&source.input);
        assert try! loader::identifyInput(&PACKAGES, &source.input, &program) == object;
        let same = try! lifecycle::finishLoad(&mut CALLS, &mut DOMAINS, &mut PAGES, &mut LOADER,
            &mut PACKAGES, reuse, lifecycle::LoadResult::Existing(object));
        assert try! registry::image(&PACKAGES, &MEMORY.table, same, abi::Rights(0)) == object;
        assert free() == available and not LOADER.busy;
    }
}

/// Reserved input stays private across allocation, source mutation, and contention.
@test unsafe fn stagedInput() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();

test/dispatch/kernel/dispatchcheck.rad +27 -3

use kernel::domains;
use kernel::budgets;
use kernel::dispatch;
use kernel::boot;
use kernel::events;
use kernel::sync;
use kernel::dispatchinput;

/// Bootstrap authority used to create fixture domains and CPU windows.
unsafe static TABLE: capability::Table = undefined;
/// Domain handles used for budget binding.

    let executable = try! registry::install(&registry::STORE, &mut user.memory.table, root);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let stackCap = try! capability::get(&TABLE, stack);
    let stackPage = try! pages::get(&pages::STORE, stackCap.object);
    let args = @sliceOf(memory(stackPage.base), 32);
    let parent = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: domains::STORE.contexts[CONTEXTS[2].index].owner, rights: abi::Rights(abi::WAKE),
    });
    let args = @sliceOf(memory(stackPage.base), 40);
    let words = [*granted, *executable, *selfAuthority];
    for word in 0..3 {
        for byte in 0..8 {
            set args[word * 8 + byte] = (words[word] >> (byte as u64 * 8)) as u8;
        }
    }
    try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, HANDLES[0], stackPage.base + 4096, stackPage.base, 32);
    for byte in 0..8 {
        set args[32 + byte] = (*parent >> (byte as u64 * 8)) as u8;
    }
    try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, HANDLES[0], stackPage.base + 4096, stackPage.base, 40);
    let imageRef = try! registry::image(&registry::STORE, &TABLE, image, abi::Rights(abi::EXECUTE));
    let target = try! registry::exported(&registry::STORE, imageRef, &"spin::count"[..]);
    let case shared::Target::Data(data) = target else panic "counter";
    let timer = try! dispatch::timer(&boot::PLATFORM, 0);
    let clock = dispatch::now(timer);
    let start = clock + 1000000;
    let span: u64 = 4000000;
    let span: u64 = 8000000;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.pc = worker();
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[10] = user.graph.table[data.slot] + data.offset as u64;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[11] = timer.clock;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[12] = start + 3 * span;
    set domains::STORE.contexts[CONTEXTS[1].index].frame.registers[13] = start + 4 * span;

    let checked = try! registry::exported(&registry::STORE, domain.image, &"spin::destroyed"[..]);
    let case shared::Target::Data(flag) = checked else panic "destruction flag";
    if *memory(domain.graph.table[flag.slot] + flag.offset as u64) <> 1 {
        failed(7);
    }
    let child = try! registry::exported(&registry::STORE, domain.image, &"spin::childId"[..]);
    let case shared::Target::Data(identity) = child else panic "child identity";
    let idBytes = @sliceOf(memory(domain.graph.table[identity.slot] + identity.offset as u64), 8);
    let mut childId: u64 = 0;
    for byte in 0..8 {
        set childId |= idBytes[byte] as u64 << (byte as u64 * 8);
    }
    let owner = domains::STORE.contexts[CONTEXTS[2].index].owner;
    let guard = sync::enter();
    let mut checker = try! domains::get(&domains::STORE, owner);
    assert try! events::flush(&mut domains::STORE.events, owner, &mut checker.memory.ring) == 0;
    assert not try! events::acknowledged(&domains::STORE.events, abi::reference(childId));
    let exited = try! events::pop(&mut checker.memory.ring) else panic "child exit";
    assert exited.kind == events::CHILD_EXIT and exited.code == 0 and exited.value == childId;
    try! events::refresh(&mut domains::STORE.events, owner, &checker.memory.ring);
    assert try! events::acknowledged(&domains::STORE.events, abi::reference(childId));
    sync::leave(guard);
    let wake = try! events::pop(&mut domain.memory.ring) else panic "wake token";
    assert wake.kind == events::WAKEUP and wake.code == 123;
    let timeout = try! events::pop(&mut domain.memory.ring) else panic "timeout token";
    assert timeout.kind == events::TIMEOUT and timeout.code == 77 and timeout.value == 0;
    assert try! events::pop(&mut domain.memory.ring) == nil;

test/dispatch/spin.rad +10 -5

    image: u64,
    /// Local self-Domain capability with allocation authority.
    domain: u64,
    /// Future window that the user can partition between its contexts.
    future: u64,
    /// Completion checker with authority to receive child lifecycle events.
    parent: u64,
}

/// Startup environment supplied by the kernel.
record Env: Copy {
    /// Readable startup capabilities.

export static count: u64 = 0;
/// Additional-context progress in the same private package state.
export static extraCount: u64 = 0;
/// Successful user-side return and destruction checks.
export static destroyed: u64 = 0;
/// Generation-bearing identity of the child that returns normally.
export static childId: u64 = 0;
/// Local self-Domain authority shared by both user contexts.
static SELF: abi::Handle = abi::Handle(0);

/// Update shared private state and return from an additional context.
export fn worker(args: *opaque, size: u64) {
    assert size == 0;
    set extraCount += 1;
    try! sys::timeout(1000000, 77);
    try! sys::timeout(2000000, 77);
    sys::wait();
    set extraCount += 1;
    set count += 1;
}
/// Check direct calls and handoff, then exercise independently bound contexts.

    if env.argsSize == 0 {
        try! sys::timeout(10000, 88);
        sys::wait();
        return;
    }
    assert env.argsSize == 32;
    assert env.argsSize == 40;
    set SELF = abi::Handle(env.argsPointer.domain);
    let handle = env.argsPointer.stack as i64;
    let info = sys::queryPage(abi::Handle(handle as u64));
    assert info.base <> 0 and info.count == 1;
    let mut rejected = false;

    let domain = sys::queryDomain(abi::Handle(env.argsPointer.domain));
    assert domain.id <> 0 and domain.state == 1;
    let image = try! sys::queryImage(abi::Handle(env.argsPointer.image));
    assert image.entry <> 0 and image.codeBase <> 0 and image.codeBytes > 0 and image.dataBytes > 0;
    let child = try! sys::domainCreate(abi::Handle(0), abi::Handle(env.argsPointer.image));
    set childId = sys::queryDomain(child).id;
    try! sys::domainReparent(child, abi::Handle(env.argsPointer.parent));
    let stack = try! sys::pageAllocate(abi::Handle(0), 1);
    let granted = try! sys::capabilityGrant(stack, child, (abi::READ | abi::WRITE) as u64);
    let childStack = sys::queryPage(stack);
    try! sys::domainActivate(child, childStack.base + 4096, (&*env) as *unsafe opaque, 0);
    assert sys::queryDomain(child).state == 1;

    assert rejected and sys::currentContext().budget == current.budget;
    set count = 1;
    try! sys::yield(child);
    let continued = sys::currentContext();
    assert continued.context == current.context and continued.budget == *future;
    assert sys::queryDomain(child).state == 2;
    while extraCount == 0 {
        set count += 1;
    }
    let waiting = try! sys::queryContext(SELF, extra);
    assert waiting.state == 2 and waiting.hart == 0xffffffffffffffff;
    try! sys::wakeup(SELF, 123);
    assert extraCount == 1;
    assert extraCount == 1 or extraCount == 2;
    while extraCount == 1 {
        set count += 1;
    }
    let stopped = try! sys::queryContext(abi::Handle(env.argsPointer.domain), extra);
    assert stopped.state == 1 and stopped.hart == 0xffffffffffffffff;

    set rejected = false;
    try sys::queryContext(abi::Handle(env.argsPointer.domain), extra) catch error {
        assert error == abi::Error::BadHandle;
        set rejected = true;
    };
    assert rejected and sys::queryDomain(child).state == 2;
    assert rejected;
    set destroyed = 1;
    while true {
        set count += 1;
    }
}

test/modules/run added +21 -0

1	+	#!/bin/sh
2	+	# Resolve and execute references beyond module index 127.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	work=$(mktemp -d)
6	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
7	+	mkdir "$work/capacity"
8	+	set -- -pkg capacity -mod "$work/capacity.rad"
9	+	index=0
10	+	while [ "$index" -lt 127 ]; do
11	+	printf 'export mod m%s;\n' "$index" >> "$work/capacity.rad"
12	+	printf '/// Return this module index.\nexport fn value() -> u64 { return %s; }\n' "$index" > "$work/capacity/m$index.rad"
13	+	set -- "$@" -mod "$work/capacity/m$index.rad"
14	+	index=$((index + 1))
15	+	done
16	+	printf 'use capacity::m126;\n/// Check a package root beyond module index 127.\n@default fn main() -> u64 { assert m126::value() == 126; return 0; }\n' > "$work/entry.rad"
17	+	set -- "$@" -pkg entry -mod "$work/entry.rad"
18	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
19	+	-run bin/radiance.rv64.dev "$@" -entry entry -o "$work/capacity.rv64"
20	+	"$emulator" -run "$work/capacity.rv64"
21	+	printf 'module scopes: references beyond index 127 passed\n'

test/smp/kernel/dispatchcheck.rad +235 -5

use kernel::calls;
use kernel::events;
use kernel::timers;
use kernel::interrupts;
use kernel::plic;
use kernel::lifecycle;
use kernel::dispatchinput;

/// Bootstrap resource authority.
unsafe static TABLE: capability::Table = undefined;
/// Release/acquire barrier after fixture publication.
static READY: u64 = 0;
/// Shared user domain generation.
unsafe static USER: abi::Ref = undefined;
/// Pending domain in the user's creation subtree.
unsafe static DESCENDANT: abi::Ref = undefined;
/// Per-hart user execution contexts.
unsafe static CONTEXTS: [abi::Ref; 8] = undefined;
/// Mapped progress counters in the user's private state.
static COUNTERS: u64 = 0;
/// Per-context migration observations in the user's private state.

static IDLE_STACK: u64 = 0;
/// Generation of the idle-wakeup domain.
unsafe static IDLE_DOMAIN: abi::Ref = undefined;
/// Completion checker's live domain generation.
unsafe static CHECKER: abi::Ref = undefined;
/// Unbound window that expires before the checker starts.
unsafe static EXPIRED: abi::Handle = undefined;
/// Checker's authority to activate and notify the idle-wakeup domain.
unsafe static RECEIVER: abi::Handle = undefined;
/// Unbound authority reserved for the idle-wakeup check on hart one.
unsafe static IDLE_WINDOW: abi::Handle = undefined;
/// Validated physical memory mapping.

    let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
        loader::Request { authority, source, offset: 0, length: length as u64 });
    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let cap = try! capability::get(&TABLE, handle);
    set USER = cap.object;
    createDescendant();
    let mut user = try! domains::get(&domains::STORE, USER);
    let self = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE | abi::WAKE),
    });
    assert boot::PLATFORM.irqSources == 31;

                &TABLE, handle, abi::ContextStart { entry, stack: page.base + 4096, args: page.base, size: 8 });
        }
    }
    let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
    let checkerCap = try! capability::get(&TABLE, checkerHandle);
    let checker = try! domains::get(&domains::STORE, checkerCap.object);
    let mut checker = try! domains::get(&domains::STORE, checkerCap.object);
    set CHECKER = checkerCap.object;
    set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;
    set domains::STORE.contexts[checker.initial.index].frame.pc = completion();
    set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;
    set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;

    let windowBase = dataAddress(&user, &"spin::windows"[..]);
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) == 0 {
            continue;
        }
        let budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);
        let begin = clock if hart == 0 else start;
        let mut budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, begin, 0xffffffffffffffff);
        if hart == 0 {
            let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, start, clock);
            set EXPIRED = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut checker.memory.table,
                budget, budgets::DEFAULT_RIGHTS as u64, clock);
            set budget = next;
        }
        let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);
        let checking = try! budgets::split(&mut budgets::STORE, &mut TABLE, final, migrationEnd, clock);
        let future = try! budgets::split(&mut budgets::STORE, &mut TABLE, checking, migrationEnd + 10000000000, clock);
        let transferred = try! budgets::transfer(&mut budgets::STORE, &mut TABLE, &mut user.memory.table,
            future, budgets::DEFAULT_RIGHTS as u64, clock);

            set IDLE_WINDOW = checking;
        }
    }
}

/// Create an actual pending descendant before secondary harts enter dispatch.
unsafe fn createDescendant() {
    let mut user = try! domains::get(&domains::STORE, USER);
    let root = user.image;
    let authority = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::CREATE),
    });
    let image = try! registry::install(&registry::STORE, &mut user.memory.table, root);
    let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
        &registry::STORE, &mut user.memory.table, authority, image);
    set DESCENDANT = (try! capability::get(&user.memory.table, handle)).object;
}

/// Resolve one exported private-data address in the fixture's user instance.
unsafe fn dataAddress(user: &domains::Domain, name: &[u8]) -> u64 {
    let target = try! registry::exported(&registry::STORE, user.image, name);
    let case shared::Target::Data(data) = target else panic "fixture data export";
    return user.graph.table[data.slot] + data.offset as u64;

    set IDLE_HANDLE = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
        &registry::STORE, &mut TABLE, authority, image);
    let permit = try! capability::get(&TABLE, IDLE_HANDLE);
    set IDLE_DOMAIN = permit.object;
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let allocation = try! capability::install(&mut domain.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::ALLOCATE),
    });
    set IDLE_CONTEXT = domain.initial;
    set WAKE_STATE = dataAddress(&domain, &"spin::wakeState"[..]);
    sync::storeRelease(memory(dataAddress(&domain, &"spin::idle"[..])) as *mut u64, 1);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut domain.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let permission = try! capability::get(&TABLE, stack);
    let page = try! pages::get(&pages::STORE, permission.object);
    set IDLE_STACK = page.base + 4096;
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    set RECEIVER = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE),
        kind: abi::Kind::Domain, object: IDLE_DOMAIN, rights: abi::Rights(abi::WAKE | abi::EXECUTE | abi::DESTROY),
    });
}

/// Check shared progress and context isolation after the user windows finish.
export unsafe fn verify() {
    let guard = sync::enter();
    let user = try! domains::get(&domains::STORE, USER);
    let mut counts: [u64; 8] = [0; 8];
    let mut notifications: u32 = 0;
    for hart in 0..8 {
        if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
            set notifications += 1;

            continue;
        }
        let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);
        set counts[hart] = value;
        let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);
        assert context.owner == USER and (context.frame.status & 0x1800) == 0;
        assert context.owner == USER;
        let mode = context.frame.status & 0x1800;
        assert mode == 0 or mode == 0x1800;
        if mode == 0x1800 {
            assert context.frame.registers[3] == kernelGp();
            assert context.frame.registers[2] >= context.kernelStack.start and context.frame.registers[2] <= context.kernelStack.end;
        } else {
            assert context.frame.registers[3] == user.graph.table.ptr as u64;
        }
        let destination = (hart + notifications - 1) % notifications;
        assert context.hart == nil or context.hart == destination;
        assert sync::loadAcquire(memory(MIGRATIONS + hart as u64 * 8) as *u64) == destination as u64 + 1;
        assert context.pending == nil;
    }
    assert domains::STORE.events.queues[USER.index].tail == notifications + 1;
    let user = try! domains::get(&domains::STORE, USER);
    assert user.memory.ring.data[0].kind == events::INTERRUPT and user.memory.ring.data[0].code == 7;
    sync::leave(guard);
    for byte in "smp completed transactions:" {
        put(byte);
    }

        }
    }
    for byte in "smp request acknowledgements passed\n" {
        put(byte);
    }
    expiredCall();
    if (boot::PLATFORM.harts & 2) <> 0 {
        idleWakeup();
    }
    interruptOwnership();
    interruptStorm();
    if (boot::PLATFORM.harts & 2) <> 0 {
        terminalWakeup();
        cancelAllocation();
        reclaimedDomains();
    }
    let maximum = sync::maximum();
    assert maximum > 0 and maximum < 1000000;
    for byte in "smp metadata instructions: 0x" {
        put(byte);
    }
    number(maximum);
    put(10);
    finish();
}

/// Validate elapsed authority against the current clock at the serialized boundary.
unsafe fn expiredCall() {
    let guard = sync::enter();
    let checker = try! domains::get(&domains::STORE, CHECKER);
    let before = try! budgets::query(&budgets::STORE, &checker.memory.table, EXPIRED);
    assert before.end < dispatch::clock();
    sync::leave(guard);
    let mut rejected = false;
    try calls::synchronized(CHECKER, 70, &[*EXPIRED, before.start + (before.end - before.start) / 2, 0, 0], 0) catch error {
        assert error == abi::Error::Busy; set rejected = true;
    };
    assert rejected;
    let checking = sync::enter();
    let after = try! budgets::query(&budgets::STORE, &checker.memory.table, EXPIRED);
    assert after.start == before.start and after.end == before.end;
    sync::leave(checking);
    for byte in "smp resumed call rejected expired authority\n" {
        put(byte);
    }
}

/// Transfer a queued IRQ, rearm after consumption, and mask it before capability drop.
unsafe fn interruptOwnership() {
    let guard = sync::enter();
    let mut user = try! domains::get(&domains::STORE, USER);
    let mut checker = try! domains::get(&domains::STORE, CHECKER);

    for byte in "smp remote timeout passed\n" {
        put(byte);
    }
}

/// Wake a parent waiting on hart one through terminal delivery on hart zero.
unsafe fn terminalWakeup() {
    let guard = sync::enter();
    let mut receiver = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let timeout = try! events::pop(&mut receiver.memory.ring) else panic "missing idle timeout";
    assert timeout.kind == events::TIMEOUT and timeout.code == 0x66;
    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 7);
    sync::leave(guard);
    assert waiting();
    let stopping = sync::enter();
    let mut checker = try! domains::get(&domains::STORE, CHECKER);
    let child = try! capability::install(&mut checker.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::DESTROY),
    });
    let parent = try! calls::invoke(CHECKER, 23, &[*child, *RECEIVER, 0, 0], 0);
    assert parent == 0;
    let cancelled = try! dispatch::request(1, remote::Action::Budget(remote::Call {
        owner: USER, context: CONTEXTS[0], operation: 70, arguments: [0; 4],
    }));
    set domains::STORE.contexts[CONTEXTS[0].index].pending = cancelled;
    let readiness = try! remote::reserve(&mut remote::STORE, 0, 2);
    let batch = remote::publish(&mut remote::STORE, readiness, remote::Action::Ready(CONTEXTS[1]));
    dispatch::startReady(CONTEXTS[1], &batch);
    let stopped = try! domains::terminateTree(&mut domains::STORE, USER, events::CHILD_EXIT, 23);
    assert stopped == 0;
    assert domains::STORE.records[DESCENDANT.index].state == domains::Lifecycle::Dead;
    assert try! events::acknowledged(&domains::STORE.events, DESCENDANT);
    let request = try! dispatch::request(0, remote::Action::Reschedule);
    sync::leave(stopping);
    let acknowledged = try! dispatch::awaitRequest(0, request);
    assert acknowledged == 0;
    let mut resumed = false;
    for attempt in 0..100000 {
        if sync::loadAcquire(memory(WAKE_STATE) as *u64) == 9 {
            set resumed = true;
            break;
        }
    }
    assert resumed;
    let checking = sync::enter();
    assert remote::STORE.slots[cancelled.index].state == slots::State::Free;
    assert remote::STORE.slots[batch.ids[1].index].state == slots::State::Free;
    assert domains::STORE.contexts[CONTEXTS[0].index].pending == nil;
    assert domains::STORE.contexts[CONTEXTS[1].index].pending == nil;
    let mut domain = try! domains::get(&domains::STORE, IDLE_DOMAIN);
    let event = try! events::pop(&mut domain.memory.ring) else panic "missing remote exit";
    assert event.kind == events::CHILD_EXIT and event.code == 23 and event.value == abi::id(USER);
    sync::leave(checking);
    for byte in "smp cascade cancelled calls and woke remote parent\n" {
        put(byte);
    }
}

/// Stop a real page-clear continuation and recover its unpublished frames.
unsafe fn cancelAllocation() {
    sync::storeRelease(memory(WAKE_STATE) as *mut u64, 10);
    let mut allocation: pages::Allocation = undefined;
    let mut request: ?abi::Ref = nil;
    for attempt in 0..100000 {
        let guard = sync::enter();
        let index = IDLE_CONTEXT.index;
        if (lifecycle::CALLS.active[index / 32] & (1 << (index % 32))) <> 0 {
            match lifecycle::CALLS.records[index].allocation {
                case lifecycle::Allocation::Page(value) => {
                    set allocation = value;
                },
                else => panic "page allocation required",
            }
            assert allocation.owner == IDLE_DOMAIN and allocation.run.count == 4096;
            sync::leave(guard);
            let result = try! calls::synchronized(CHECKER, 22, &[*RECEIVER, 0, 0, 0], dispatch::clock());
            assert result == 0;
            let checking = sync::enter();
            assert domains::STORE.records[IDLE_DOMAIN.index].state == domains::Lifecycle::Dead;
            set request = try! dispatch::request(1, remote::Action::Reschedule);
            sync::leave(checking);
            break;
        }
        sync::leave(guard);
    }
    let stopped = request else panic "page call did not reserve frames";
    let acknowledged = try! dispatch::awaitRequest(0, stopped);
    assert acknowledged == 0;
    let snapshot = sync::enter();
    let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);
    for byte in "smp cancelled page status and progress: " {
        put(byte);
    }
    number(context.frame.status); put(32);
    number(sync::loadAcquire(memory(WAKE_STATE) as *u64)); put(10);
    assert context.hart == nil and context.state == domains::ContextState::Stopped;
    assert (context.frame.status & 0x1800) == 0x1800;
    assert sync::loadAcquire(memory(WAKE_STATE) as *u64) == 11;
    sync::leave(snapshot);
    let mut reclaimed = false;
    for attempt in 0..1000 {
        let checking = sync::enter();
        set reclaimed = pages::STORE.backings.pool.retiredCount == 0;
        if reclaimed {
            sync::leave(checking);
            break;
        }
        let progress = try! dispatch::request(0, remote::Action::Reschedule);
        sync::leave(checking);
        let result = try! dispatch::awaitRequest(0, progress);
        assert result == 0;
    }
    assert reclaimed;
    let guard = sync::enter();
    assert (lifecycle::CALLS.active[IDLE_CONTEXT.index / 32] & (1 << (IDLE_CONTEXT.index % 32))) == 0;
    assert pages::STORE.slots[allocation.page.index].state == slots::State::Free;
    assert pages::STORE.backings.slots[allocation.backing.index].state == slots::State::Free;
    for i in allocation.run.first..allocation.run.first + allocation.run.count {
        assert pages::STORE.backings.pool.free[i];
    }
    sync::leave(guard);
    for byte in "smp cancelled preempted page allocation\n" {
        put(byte);
    }
}

/// Wait for dispatch-driven teardown and verify all dead execution ownership is gone.
unsafe fn reclaimedDomains() {
    for attempt in 0..100000 {
        let guard = sync::enter();
        let mut done = pages::STORE.backings.pool.retiredCount == 0;
        for owner in [USER, IDLE_DOMAIN, DESCENDANT] {
            if slots::matches(&domains::STORE.slots[..], owner, slots::State::Live) {
                set done = false;
            }
        }
        if done {
            assert domains::STORE.dead == 0;
            for owner in [USER, IDLE_DOMAIN, DESCENDANT] {
                assert domains::STORE.slots[owner.index].state == slots::State::Free;
                assert pages::STORE.backings.domains[owner.index] == 0;
                for i in 0..domains::STORE.contextSlots.len {
                    if domains::STORE.contextSlots[i].state == slots::State::Live {
                        assert domains::STORE.contexts[i].owner <> owner;
                    }
                }
                for i in 0..budgets::STORE.slots.len {
                    if budgets::STORE.slots[i].state == slots::State::Live {
                        assert budgets::STORE.windows[i].owner <> owner;
                    }
                }
                for i in 0..interrupts::STORE.slots.len {
                    if interrupts::STORE.slots[i].state == slots::State::Live {
                        assert interrupts::STORE.records[i].owner <> owner;
                    }
                }
            }
            sync::leave(guard);
            for byte in "smp reaped dead domains and recovered ownership\n" {
                put(byte);
            }
            return;
        }
        let progress = try! dispatch::request(0, remote::Action::Reschedule);
        sync::leave(guard);
        let result = try! dispatch::awaitRequest(0, progress);
        assert result == 0;
    }
    panic "domain reaper did not finish";
}

/// Confirm the waiting context has released hart ownership before its wakeup.
unsafe fn waiting() -> bool {
    for attempt in 0..100000 {
        let guard = sync::enter();
        let context = try! domains::context(&domains::STORE, IDLE_DOMAIN, IDLE_CONTEXT);

test/smp/spin.rad +11 -1

export static interrupt: u64 = 0;

/// Observed destination hart plus one after each context changes budget.
export static migrations: [u64; 8] = [0; 8];

/// Wait handshake: 2/3 bracket wakeup; 4 requests a second wait, bracketed by 5/6.
/// Progress shared with the machine fixture for waits and interrupted allocation.
export static wakeState: u64 = 0;

/// Nonzero selects the independently activated idle-wakeup context.
export static idle: u64 = 0;


        while wakeState <> 4 {
        }
        set wakeState = 5;
        sys::wait();
        set wakeState = 6;
        while wakeState <> 7 {
        }
        set wakeState = 8;
        sys::wait();
        set wakeState = 9;
        while wakeState <> 10 {
        }
        set wakeState = 11;
        let allocation = try! sys::pageAllocate(abi::Handle(0), 4096);
        set wakeState = 12;
        while true {
        }
    }
    run(0);
}

test/termination/kernel/dispatchcheck.rad added +105 -0

1	+	//! Native exit and fault reporting to a surviving parent.
2	+	use std::mem;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::backing;
6	+	use kernel::pages;
7	+	use kernel::capability;
8	+	use kernel::registry;
9	+	use kernel::loader;
10	+	use kernel::domains;
11	+	use kernel::budgets;
12	+	use kernel::dispatch;
13	+	use kernel::boot;
14	+	use kernel::events;
15	+	use kernel::sync;
16	+	use kernel::dispatchinput;
17	+
18	+	/// Bootstrap creation, allocation, and budget authority.
19	+	unsafe static TABLE: capability::Table = undefined;
20	+	/// Child domain identities retained by terminal events.
21	+	static CHILDREN: [abi::Ref; 5] = undefined;
22	+	/// Surviving event receiver and native completion checker.
23	+	unsafe static PARENT: abi::Ref = undefined;
24	+	/// Map a validated physical page.
25	+	fn memory(address: u64) -> *mut u8;
26	+	/// Current kernel package-state base.
27	+	fn kernelGp() -> u64;
28	+	/// Native entry for the completion checker.
29	+	fn completion() -> u64;
30	+	/// Signal successful completion to the machine fixture.
31	+	fn finish();
32	+
33	+	/// Give four children disjoint windows, followed by the surviving parent.
34	+	export unsafe fn setup() {
35	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
36	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
37	+	capability::initialize(&mut TABLE, owner);
38	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
39	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
40	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
41	+	});
42	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
43	+	let page = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, source)).object);
44	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
45	+	let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
46	+	let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
47	+	loader::Request { authority, source, offset: 0, length: length as u64 });
48	+	let parentHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
49	+	set PARENT = (try! capability::get(&TABLE, parentHandle)).object;
50	+	let parent = try! domains::get(&domains::STORE, PARENT);
51	+	set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;
52	+	set domains::STORE.contexts[parent.initial.index].frame.pc = completion();
53	+	set domains::STORE.contexts[parent.initial.index].frame.status = 0x1880;
54	+	set domains::STORE.contexts[parent.initial.index].frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;
55	+	set domains::STORE.contexts[parent.initial.index].frame.registers[3] = kernelGp();
56	+	let mut handles: [abi::Handle; 5] = undefined;
57	+	let mut contexts: [abi::Ref; 5] = undefined;
58	+	for i in 0..5 {
59	+	let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
60	+	let object = (try! capability::get(&TABLE, handle)).object;
61	+	let mut child = try! domains::get(&domains::STORE, object);
62	+	set CHILDREN[i] = object;
63	+	set handles[i] = handle;
64	+	set contexts[i] = child.initial;
65	+	try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);
66	+	let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
67	+	let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
68	+	let storage = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, stack)).object);
69	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, i as u64);
70	+	}
71	+	let timer = try! dispatch::timer(&boot::PLATFORM, 0);
72	+	let clock = dispatch::now(timer);
73	+	let start = clock + 1000000;
74	+	let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 12000000);
75	+	for i in 0..5 {
76	+	let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, remaining, start + (i as u64 + 1) * 2000000, clock);
77	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
78	+	budgets::Binding { budget: remaining, domain: handles[i], context: contexts[i] }, clock);
79	+	set remaining = next;
80	+	}
81	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
82	+	budgets::Binding { budget: remaining, domain: parentHandle, context: parent.initial }, clock);
83	+	}
84	+
85	+	/// Check each terminal kind, status, and generation while the parent continues to run.
86	+	export unsafe fn verify() {
87	+	let guard = sync::enter();
88	+	let mut parent = try! domains::get(&domains::STORE, PARENT);
89	+	let codes = [37 as u32, abi::FAULT_ABORT, 3, 0, abi::FAULT_ABORT];
90	+	for i in 0..5 {
91	+	let event = try! events::pop(&mut parent.memory.ring) else panic "missing termination event";
92	+	assert event.value == abi::id(CHILDREN[i]) and event.code == codes[i];
93	+	if i == 0 or i == 3 {
94	+	assert event.kind == events::CHILD_EXIT;
95	+	}
96	+	else {
97	+	assert event.kind == events::FAULT;
98	+	}
99	+	assert domains::STORE.records[CHILDREN[i].index].state == domains::Lifecycle::Dead;
100	+	}
101	+	try! events::refresh(&mut domains::STORE.events, PARENT, &parent.memory.ring);
102	+	assert try! events::pop(&mut parent.memory.ring) == nil;
103	+	sync::leave(guard);
104	+	finish();
105	+	}

test/termination/machine.ras added +25 -0

1	+	//! Boot one hart and check user termination through production dispatch.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::setup;
5	+	call @kernel::boot::run;
6	+	.export @kernel::dispatchcheck::memory;
7	+	.export @kernel::dispatchcheck::kernelGp;
8	+	.export @kernel::dispatchcheck::completion;
9	+	.export @kernel::dispatchcheck::finish;
10	+	@kernel::dispatchcheck::memory
11	+	ret;
12	+	@kernel::dispatchcheck::kernelGp
13	+	mv %a0 %gp;
14	+	ret;
15	+	@kernel::dispatchcheck::completion
16	+	la %a0 @complete;
17	+	ret;
18	+	@complete
19	+	call @kernel::dispatchcheck::verify;
20	+	ebreak;
21	+	@kernel::dispatchcheck::finish
22	+	li %t0 0x10001000;
23	+	li %t1 0x5555;
24	+	sw %t1 0(%t0);
25	+	ebreak;

test/termination/spin.rad added +36 -0

1	+	//! User termination and CPU-fault entry paths.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Issue a raw call to test the kernel error boundary.
6	+	@intrinsic fn ecall(operation: u32, a0: i64, a1: i64, a2: i64, a3: i64) -> i64;
7	+
8	+	/// Startup values used to select the termination operation.
9	+	record Env: Copy {
10	+	/// Optional argument bytes supplied by the parent.
11	+	argsPointer: *u8,
12	+	/// Termination case selected by the parent.
13	+	argsSize: u64,
14	+	/// Installed event handle.
15	+	eventsHandle: u64,
16	+	/// Shared event-ring address.
17	+	eventsPointer: u64,
18	+	}
19	+
20	+	/// Exercise an explicit exit, explicit abort, or a user breakpoint fault.
21	+	@default fn main(env: *Env) {
22	+	if env.argsSize == 0 {
23	+	sys::exit(37);
24	+	}
25	+	if env.argsSize == 1 {
26	+	sys::abort();
27	+	}
28	+	if env.argsSize == 3 {
29	+	try! sys::domainDestroy(abi::Handle(0), 0);
30	+	}
31	+	if env.argsSize == 4 {
32	+	let result = ecall(47, 0, 0, 0, 0);
33	+	sys::exit(99);
34	+	}
35	+	assert false;
36	+	}

558	664		{
559	665		assert trap::fromUser(frame) and frame.cause == 8;
560	666		assert trap::advanceCall(frame);
561	667		let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
562	668		let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
	669	+	let operation = frame.registers[17];
	670	+	if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
	671	+	try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
	672	+	assert stopped == abi::Error::BadHandle;
	673	+	};
	674	+	}
563	675		set frame.registers[10] = (0 as u64) - error as u64;
564	676		return;
565	677		};
566	678		set frame.registers[10] = result;
567	679		}

573	790		// MPIE enables interrupts after the first return into U-mode.
574	791		let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
575	792		set frame.registers[1] = returnAddress();
576	793		set frame.registers[3] = graph.table.ptr as u64;
577	794		set store.contexts[initial.index] = Context {
578		-	owner: object, state: ContextState::Ready, hart: nil, pending: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
	795	+	owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
579	796		userStack: range::Range { start: 0, end: 0 }, frame,
580	797		};
	798	+	set store.reclaimed[object.index] = false;
	799	+	set store.reaping[object.index] = false;
581	800		set store.records[object.index] = Domain {
582	801		state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
583	802		allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
584	803		};
	804	+	set store.ancestors[object.index] = store.ancestors[creation.owner.index];
	805	+	set store.ancestors[object.index][creation.owner.index / 32] \|= 1 << (creation.owner.index % 32);
585	806		let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
586	807		let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
587	808		return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
588	809		kind: abi::Kind::Domain, object: domain, rights: creation.rights,
589	810		});

20	20		export mod backing;
21	21		export mod pages;
22	22		export mod loader;
23	23		export mod instances;
24	24		export mod domains;
	25	+	export mod lifecycle;
25	26		export mod budgets;
26	27		export mod dispatch;
27	28		export mod timers;
28	29		export mod remote;
29	30		export mod calls;

133	133		export constant DESTROY: u16 = 256;
134	134		/// All defined right bits.
135	135		export constant RIGHTS_MASK: u16 = 511;
136	136		/// Cascade destruction through creation ancestry.
137	137		export constant CASCADE: u64 = 1;
	138	+	/// Fault code for explicit user Abort, separate from CPU exception codes.
	139	+	export constant FAULT_ABORT: u32 = 256;
138	140
139	141		/// Object kind stored in the high four handle bits.
140	142		export union Kind: Copy {
141	143		/// Unoccupied table entry.
142	144		Empty = 0,

27	27		pool: frames::Pool,
28	28		/// Backing-record lifetimes.
29	29		slots: [slots::Slot; limits::ALLOCATIONS],
30	30		/// Payloads for live backing records.
31	31		records: [Allocation; limits::ALLOCATIONS],
	32	+	/// Exclusive upper bound of backing slots that have been published.
	33	+	used: u32,
32	34		/// Live domain generation, or zero after all of that domain's exposures end.
33	35		domains: [u32; limits::DOMAINS],
	36	+	/// Zero while open; otherwise one plus the next exposure slot to inspect.
	37	+	ending: [u32; limits::DOMAINS],
34	38		}
35	39
36	40		/// Initialize fresh backing and exposure-domain tables.
37	41		export fn initialize(store: &mut Store) {
	42	+	set store.used = 0;
38	43		set store.pool.count = 0;
	44	+	set store.pool.retiredHead = 0;
	45	+	set store.pool.retiredCount = 0;
39	46		slots::initialize(&mut store.slots[..]);
40	47		for i in 0..limits::DOMAINS {
41	48		set store.domains[i] = 0;
	49	+	set store.ending[i] = 0;
42	50		}
43	51		}
44	52
45	53		/// Register a domain before it can receive allocation exposure.
46	54		export fn registerDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {

53	61		set store.domains[domain.index] = domain.generation;
54	62		}
55	63
56	64		/// Test exposure-domain generation without reading beyond the fixed table.
57	65		export fn domainLive(store: &Store, domain: abi::Ref) -> bool {
58		-	return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation;
	66	+	return domain.index < limits::DOMAINS and domain.generation <> 0 and store.domains[domain.index] == domain.generation and store.ending[domain.index] == 0;
59	67		}
60	68
61	69		/// Publish reserved metadata with one page object and its live origin exposure.
62	70		/// The caller owns the committed run and has validated the origin domain.
63	71		export unsafe fn publish(store: &mut Store, reservation: slots::Reservation, run: frames::Run, origin: abi::Ref) -> abi::Ref {
64	72		assert domainLive(store, origin);
65	73		let object = slots::reference(&reservation);
66	74		assert slots::matches(&store.slots[..], object, slots::State::Reserved);
	75	+	if object.index >= store.used {
	76	+	set store.used = object.index + 1;
	77	+	}
67	78		set store.records[object.index].run = run;
68	79		set store.records[object.index].pages = 1;
69	80		set store.records[object.index].exposed = 1;
70	81		let mut bits = bitset::init(&mut store.records[object.index].exposures[..]);
71	82		bitset::put(&mut bits, origin.index);

104	115		bitset::put(&mut bits, domain.index);
105	116		set store.records[object.index].exposed += 1;
106	117		}
107	118
108	119		/// Reclaim only when no page object and no domain exposure remains.
109		-	fn reclaim(store: &mut Store, object: abi::Ref) {
	120	+	fn reclaim(store: &mut Store, object: abi::Ref, release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) {
110	121		if store.records[object.index].pages <> 0 or store.records[object.index].exposed <> 0 {
111	122		return;
112	123		}
113	124		let run = store.records[object.index].run;
114		-	try! frames::release(&mut store.pool, run);
	125	+	try! release(&mut store.pool, run);
115	126		try! slots::release(&mut store.slots[..], object);
116	127		}
117	128
118	129		/// Release one page object after its final handle disappears.
119	130		export fn releasePage(store: &mut Store, object: abi::Ref) throws (abi::Error) {
120	131		try require(store, object);
121	132		if store.records[object.index].pages == 0 {
122	133		throw abi::Error::BadHandle;
123	134		}
124	135		set store.records[object.index].pages -= 1;
125		-	reclaim(store, object);
	136	+	reclaim(store, object, frames::release);
126	137		}
127	138
128		-	/// End all exposure for one domain before its slot can be registered again.
129		-	/// Callers serialize this bounded scan with allocation and capability changes.
	139	+	/// End all exposure while the caller has exclusive access to metadata.
130	140		export unsafe fn endDomain(store: &mut Store, domain: abi::Ref) throws (abi::Error) {
131		-	if not domainLive(store, domain) {
132		-	throw abi::Error::BadHandle;
	141	+	assert try end(store, domain, limits::ALLOCATIONS, frames::release);
	142	+	}
	143	+
	144	+	/// Inspect at most limit backing slots and retire allocations whose lifetimes end.
	145	+	/// Returns true after all exposure is clear and the domain slot can be registered again.
	146	+	export unsafe fn endDomainStep(store: &mut Store, domain: abi::Ref, limit: u32) -> bool throws (abi::Error) {
	147	+	return try end(store, domain, limit, frames::retire);
	148	+	}
	149	+
	150	+	/// Close exposure admission and advance one generation's retained scan cursor.
	151	+	unsafe fn end(store: &mut Store, domain: abi::Ref, limit: u32,
	152	+	release: fn(&mut frames::Pool, frames::Run) throws (abi::Error)) -> bool throws (abi::Error)
	153	+	{
	154	+	if domain.index >= limits::DOMAINS or domain.generation == 0
	155	+	or store.domains[domain.index] <> domain.generation {
	156	+	throw abi::Error::BadHandle;
	157	+	}
	158	+	if limit == 0 {
	159	+	throw abi::Error::InvalidArg;
	160	+	}
	161	+	if store.ending[domain.index] == 0 {
	162	+	set store.ending[domain.index] = 1;
133	163		}
134		-	for i in 0..limits::ALLOCATIONS {
	164	+	let first = store.ending[domain.index] - 1;
	165	+	let mut count = store.used - first;
	166	+	if count > limit {
	167	+	set count = limit;
	168	+	}
	169	+	let next = first + count;
	170	+	for i in first..next {
135	171		if store.slots[i].state <> slots::State::Live {
136	172		continue;
137	173		}
138	174		let mut bits = bitset::new(&mut store.records[i].exposures[..]);
139	175		if not bitset::contains(&bits, domain.index) {
140	176		continue;
141	177		}
142	178		bitset::clear(&mut bits, domain.index);
143	179		set store.records[i].exposed -= 1;
144		-	reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation });
	180	+	reclaim(store, abi::Ref { index: i, generation: store.slots[i].generation }, release);
	181	+	}
	182	+	if next < store.used {
	183	+	set store.ending[domain.index] = next + 1;
	184	+	return false;
145	185		}
146	186		set store.domains[domain.index] = 0;
	187	+	set store.ending[domain.index] = 0;
	188	+	return true;
147	189		}

7	7		use super::sync;
8	8		use super::trap;
9	9		use super::pages;
10	10		use super::registry;
11	11		use super::domains;
	12	+	use super::lifecycle;
12	13		use super::budgets;
13	14		use super::dispatch;
14	15		use super::timers;
15	16		use super::remote;
16	17		use super::interrupts;

24	25		/// Number of harts that validated their machine stack.
25	26		static ARRIVED: u64 = 0;
26	27		/// Trap anchors owned by their physical harts.
27	28		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
28	29
29		-	/// Handle timer boundaries and user calls, and stop on unsupported traps.
	30	+	/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
30	31		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
31	32		let cause = trap::classify(frame.cause);
32	33		if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
33	34		dispatch::interrupt(frame, hart);
34		-	return;
35	35		}
36	36		if cause == trap::Cause::UserCall and trap::fromUser(frame) {
37	37		dispatch::call(frame, hart);
38		-	return;
	38	+	}
	39	+	if trap::fromUser(frame) {
	40	+	dispatch::fault(frame, hart);
39	41		}
40	42		print("kernel: unexpected trap\n");
41	43		trap::halt();
42	44		}
43	45

85	87		let treeRange = range::new(treeAddress, size as u64) else panic "FDT range";
86	88		assert platform::inRam(&PLATFORM, treeRange);
87	89		try! platform::protect(&mut PLATFORM, treeRange);
88	90		try! pages::initialize(&mut pages::STORE, &PLATFORM);
89	91		domains::initialize(&mut domains::STORE);
	92	+	lifecycle::initialize(&mut lifecycle::CALLS);
90	93		budgets::initialize(&mut budgets::STORE);
91	94		timers::initialize(&mut timers::STORE);
92	95		remote::initialize(&mut remote::STORE, PLATFORM.harts);
93	96		interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
94	97		try! plic::initialize(&PLATFORM);

270	270		if window.hart as u64 <> current.hart or window.context <> abi::reference(current.context)
271	271		or now < window.start or now >= window.end {
272	272		throw abi::Error::Busy;
273	273		}
274	274		let permit = try capability::lookup(table, target, abi::Kind::Domain, abi::Rights(abi::WAKE));
275		-	let receiver = try domains::get(domainStore, permit.object) catch {
276		-	throw abi::Error::Busy;
277		-	};
	275	+	let receiver = try domains::get(domainStore, permit.object);
	276	+	if receiver.state == domains::Lifecycle::Dead {
	277	+	throw abi::Error::BadHandle;
	278	+	}
278	279		if receiver.state <> domains::Lifecycle::Active or abi::id(receiver.initial) == current.context {
279	280		throw abi::Error::Busy;
280	281		}
281	282		let context = try domains::context(domainStore, permit.object, receiver.initial) catch {
282	283		throw abi::Error::Busy;

56	57		/// Validate resident object generations for capabilities without page ownership.
57	58		unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
58	59		match entry.kind {
59	60		case abi::Kind::Domain, abi::Kind::Events => {
60	61		let object = try domains::get(&domains::STORE, entry.object);
	62	+	if object.state == domains::Lifecycle::Dead {
	63	+	throw abi::Error::BadHandle;
	64	+	}
61	65		},
62	66		case abi::Kind::Image => {
63	67		let object = try registry::get(&registry::STORE, entry.object);
64	68		},
65	69		else => throw abi::Error::BadHandle,
66	70		}
67	71		}
68	72
	73	+	/// Reject expired domain authority before an allocation can reserve resources.
	74	+	unsafe fn allocationAuthority(table: &capability::Table, operation: u64, handle: abi::Handle) throws (abi::Error) {
	75	+	if operation <> 20 and operation <> 30 and operation <> 51 {
	76	+	return;
	77	+	}
	78	+	let rights = abi::CREATE if operation == 20 else abi::ALLOCATE;
	79	+	try resident(try capability::authority(table, handle, abi::Rights(rights)));
	80	+	}
	81	+
69	82		/// Grant or transfer authority while both domain tables remain serialized.
70	83		unsafe fn delegate(domain: &mut domains::Domain, handle: abi::Handle, target: abi::Handle, rights: u64, moving: bool, now: u64)
71	84		-> abi::Handle throws (abi::Error)
72	85		{
73	86		let owner = domain.memory.table.owner;

179	192		for i in 0..words.len {
180	193		set output[i] = words[i];
181	194		}
182	195		}
183	196
	197	+	/// Validate a termination call and return harts that must release stopped contexts.
	198	+	export unsafe fn termination(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u32 throws (abi::Error) {
	199	+	assert arguments.len == 4;
	200	+	let domain = try domains::get(&domains::STORE, owner);
	201	+	if domain.state <> domains::Lifecycle::Active {
	202	+	throw abi::Error::BadHandle;
	203	+	}
	204	+	if arguments[2] <> 0 or arguments[3] <> 0 {
	205	+	throw abi::Error::InvalidArg;
	206	+	}
	207	+	match operation {
	208	+	case 49 => {
	209	+	if arguments[0] > 0xffffffff or arguments[1] <> 0 {
	210	+	throw abi::Error::InvalidArg;
	211	+	}
	212	+	return try domains::terminate(&mut domains::STORE, owner, events::CHILD_EXIT, arguments[0] as u32);
	213	+	},
	214	+	case 43 => {
	215	+	if arguments[0] <> 0 or arguments[1] <> 0 {
	216	+	throw abi::Error::InvalidArg;
	217	+	}
	218	+	return try domains::terminate(&mut domains::STORE, owner, events::FAULT, abi::FAULT_ABORT);
	219	+	},
	220	+	case 22 => {
	221	+	let flags = arguments[1];
	222	+	if (flags & ~abi::CASCADE) <> 0 or (arguments[0] == 0 and flags <> 0) {
	223	+	throw abi::Error::InvalidArg;
	224	+	}
	225	+	let mut target = owner;
	226	+	if arguments[0] <> 0 {
	227	+	let entry = try capability::lookup(&domain.memory.table, abi::Handle(arguments[0]), abi::Kind::Domain, abi::Rights(abi::DESTROY));
	228	+	set target = entry.object;
	229	+	if (try domains::get(&domains::STORE, target)).state == domains::Lifecycle::Dead {
	230	+	throw abi::Error::BadHandle;
	231	+	}
	232	+	}
	233	+	if flags == abi::CASCADE {
	234	+	return try domains::terminateTree(&mut domains::STORE, target, events::CHILD_EXIT, 0);
	235	+	}
	236	+	return try domains::terminate(&mut domains::STORE, target, events::CHILD_EXIT, 0);
	237	+	},
	238	+	else => throw abi::Error::InvalidArg,
	239	+	}
	240	+	}
	241	+
184	242		/// Execute one operation with domain and resource metadata serialized.
185	243		export unsafe fn invoke(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
186	244		assert arguments.len == 4;
187	245		let mut domain = try domains::get(&domains::STORE, owner);
188	246		let handle = abi::Handle(arguments[0]);
	247	+	try allocationAuthority(&domain.memory.table, operation, handle);
189	248		match operation {
	249	+	case 22, 43, 49 => {
	250	+	let stopped = try termination(owner, operation, arguments);
	251	+	return 0;
	252	+	},
190	253		case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
191	254		case 12 => {
192	255		try drop(&mut domain, handle);
193	256		return 0;
194	257		},

197	260		case 21 => {
198	261		try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,
199	262		handle, arguments[1], arguments[2], arguments[3]);
200	263		return 0;
201	264		},
	265	+	case 23 => {
	266	+	if arguments[2] <> 0 or arguments[3] <> 0 {
	267	+	throw abi::Error::InvalidArg;
	268	+	}
	269	+	try domains::reparent(&mut domains::STORE, &domain.memory.table, handle, abi::Handle(arguments[1]));
	270	+	return 0;
	271	+	},
202	272		case 30 => return *try pages::allocate(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
203	273		case 31 => return *try pages::split(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
204	274		case 40 => {
205	275		if arguments[1] <> 0 or arguments[2] <> 0 or arguments[3] <> 0 {
206	276		throw abi::Error::InvalidArg;

240	310		return 0;
241	311		},
242	312		case 47 => {
243	313		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
244	314		let target = try domains::get(&domains::STORE, entry.object);
	315	+	if target.state == domains::Lifecycle::Dead {
	316	+	throw abi::Error::BadHandle;
	317	+	}
245	318		let mut parent: u64 = 0;
246	319		if let object = target.parent {
247	320		set parent = abi::id(object);
248	321		}
249	322		try reply(&domain, arguments, &[abi::id(entry.object), abi::id(target.creator), parent, target.state as u64]);
250	323		return 0;
251	324		},
252	325		case 48 => {
253	326		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Events, abi::Rights(0));
254	327		let target = try domains::get(&domains::STORE, entry.object);
	328	+	if target.state == domains::Lifecycle::Dead {
	329	+	throw abi::Error::BadHandle;
	330	+	}
255	331		try reply(&domain, arguments, &[events::CAPACITY as u64]);
256	332		return 0;
257	333		},
258	334		case 51 => return *try loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut domain.memory.table,
259	335		loader::Request { authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3] }),

293	369		return 0;
294	370		},
295	371		case 62 => {
296	372		let permission = try capability::authority(&domain.memory.table, handle, abi::Rights(0));
297	373		let target = try domains::get(&domains::STORE, permission.object);
	374	+	if target.state == domains::Lifecycle::Dead {
	375	+	throw abi::Error::BadHandle;
	376	+	}
298	377		let mut object = target.initial;
299	378		if arguments[1] <> 0 {
300	379		set object = abi::reference(arguments[1]);
301	380		}
302	381		let context = try domains::context(&domains::STORE, permission.object, object);

345	424		let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
346	425		catch error {
347	426		sync::leave(guard);
348	427		throw error;
349	428		};
	429	+	let context = abi::reference(dispatch::current().context);
	430	+	let lease = lifecycle::holdLoad(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
	431	+	let retained = lifecycle::load(&lifecycle::CALLS, &lease);
350	432		sync::leave(guard);
351		-	let input = try loader::decode(&pending) catch error {
	433	+	let input = try loader::decodeInput(&retained.input) catch error {
352	434		let guard = sync::enter();
353		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	435	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	436	+	&mut loader::STATE, &mut registry::STORE, lease);
354	437		sync::leave(guard); throw error;
355	438		};
356		-	let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
	439	+	let existing = try loader::identifyInput(&registry::STORE, &retained.input, &input) catch error {
357	440		let guard = sync::enter();
358		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	441	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	442	+	&mut loader::STATE, &mut registry::STORE, lease);
359	443		sync::leave(guard); throw error;
360	444		};
361	445		let allocationGuard = sync::enter();
362	446		if not live(owner) {
363		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	447	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	448	+	&mut loader::STATE, &mut registry::STORE, lease);
364	449		sync::leave(allocationGuard); throw abi::Error::BadHandle;
365	450		}
366	451		if let object = existing {
367		-	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	452	+	let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	453	+	&mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Existing(object))
368	454		catch error {
369	455		sync::leave(allocationGuard);
370	456		throw error;
371	457		};
372	458		sync::leave(allocationGuard); return *result;
373	459		}
374		-	let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
375		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	460	+	try lifecycle::reserveOutput(&mut lifecycle::CALLS, &lease, &mut pages::STORE, &mut registry::STORE) catch error {
	461	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	462	+	&mut loader::STATE, &mut registry::STORE, lease);
376	463		sync::leave(allocationGuard); throw error;
377	464		};
	465	+	let compiling = lifecycle::load(&lifecycle::CALLS, &lease);
	466	+	let output = compiling.output else panic "missing loader output";
378	467		sync::leave(allocationGuard);
379		-	let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
	468	+	let compiled = try loader::compileInput(&registry::STORE, &compiling.input, &input, &output) catch error {
380	469		let guard = sync::enter();
381		-	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
382		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	470	+	lifecycle::cancelLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	471	+	&mut loader::STATE, &mut registry::STORE, lease);
383	472		sync::leave(guard); throw error;
384	473		};
385	474		let publicationGuard = sync::enter();
386		-	if not live(owner) {
387		-	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
388		-	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
389		-	sync::leave(publicationGuard); throw abi::Error::BadHandle;
390		-	}
391		-	let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
392		-	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	475	+	let result = try lifecycle::finishLoad(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE,
	476	+	&mut loader::STATE, &mut registry::STORE, lease, lifecycle::LoadResult::Compiled(compiled))
393	477		catch error {
394	478		sync::leave(publicationGuard);
395	479		throw error;
396	480		};
397	481		sync::leave(publicationGuard);

401	485		/// Execute a call with short metadata transactions and private bulk initialization.
402	486		/// Context ownership and pending calls retain storage across dispatch boundaries.
403	487		export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
404	488		assert arguments.len == 4;
405	489		if operation == 21 or operation == 50 {
406		-	return try runtimeReady(owner, operation, arguments, now);
	490	+	return try runtimeReady(owner, operation, arguments);
407	491		}
408	492		let guard = sync::enter();
409	493		let mut domain = try domains::get(&domains::STORE, owner) catch error {
410	494		sync::leave(guard);
411	495		throw error;
412	496		};
413	497		if domain.state <> domains::Lifecycle::Active {
414	498		sync::leave(guard);
415	499		throw abi::Error::BadHandle;
416	500		}
	501	+	try allocationAuthority(&domain.memory.table, operation, abi::Handle(arguments[0])) catch error {
	502	+	sync::leave(guard); throw error;
	503	+	};
	504	+	if operation == 22 or operation == 43 or operation == 49 {
	505	+	let stopped = try termination(owner, operation, arguments) catch error {
	506	+	sync::leave(guard);
	507	+	throw error;
	508	+	};
	509	+	dispatch::reschedule(stopped);
	510	+	sync::leave(guard); return 0;
	511	+	}
417	512		let target = try budgetHart(&domain.memory.table, operation, arguments) catch error {
418	513		sync::leave(guard);
419	514		throw error;
420	515		};
421	516		if let destination = target {

438	533		let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
439	534		&mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error {
440	535		sync::leave(guard);
441	536		throw error;
442	537		};
	538	+	let context = abi::reference(dispatch::current().context);
	539	+	let lease = lifecycle::holdDomain(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
443	540		sync::leave(guard);
444		-	try domains::prepare(&registry::STORE, &pending) catch error {
	541	+	try lifecycle::prepareDomain(&lifecycle::CALLS, &registry::STORE, &lease) catch error {
445	542		let guard = sync::enter();
446		-	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	543	+	let failed = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
	544	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, failed);
447	545		sync::leave(guard); throw error;
448	546		};
449	547		let guard = sync::enter();
	548	+	let ready = lifecycle::takeDomain(&mut lifecycle::CALLS, &mut domains::STORE, lease);
450	549		if not live(owner) {
451		-	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	550	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready);
452	551		sync::leave(guard); throw abi::Error::BadHandle;
453	552		}
454		-	let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
	553	+	let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, ready)
455	554		catch error {
456	555		sync::leave(guard);
457	556		throw error;
458	557		};
459	558		sync::leave(guard); return *result;

462	561		let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
463	562		catch error {
464	563		sync::leave(guard);
465	564		throw error;
466	565		};
	566	+	let context = abi::reference(dispatch::current().context);
	567	+	let lease = lifecycle::holdPage(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
467	568		sync::leave(guard);
468		-	pages::clear(&pending);
	569	+	lifecycle::clearPage(&lifecycle::CALLS, &lease);
469	570		let guard = sync::enter();
	571	+	let ready = lifecycle::takePage(&mut lifecycle::CALLS, &mut domains::STORE, lease);
470	572		if not live(owner) {
471		-	pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
	573	+	pages::cancel(&mut pages::STORE, &mut domain.memory.table, ready);
472	574		sync::leave(guard); throw abi::Error::BadHandle;
473	575		}
474		-	let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
	576	+	let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, ready);
475	577		sync::leave(guard); return *result;
476	578		},
477	579		case 51 => return try runtimeLoad(&mut domain, loader::Request {
478	580		authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
479	581		}, guard),

493	595		match &pending {
494	596		case domains::ContextReservation::Held(allocation) => {
495	597		set target = allocation.owner;
496	598		},
497	599		}
	600	+	let context = abi::reference(dispatch::current().context);
	601	+	let lease = lifecycle::holdContext(&mut lifecycle::CALLS, &mut domains::STORE, context, pending);
498	602		sync::leave(guard);
499		-	domains::contextClear(&pending);
	603	+	lifecycle::clearContext(&lifecycle::CALLS, &lease);
500	604		let guard = sync::enter();
	605	+	let ready = lifecycle::takeContext(&mut lifecycle::CALLS, &mut domains::STORE, lease);
501	606		if not live(owner) or not live(target) {
502		-	domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
	607	+	domains::contextCancel(&mut domains::STORE, &mut pages::STORE, ready);
503	608		sync::leave(guard); throw abi::Error::BadHandle;
504	609		}
505		-	let result = domains::contextPublish(&mut domains::STORE, pending);
	610	+	let result = domains::contextPublish(&mut domains::STORE, ready);
506	611		sync::leave(guard); return abi::id(result);
507	612		},
508	613		else => {
509		-	let result = try invoke(owner, operation, arguments, now) catch error {
	614	+	let result = try invoke(owner, operation, arguments, dispatch::clock()) catch error {
510	615		sync::leave(guard);
511	616		throw error;
512	617		};
513	618		sync::leave(guard); return result;
514	619		},
515	620		}
516	621		}
517	622
518	623		/// Reserve remote dispatch capacity before activation or event publication.
519		-	unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
	624	+	unsafe fn runtimeReady(owner: abi::Ref, operation: u64, arguments: &[u64]) -> u64 throws (abi::Error) {
520	625		if operation == 50 and (arguments[1] > 0xffffffff or arguments[2] <> 0 or arguments[3] <> 0) {
521	626		throw abi::Error::InvalidArg;
522	627		}
523	628		let guard = sync::enter();
524	629		let domain = try domains::get(&domains::STORE, owner) catch error {

534	639		let permit = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(rights)) catch error {
535	640		sync::leave(guard); throw error;
536	641		};
537	642		let source = dispatch::hart();
538	643		let context = abi::reference(dispatch::current().context);
	644	+	let now = dispatch::clock();
539	645		let targets = budgets::wakeTargets(&budgets::STORE, &domains::STORE, permit.object, now) & ~(1 << source);
540	646		let pending = try remote::reserve(&mut remote::STORE, source, targets) catch error {
541	647		sync::leave(guard);
542	648		throw error;
543	649		};

42	49		record Runtime: Copy {
43	50		/// Current context and retained idle frame.
44	51		state: State,
45	52		/// Validated timer registers for this hart.
46	53		timer: Timer,
	54	+	/// Minimum ticks between scheduled maintenance passes.
	55	+	maintenance: u64,
	56	+	/// Earliest tick for another background maintenance pass.
	57	+	nextMaintenance: u64,
47	58		}
48	59
49	60		/// Runtime slots initialized before each hart enables timer interrupts.
50	61		unsafe static HARTS: [Runtime; limits::HARTS] = undefined;
51	62

65	76		let device = try timer(machine, id);
66	77		let stack = machine.stacks[id];
67	78		let mut idle = trap::Frame { registers: [0; 32], pc: idleAddress(), status: 0x1880, cause: 0, value: 0 };
68	79		set idle.registers[2] = stack.end;
69	80		set idle.registers[3] = kernelGp;
	81	+	let mut maintenance = machine.timebase as u64 / 100;
	82	+	if maintenance == 0 {
	83	+	set maintenance = 1;
	84	+	}
70	85		set HARTS[id] = Runtime {
71		-	state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
	86	+	state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device, maintenance, nextMaintenance: 0,
72	87		};
73	88		arm(device, 0xffffffffffffffff);
74	89		}
75	90
76	91		/// Tail-enter dispatch on the firmware stack and resume the selected frame.

108	123		set deliveries[count] = delivery;
109	124		set count += 1;
110	125		}
111	126		let clock = now(HARTS[id].timer);
112	127		let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, &budgets::STORE, id, clock);
113		-	let notified = timeout.harts \| plic::service(clock);
	128	+	let notified = timeout.harts \| plic::service(clock) \| lifecycle::service(&mut domains::STORE, &budgets::STORE, clock);
114	129		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
115	130		let mut deadline = choice.deadline;
116	131		if timeout.deadline < deadline {
117	132		set deadline = timeout.deadline;
118	133		}
119		-	for target in 0..limits::HARTS {
120		-	if target <> id and (notified & (1 << target)) <> 0 {
121		-	signal(target);
122		-	}
123		-	}
124		-	arm(HARTS[id].timer, deadline);
	134	+	reschedule(notified);
125	135		if previous <> choice.context {
126	136		if let context = previous {
127	137		signalContext(context, clock);
128	138		}
129	139		}

137	147		}
138	148		if let case remote::Action::Ready(context) = delivery.request.action {
139	149		signalContext(context, clock);
140	150		}
141	151		}
	152	+	lifecycle::cancelRequests(&mut domains::STORE, &mut remote::STORE);
	153	+	let cancelled = lifecycle::cancelAllocations(&mut lifecycle::CALLS, &mut domains::STORE, &mut pages::STORE, &mut loader::STATE, &mut registry::STORE);
	154	+	let maintenanceClock = now(HARTS[id].timer);
	155	+	let mut maintenance = cancelled or domains::STORE.dead > 0 or pages::STORE.backings.pool.retiredCount > 0;
	156	+	if maintenance and maintenanceClock >= HARTS[id].nextMaintenance {
	157	+	let reaped = lifecycle::reapNext(&mut domains::STORE, &mut pages::STORE, &mut budgets::STORE, &mut interrupts::STORE);
	158	+	let reclaimed = frames::reclaim(&mut pages::STORE.backings.pool, 256);
	159	+	set maintenance = cancelled or reaped or pages::STORE.backings.pool.retiredCount > 0;
	160	+	let after = now(HARTS[id].timer);
	161	+	let interval = HARTS[id].maintenance;
	162	+	set HARTS[id].nextMaintenance = 0xffffffffffffffff;
	163	+	if interval < 0xffffffffffffffff - after {
	164	+	set HARTS[id].nextMaintenance = after + interval;
	165	+	}
	166	+	}
	167	+	if maintenance and HARTS[id].nextMaintenance < deadline {
	168	+	set deadline = HARTS[id].nextMaintenance;
	169	+	}
	170	+	arm(HARTS[id].timer, deadline);
142	171		sync::leave(guard);
143	172		}
144	173
145	174		/// Signal an initialized online hart while request metadata is serialized.
146	175		unsafe fn signal(target: u32) {
147	176		writeSoftware(HARTS[target].timer.software, 1);
148	177		}
149	178
	179	+	/// Signal other harts that retain stopped contexts under the metadata lock.
	180	+	export unsafe fn reschedule(harts: u32) {
	181	+	let source = hart();
	182	+	for target in 0..limits::HARTS {
	183	+	if target <> source and (harts & (1 << target)) <> 0 {
	184	+	signal(target);
	185	+	}
	186	+	}
	187	+	}
	188	+
150	189		/// Notify the current owner or the eligible hart after a context is released.
151	190		unsafe fn signalContext(context: abi::Ref, now: u64) {
152	191		if not slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live) {
153	192		return;
154	193		}

251	290		};
252	291		releaseRequest(source, request);
253	292		sync::leave(guard); return result;
254	293		}
255	294
	295	+	/// Read mtime through the executing hart's validated timer mapping.
	296	+	export unsafe fn clock() -> u64 { return now(HARTS[hart()].timer); }
	297	+
256	298		/// Read the running context's identity and domain-relative CPU authority.
257	299		export unsafe fn current() -> abi::CurrentContextInfo {
258	300		let id = hart();
259	301		assert id < limits::HARTS;
260	302		let context = HARTS[id].state.current else panic "no current context";

271	313		let guard = sync::enter();
272	314		assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
273	315		let owner = domains::STORE.contexts[context.index].owner;
274	316		if frame.registers[17] == abi::Operation::ContextReturn as u64 {
275	317		assert trap::advanceCall(frame);
276		-	try! domains::returned(&mut domains::STORE, owner, context);
	318	+	let stopped = try! domains::returned(&mut domains::STORE, owner, context);
	319	+	reschedule(stopped);
277	320		sync::leave(guard);
278	321		} else {
279	322		sync::leave(guard);
	323	+	sync::restoreInterrupts(8);
280	324		calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
	325	+	let interrupts = sync::maskInterrupts();
281	326		}
282	327		interrupt(frame, &mut *localAnchor());
283	328		}
284	329
	330	+	/// Stop a faulting user domain and resume another authorized context.
	331	+	export unsafe fn fault(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
	332	+	assert trap::fromUser(frame);
	333	+	let guard = sync::enter();
	334	+	let context = HARTS[hart()].state.current else panic "fault without current context";
	335	+	assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
	336	+	let owner = domains::STORE.contexts[context.index].owner;
	337	+	let stopped = try! domains::terminate(&mut domains::STORE, owner, events::FAULT, frame.cause as u32);
	338	+	reschedule(stopped);
	339	+	sync::leave(guard);
	340	+	interrupt(frame, &mut *localAnchor());
	341	+	}
	342	+
285	343		/// Enter the first authorized context or the hart's retained idle frame.
286	344		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
287	345		let id = hart();
288	346		assert id < limits::HARTS;
289	347		let mut frame = HARTS[id].state.idle;

18	18		export constant KERNEL_STACK_PAGES: u32 = 64;
19	19
20	20		/// Management rights returned to a domain's creator.
21	21		export constant MANAGEMENT_RIGHTS: u16 = abi::DESTROY \| abi::EXECUTE \| abi::GRANT \| abi::TRANSFER \| abi::WAKE;
22	22
	23	+	/// Words needed to retain one ancestry bit per domain slot.
	24	+	constant ANCESTRY_WORDS: u32 = (limits::DOMAINS + 31) / 32;
	25	+
23	26		/// Protection-domain lifetime independent of execution budgets.
24	27		export union Lifecycle: Copy {
25	28		/// Resources can be installed before one-time activation.
26	29		Pending,
27	30		/// The domain can execute when a context has CPU authority.
28	31		Active,
29		-	/// Execution has stopped and the identity can remain as a tombstone.
	32	+	/// Contexts cannot be selected; hart release and resource cleanup can remain.
30	33		Dead,
31	34		}
32	35
33	36		/// Private domain metadata and its shared event ring.
34	37		export record Memory: Copy {

58	61		state: ContextState,
59	62		/// Hart retaining this context as its current execution owner.
60	63		hart: ?u32,
61	64		/// Remote call whose retained continuation requires this context's storage.
62	65		pending: ?abi::Ref,
	66	+	/// A kernel call retains unpublished resources and requires this context's stack.
	67	+	reservation: bool,
63	68		/// Physical frames retained until this context stops.
64	69		kernelFrames: frames::Run,
65	70		/// Exclusive stack bounds used by trap entry and suspended M-mode calls.
66	71		kernelStack: range::Range,
67	72		/// Exclusive user stack extent; empty until initial activation.

96	101		export record Store: Copy {
97	102		/// Domain generations, including retained dead identities.
98	103		slots: [slots::Slot; limits::DOMAINS],
99	104		/// Payload for each published domain.
100	105		records: [Domain; limits::DOMAINS],
	106	+	/// True after private storage is retired, while terminal identity can remain.
	107	+	reclaimed: [bool; limits::DOMAINS],
	108	+	/// True after quiescence admits a domain to resource teardown.
	109	+	reaping: [bool; limits::DOMAINS],
	110	+	/// Number of dead domain identities retained by the store.
	111	+	dead: u32,
	112	+	/// First domain slot considered by the next reaper pass.
	113	+	reapCursor: u32,
	114	+	/// Creation ancestors by domain slot. Clear a column before its ancestor slot is reused.
	115	+	ancestors: [[u32; ANCESTRY_WORDS]; limits::DOMAINS],
101	116		/// Context generations shared by initial and additional contexts.
102	117		contextSlots: [slots::Slot; limits::CONTEXTS],
103	118		/// Payload for each published execution context.
104	119		contexts: [Context; limits::CONTEXTS],
105	120		/// Trusted event producers and terminal records.

116	131		/// Address of the U-mode normal-return call sequence.
117	132		fn returnAddress() -> u64;
118	133
119	134		/// Initialize fresh metadata before boot-domain publication.
120	135		export fn initialize(store: &mut Store) {
	136	+	set store.dead = 0;
	137	+	set store.reapCursor = 0;
121	138		slots::initialize(&mut store.slots[..]);
122	139		slots::initialize(&mut store.contextSlots[..]);
	140	+	for i in 0..limits::CONTEXTS {
	141	+	set store.contexts[i].reservation = false;
	142	+	}
	143	+	for i in 0..limits::DOMAINS {
	144	+	set store.ancestors[i] = [0;ANCESTRY_WORDS];
	145	+	set store.reclaimed[i] = false;
	146	+	set store.reaping[i] = false;
	147	+	}
123	148		events::initialize(&mut store.events);
124	149		}
125	150
126	151		/// Read a domain record after validating its retained generation.
127	152		export fn get(store: &Store, object: abi::Ref) -> Domain throws (abi::Error) {
128		-	if not slots::matches(&store.slots[..], object, slots::State::Live) {
	153	+	if not slots::matches(&store.slots[..], object, slots::State::Live) or store.reclaimed[object.index] {
129	154		throw abi::Error::BadHandle;
130	155		}
131	156		return store.records[object.index];
132	157		}
133	158
	159	+	/// Test whether all published and unpublished execution ownership has ended.
	160	+	export fn quiescent(store: &Store, owner: abi::Ref) -> bool throws (abi::Error) {
	161	+	let domain = try get(store, owner);
	162	+	if domain.state <> Lifecycle::Dead {
	163	+	return false;
	164	+	}
	165	+	for i in 0..limits::CONTEXTS {
	166	+	let state = store.contextSlots[i].state;
	167	+	if state <> slots::State::Live and state <> slots::State::Reserved {
	168	+	continue;
	169	+	}
	170	+	if store.contexts[i].owner <> owner {
	171	+	continue;
	172	+	}
	173	+	if state == slots::State::Reserved or store.contexts[i].state <> ContextState::Stopped
	174	+	or store.contexts[i].hart <> nil or store.contexts[i].pending <> nil or store.contexts[i].reservation {
	175	+	return false;
	176	+	}
	177	+	}
	178	+	return true;
	179	+	}
	180	+
	181	+	/// Retire at most one stopped context after all domain execution ownership ends.
	182	+	export fn retireContext(store: &mut Store, pool: &mut frames::Pool, owner: abi::Ref) -> bool throws (abi::Error) {
	183	+	if not try quiescent(store, owner) {
	184	+	throw abi::Error::Busy;
	185	+	}
	186	+	for i in 0..limits::CONTEXTS {
	187	+	if store.contextSlots[i].state <> slots::State::Live or store.contexts[i].owner <> owner {
	188	+	continue;
	189	+	}
	190	+	let object = abi::Ref { index: i, generation: store.contextSlots[i].generation };
	191	+	try! frames::retire(pool, store.contexts[i].kernelFrames);
	192	+	try! slots::release(&mut store.contextSlots[..], object);
	193	+	return true;
	194	+	}
	195	+	return false;
	196	+	}
	197	+
	198	+	/// Retire private domain storage after contexts, handles, and exposures are gone.
	199	+	export unsafe fn retireStorage(store: &mut Store, backings: &mut backing::Store, owner: abi::Ref) throws (abi::Error) {
	200	+	let domain = try get(store, owner);
	201	+	if not try quiescent(store, owner) {
	202	+	throw abi::Error::Busy;
	203	+	}
	204	+	for i in 0..limits::CONTEXTS {
	205	+	if store.contextSlots[i].state == slots::State::Live and store.contexts[i].owner == owner {
	206	+	throw abi::Error::Busy;
	207	+	}
	208	+	}
	209	+	for i in 0..limits::HANDLES {
	210	+	let state = domain.memory.table.slots[i].state;
	211	+	if state == slots::State::Live or state == slots::State::Reserved {
	212	+	throw abi::Error::Busy;
	213	+	}
	214	+	}
	215	+	if backings.domains[owner.index] <> 0 or store.events.queues[owner.index].generation <> 0 {
	216	+	throw abi::Error::Busy;
	217	+	}
	218	+	try! frames::retire(&mut backings.pool, domain.graph.frames);
	219	+	try! frames::retire(&mut backings.pool, domain.allocation);
	220	+	set store.reclaimed[owner.index] = true;
	221	+	}
	222	+
	223	+	/// Release an acknowledged terminal identity and clear its creation-ancestry column.
	224	+	export fn releaseIdentity(store: &mut Store, owner: abi::Ref) throws (abi::Error) {
	225	+	if not slots::matches(&store.slots[..], owner, slots::State::Live) {
	226	+	throw abi::Error::BadHandle;
	227	+	}
	228	+	if not store.reclaimed[owner.index] or not try events::acknowledged(&store.events, owner) {
	229	+	throw abi::Error::Busy;
	230	+	}
	231	+	let word = owner.index / 32;
	232	+	let bit = 1 << (owner.index % 32);
	233	+	for i in 0..limits::DOMAINS {
	234	+	set store.ancestors[i][word] &= ~bit;
	235	+	}
	236	+	try! events::release(&mut store.events, owner);
	237	+	try! slots::release(&mut store.slots[..], owner);
	238	+	assert store.dead > 0;
	239	+	set store.dead -= 1;
	240	+	}
	241	+
134	242		/// Read a live context after checking its generation and owning domain.
135	243		export fn context(store: &Store, owner: abi::Ref, object: abi::Ref) -> Context throws (abi::Error) {
136	244		let domain = try get(store, owner);
137	245		if not slots::matches(&store.contextSlots[..], object, slots::State::Live)
138	246		or store.contexts[object.index].owner <> owner {
139	247		throw abi::Error::BadHandle;
140	248		}
141	249		return store.contexts[object.index];
142	250		}
143	251
	252	+	/// Change the live lifecycle receiver under shared metadata serialization.
	253	+	export fn reparent(store: &mut Store, table: &capability::Table, child: abi::Handle, parent: abi::Handle)
	254	+	throws (abi::Error)
	255	+	{
	256	+	let target = try capability::lookup(table, child, abi::Kind::Domain, abi::Rights(abi::DESTROY));
	257	+	let receiver = try capability::lookup(table, parent, abi::Kind::Domain, abi::Rights(abi::WAKE));
	258	+	let domain = try get(store, target.object);
	259	+	let destination = try get(store, receiver.object);
	260	+	if domain.state == Lifecycle::Dead or destination.state == Lifecycle::Dead {
	261	+	throw abi::Error::BadHandle;
	262	+	}
	263	+	set store.records[target.object.index].parent = receiver.object;
	264	+	}
	265	+
	266	+	/// Retain a terminal event, stop execution, and report harts that must release contexts.
	267	+	export fn terminate(store: &mut Store, owner: abi::Ref, kind: u16, code: u32) -> u32 throws (abi::Error) {
	268	+	return try stop(store, owner, false, kind, code);
	269	+	}
	270	+
	271	+	/// Stop a domain and its creation descendants without changing creation ancestry.
	272	+	export fn terminateTree(store: &mut Store, owner: abi::Ref, kind: u16, code: u32) -> u32 throws (abi::Error) {
	273	+	return try stop(store, owner, true, kind, code);
	274	+	}
	275	+
	276	+	/// Commit terminal state for a selected set before releasing the metadata lock.
	277	+	fn stop(store: &mut Store, owner: abi::Ref, tree: bool, kind: u16, code: u32) -> u32 throws (abi::Error) {
	278	+	if kind <> events::CHILD_EXIT and kind <> events::FAULT {
	279	+	throw abi::Error::InvalidArg;
	280	+	}
	281	+	let domain = try get(store, owner);
	282	+	let mut generations: [u32; limits::DOMAINS] = [0; limits::DOMAINS];
	283	+	for i in 0..limits::DOMAINS {
	284	+	if store.slots[i].state <> slots::State::Live or store.records[i].state == Lifecycle::Dead {
	285	+	continue;
	286	+	}
	287	+	if i <> owner.index and (not tree or (store.ancestors[i][owner.index / 32] & (1 << (owner.index % 32))) == 0) {
	288	+	continue;
	289	+	}
	290	+	if store.events.terminals[i].state <> events::State::Free {
	291	+	throw abi::Error::Busy;
	292	+	}
	293	+	if store.events.queues[i].generation <> store.slots[i].generation {
	294	+	throw abi::Error::BadHandle;
	295	+	}
	296	+	set generations[i] = store.slots[i].generation;
	297	+	}
	298	+	for i in 0..limits::DOMAINS {
	299	+	if generations[i] == 0 {
	300	+	continue;
	301	+	}
	302	+	let object = abi::Ref { index: i, generation: generations[i] };
	303	+	let terminal = try! events::reserve(&mut store.events, object);
	304	+	let mut receiver = abi::Ref { index: 0, generation: 0 };
	305	+	if let parent = store.records[i].parent {
	306	+	set receiver = parent;
	307	+	}
	308	+	events::finish(&mut store.events, terminal, receiver, kind, code);
	309	+	set store.records[i].state = Lifecycle::Dead;
	310	+	set store.dead += 1;
	311	+	}
	312	+	try! events::closeSet(&mut store.events, &generations[..]);
	313	+	let mut harts: u32 = 0;
	314	+	for i in 0..limits::CONTEXTS {
	315	+	if store.contextSlots[i].state <> slots::State::Live {
	316	+	continue;
	317	+	}
	318	+	let target = store.contexts[i].owner;
	319	+	if generations[target.index] == target.generation {
	320	+	set store.contexts[i].state = ContextState::Stopped;
	321	+	if let hart = store.contexts[i].hart {
	322	+	set harts \|= 1 << hart;
	323	+	}
	324	+	}
	325	+	}
	326	+	for i in 0..limits::DOMAINS {
	327	+	if store.slots[i].state <> slots::State::Live {
	328	+	continue;
	329	+	}
	330	+	if let parent = store.records[i].parent {
	331	+	if generations[parent.index] == parent.generation {
	332	+	set store.records[i].parent = nil;
	333	+	}
	334	+	}
	335	+	}
	336	+	return harts;
	337	+	}
	338	+
144	339		/// Stop a returned context; initial return ends execution of its whole domain.
145		-	export fn returned(store: &mut Store, owner: abi::Ref, object: abi::Ref) throws (abi::Error) {
	340	+	export fn returned(store: &mut Store, owner: abi::Ref, object: abi::Ref) -> u32 throws (abi::Error) {
146	341		let current = try context(store, owner, object);
147	342		if store.records[owner.index].initial == object {
148		-	set store.records[owner.index].state = Lifecycle::Dead;
149		-	for i in 0..limits::CONTEXTS {
150		-	if store.contextSlots[i].state == slots::State::Live and store.contexts[i].owner == owner {
151		-	set store.contexts[i].state = ContextState::Stopped;
152		-	}
153		-	}
154		-	} else {
155		-	set store.contexts[object.index].state = ContextState::Stopped;
	343	+	return try terminate(store, owner, events::CHILD_EXIT, 0);
156	344		}
	345	+	set store.contexts[object.index].state = ContextState::Stopped;
	346	+	return 0;
157	347		}
158	348
159	349		/// Atomically check notifications and suspend the caller if its queue is empty.
160	350		/// The caller serializes this check with notification publication.
161	351		export unsafe fn wait(store: &mut Store, owner: abi::Ref, object: abi::Ref) throws (abi::Error) {