radiance · commit

kernel: Start secondary harts

8aaf2eee4dfb86e25f5b9d3769907ea395fe545f6a8244e19a698655b926e704

Serialize shared metadata with interrupt-masked linear guards. Keep
initialization and compilation outside the lock, exchange contexts on
hart-owned stacks, and gate dispatch on complete hart initialization.

Verify concurrent contexts of one domain on 1, 2, and 8 harts,
repeated capability transactions, register isolation, and measured
metadata work. Full library, kernel, compiler, and machine suites
pass.

Assisted-by: Codex:gpt-6

Alexis Sellier committed 13 days ago 1 parent f2d48725

Makefile +6 -1


# Verify the emulator binary exists.
EMU_PATH := $(shell command -v $(EMU) 2>/dev/null)

default: emulator $(RAD_BIN)
test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test
test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test

seed-test:
	@seed/test

# Emulator command check


# Native preemption across user code, idle, and retained machine work.
.PHONY: dispatch-test
dispatch-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run

# Concurrent user contexts and metadata transactions on every online hart.
.PHONY: smp-test
smp-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8"

kernel/kernel/boot.rad +9 -2

    assert stackTop == PLATFORM.stacks[hart as u32].end and (stackTop & 15) == 0;
    let stack = PLATFORM.stacks[hart as u32];
    set HARTS[hart as u32] = trap::Hart {
        stackTop: stack.end, stackBottom: stack.start, kernelGp: stateTable as u64, handler: unexpected,
        savedT0: 0, savedT1: 0, savedSp: 0,
        dispatchTop: stack.end, dispatchFrame: undefined,
    };
    trap::install(&mut HARTS[hart as u32]);
    try! dispatch::initialize(&PLATFORM, hart as u32, stateTable as u64);
    let mut count: u64 = 0;
    for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
    return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
}

/// Enter dispatch after this hart completes platform initialization.
export unsafe fn run() { dispatch::start(&mut HARTS[dispatch::hart()]); }
/// Enter dispatch after every online hart publishes its initialized state.
export unsafe fn run() -> ! {
    let mut count: u64 = 0;
    for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
    while sync::loadAcquire(&ARRIVED) < count {}
    sync::syncInstructions();
    dispatch::start(&mut HARTS[dispatch::hart()]);
}

kernel/kernel/calls.rad +133 -3

use super::events;
use super::transactions;
use super::loader;
use super::dispatch;
use super::timers;
use super::sync;

/// Validate resident object generations for capabilities without page ownership.
unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },

            budgets::Binding { budget: handle, domain: abi::Handle(arguments[1]), context: abi::reference(arguments[2]) }, now),
        else => throw abi::Error::InvalidArg,
    }
}

/// Complete a user ecall and preserve all registers except its scalar result.
export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64) {
/// Check a retained domain's lifetime while metadata is serialized.
unsafe fn live(owner: abi::Ref) -> bool {
    return slots::matches(&domains::STORE.slots[..], owner, slots::State::Live)
        and domains::STORE.records[owner.index].state <> domains::Lifecycle::Dead;
}

/// Load a package with metadata locked only around resource transactions.
unsafe fn runtimeLoad(domain: &mut domains::Domain, request: loader::Request, guard: sync::Guard) -> u64 throws (abi::Error) {
    let owner = domain.memory.table.owner;
    let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
        catch error { sync::leave(guard); throw error; };
    sync::leave(guard);
    let input = try loader::decode(&pending) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(guard); throw error;
    };
    let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(guard); throw error;
    };
    let allocationGuard = sync::enter();
    if not live(owner) {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(allocationGuard); throw abi::Error::BadHandle;
    }
    if let object = existing {
        let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
            catch error { sync::leave(allocationGuard); throw error; };
        sync::leave(allocationGuard); return *result;
    }
    let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(allocationGuard); throw error;
    };
    sync::leave(allocationGuard);
    let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
        let guard = sync::enter();
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(guard); throw error;
    };
    let publicationGuard = sync::enter();
    if not live(owner) {
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(publicationGuard); throw abi::Error::BadHandle;
    }
    let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
    let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
        catch error { sync::leave(publicationGuard); throw error; };
    sync::leave(publicationGuard);
    return *result;
}

/// Execute a call with short metadata transactions and private bulk initialization.
/// The running context keeps its domain storage live until dispatch releases it.
export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    let guard = sync::enter();
    let mut domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
    if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
    let handle = abi::Handle(arguments[0]);
    match operation {
        case 20 => {
            let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
                &mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error { sync::leave(guard); throw error; };
            sync::leave(guard);
            try domains::prepare(&registry::STORE, &pending) catch error {
                let guard = sync::enter();
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                sync::leave(guard); throw error;
            };
            let guard = sync::enter();
            if not live(owner) {
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
                catch error { sync::leave(guard); throw error; };
            sync::leave(guard); return *result;
        },
        case 30 => {
            let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
                catch error { sync::leave(guard); throw error; };
            sync::leave(guard);
            pages::clear(&pending);
            let guard = sync::enter();
            if not live(owner) {
                pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
            sync::leave(guard); return *result;
        },
        case 51 => return try runtimeLoad(&mut domain, loader::Request {
            authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
        }, guard),
        case 60 => {
            if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
                sync::leave(guard); throw abi::Error::InvalidArg;
            }
            let words = @sliceOf(memory(arguments[1]), 4);
            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
                &domain.memory.table, handle, start) catch error { sync::leave(guard); throw error; };
            let mut target: abi::Ref = undefined;
            match &pending { case domains::ContextReservation::Held(allocation) => { set target = allocation.owner; }, }
            sync::leave(guard);
            domains::contextClear(&pending);
            let guard = sync::enter();
            if not live(owner) or not live(target) {
                domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = domains::contextPublish(&mut domains::STORE, pending);
            sync::leave(guard); return abi::id(result);
        },
        else => {
            let result = try invoke(owner, operation, arguments, now) catch error { sync::leave(guard); throw error; };
            sync::leave(guard); return result;
        },
    }
}

/// Complete a user ecall through the supplied execution boundary.
export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
    execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
{
    assert trap::fromUser(frame) and frame.cause == 8;
    assert trap::advanceCall(frame);
    let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
    let result = try invoke(owner, frame.registers[17], &args[..], now) catch error {
    let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
        set frame.registers[10] = (0 as u64) - error as u64;
        return;
    };
    set frame.registers[10] = result;
}

kernel/kernel/dispatch.rad +26 -7

use super::platform;
use super::trap;
use super::range;
use super::calls;
use super::timers;
use super::sync;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

/// Runtime slots initialized before each hart enables timer interrupts.
unsafe static HARTS: [Runtime; limits::HARTS] = undefined;

/// Read the executing physical hart identifier.
export fn hart() -> u32;
/// Read the current machine stack pointer.
fn stackPointer() -> u64;
/// Address of the M-mode idle loop.
fn idleAddress() -> u64;
/// Enable machine timer delivery while global interrupts remain disabled.
fn enable();


        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
    };
    arm(device, 0xffffffffffffffff);
}

/// Reevaluate authority after a timer interrupt with shared metadata serialized.
export unsafe fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
/// Tail-enter dispatch on the firmware stack and resume the selected frame.
export fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> !;

/// Select authority from hart-owned storage with shared metadata serialized.
export unsafe fn choose(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
    let id = hart();
    assert id < limits::HARTS;
    let stack = HARTS[id].state.stack;
    let pointer = stackPointer();
    assert pointer >= stack.start and pointer < stack.end;
    let guard = sync::enter();
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout < deadline { set deadline = timeout; }
    arm(HARTS[id].timer, deadline);
    sync::leave(guard);
}

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();

    let window = try! budgets::get(&budgets::STORE, budget);
    return abi::CurrentContextInfo { context: abi::id(context), hart: id as u64, budget: *window.handle };
}

/// Complete the current user's call and select authority at the return boundary.
export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else { trap::halt(); return; };
    let context = HARTS[id].state.current else trap::halt();
    let guard = sync::enter();
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    let owner = domains::STORE.contexts[context.index].owner;
    if frame.registers[17] == abi::Operation::ContextReturn as u64 {
        assert trap::advanceCall(frame);
        try! domains::returned(&mut domains::STORE, owner, context);
        sync::leave(guard);
    } else {
        calls::handle(owner, frame, now(HARTS[id].timer));
        sync::leave(guard);
        calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
    }
    interrupt(frame, anchor);
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let mut frame = HARTS[id].state.idle;
    interrupt(&mut frame, anchor);
    enable();
    trap::resume(&frame);
    interrupt(&mut frame, anchor);
}

/// Read one validated, naturally aligned 64-bit device register.
fn read(address: u64) -> u64;
/// Write one validated, naturally aligned 64-bit device register.

        if not runnable(contexts, window.owner, context) { continue; }
        if window.start > now {
            if window.start < result.deadline { set result.deadline = window.start; }
            continue;
        }
        if let owner = contexts.contexts[context.index].hart {
            if owner <> hart {
                if window.end < result.deadline { set result.deadline = window.end; }
                continue;
            }
        }
        assert result.budget == nil;
        set result.budget = abi::Ref { index: i, generation: windows.slots[i].generation };
        set result.context = context;
        if window.end < result.deadline { set result.deadline = window.end; }
    }

kernel/kernel/dispatch.ras +26 -0

//! Naturally aligned RV64 CLINT access with device ordering.
.text;
.export @kernel::dispatch::read;
.export @kernel::dispatch::write;
.export @kernel::dispatch::hart;
.export @kernel::dispatch::stackPointer;
.export @kernel::dispatch::idleAddress;
.export @kernel::dispatch::enable;
.export @kernel::dispatch::interrupt;

// Retain the frame outside the context stack before publishing its release.
@kernel::dispatch::interrupt
    mv %t0 %a0;
    addi %t1 %a1 64;
    li %t2 36;
@retainDispatchFrame
    ld %t3 0(%t0);
    sd %t3 0(%t1);
    addi %t0 %t0 8;
    addi %t1 %t1 8;
    addi %t2 %t2 -1;
    bnez %t2 @retainDispatchFrame;
    ld %sp 56(%a1);
    addi %a0 %a1 64;
    call @kernel::dispatch::choose;
    csrr %a0 mscratch;
    addi %a0 %a0 64;
    j @kernel::trap::resume;

// The executing hart selects its private dispatch runtime slot.
@kernel::dispatch::hart
    csrr %a0 mhartid;
    ret;

// Dispatch releases context stacks only from the hart's firmware stack.
@kernel::dispatch::stackPointer
    mv %a0 %sp;
    ret;

// Saved idle frames enter this loop with machine interrupts enabled by mret.
@kernel::dispatch::idleAddress
    la %a0 @dispatchIdle;
    ret;
@dispatchIdle

kernel/kernel/domains.rad +216 -58

        }
    }
    throw abi::Error::InvalidArg;
}

/// Reject physical overlap with any live context's reserved user stack.
/// Reject physical overlap with published or reserved user stacks.
fn stackAvailable(store: &Store, candidate: range::Range) throws (abi::Error) {
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state <> slots::State::Live { continue; }
        if store.contextSlots[i].state <> slots::State::Live and store.contextSlots[i].state <> slots::State::Reserved { continue; }
        let used = store.contexts[i].userStack;
        if used.start < used.end and candidate.start < used.end and used.start < candidate.end {
            throw abi::Error::Busy;
        }
    }

        }
    }
    return false;
}

/// Create an additional context with private kernel storage and shared domain state.
/// The caller serializes domain, capability, and physical allocation metadata.
export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
    table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
/// Private kernel storage retained by an unpublished context.
export record ContextAllocation: Copy {
    /// Reserved context slot with an initialized user-stack claim.
    object: abi::Ref,
    /// Domain kept live until publication or cancellation.
    owner: abi::Ref,
    /// Reserved kernel-stack frames.
    frames: frames::Run,
    /// Validated physical base for private stack clearing.
    base: u64,
}

/// Context storage that must be published or cancelled under serialization.
export union ContextReservation: Once {
    /// One reserved context and its private kernel storage.
    Held(ContextAllocation),
}

/// Reserve an additional context and both stack claims under metadata serialization.
export unsafe fn contextReserve(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
    table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> ContextReservation throws (abi::Error)
{
    let permission = try capability::authority(table, authority, abi::Rights(abi::EXECUTE));
    let domain = try get(store, permission.object);
    if domain.state == Lifecycle::Dead { throw abi::Error::BadHandle; }
    if start.entry == 0 or not exportedEntry(&domain, packages, start.entry)

    let pending = try frames::reserve(&mut memory.backings.pool, KERNEL_STACK_PAGES) catch error {
        try! slots::cancel(&mut store.contextSlots[..], slot); throw error;
    };
    let kernelFrames = frames::commit(pending);
    let kernelStack = try! frames::extent(&memory.backings.pool, kernelFrames);
    zero(kernelStack.start, kernelFrames.count);
    let mut frame = trap::Frame { registers: [0; 32], pc: start.entry, status: 0x80, cause: 0, value: 0 };
    set frame.registers[1] = returnAddress();
    set frame.registers[2] = start.stack;
    set frame.registers[3] = domain.graph.table.ptr as u64;
    set frame.registers[10] = start.args;
    set frame.registers[11] = start.size;
    let object = slots::reference(&slot);
    set store.contexts[object.index] = Context {
        owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
    };
    return try! slots::commit(&mut store.contextSlots[..], slot);
    match slot {
        case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
            object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
        }),
    }
}

/// Clear an unpublished context's private kernel stack.
export fn contextClear(reservation: &ContextReservation) {
    match reservation { case ContextReservation::Held(allocation) => zero(allocation.base, allocation.frames.count), }
}

/// Check the reserved context and its retained physical allocation.
fn contextRequired(store: &Store, allocation: &ContextAllocation) {
    assert slots::matches(&store.contextSlots[..], allocation.object, slots::State::Reserved);
    let context = store.contexts[allocation.object.index];
    assert context.owner == allocation.owner and context.kernelFrames == allocation.frames;
    assert context.hart == nil;
}

/// Publish an initialized context while its domain stays live under serialization.
export fn contextPublish(store: &mut Store, reservation: ContextReservation) -> abi::Ref {
    match reservation {
        case ContextReservation::Held(allocation) => {
            contextRequired(store, &allocation);
            assert slots::matches(&store.slots[..], allocation.owner, slots::State::Live);
            return try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
        },
    }
}

/// Return an unpublished context's stack and slot under metadata serialization.
export fn contextCancel(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation) {
    match reservation {
        case ContextReservation::Held(allocation) => {
            contextRequired(store, &allocation);
            try! frames::release(&mut memory.backings.pool, allocation.frames);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
        },
    }
}

/// Create a context while the caller has exclusive access to shared metadata.
export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
    table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
{
    let pending = try contextReserve(store, memory, packages, table, authority, start);
    contextClear(&pending);
    return contextPublish(store, pending);
}

/// Memory allocated before the domain and its context become observable.
record Prepared: Copy {
export record Prepared: Copy {
    /// Frames for capabilities and the event ring.
    allocation: frames::Run,
    /// Mapped metadata at the allocation's start.
    memory: *unsafe mut Memory,
    /// Independent package-state graph.
    graph: instances::Instance,
    /// Reserved private package-state graph.
    graph: instances::Allocation,
    /// Retained stack allocation for the initial context.
    kernelFrames: frames::Run,
    /// Initial context's mapped stack bounds.
    kernelStack: range::Range,
}

/// Allocate private storage and return all earlier allocations on failure.
unsafe fn prepare(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
unsafe fn allocate(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
    let pending = try frames::reserve(pool, (@sizeOf(Memory) + 4095) / 4096);
    let allocation = frames::commit(pending);
    let graph = try instances::create(packages, pool, image) catch err {
    let graph = try instances::reserve(packages, pool, image) catch err {
        try! frames::release(pool, allocation); throw err;
    };
    let pendingStack = try frames::reserve(pool, KERNEL_STACK_PAGES) catch err {
        try! frames::release(pool, graph.frames);
        instances::cancel(pool, graph);
        try! frames::release(pool, allocation); throw err;
    };
    let kernelFrames = frames::commit(pendingStack);
    let kernelStack = try! frames::extent(pool, kernelFrames);
    let extent = try! frames::extent(pool, allocation);
    zero(extent.start, allocation.count);
    zero(kernelStack.start, kernelFrames.count);
    return Prepared { allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack };
    match graph {
        case instances::Reservation::Held(graph) => return Prepared {
            allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack,
        },
    }
}

/// Return storage that has never been exposed to a published domain.
fn discard(pool: &mut frames::Pool, prepared: Prepared) {
    try! frames::release(pool, prepared.kernelFrames);
    try! frames::release(pool, prepared.graph.frames);
    try! frames::release(pool, prepared.allocation);
}

/// Create an unscheduled domain with Events and one initial integer context.
/// The caller serializes publication and keeps the authorizing domain alive.
export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
/// Unpublished domain metadata retained during private initialization.
export record Creation: Copy {
    /// Calling domain kept live until publication or cancellation.
    owner: abi::Ref,
    /// Reserved capability slot in the calling domain.
    handle: abi::Ref,
    /// Reserved domain slot.
    domain: abi::Ref,
    /// Reserved initial-context slot.
    context: abi::Ref,
    /// Management and inherited authority selected at admission.
    rights: abi::Rights,
    /// Resident root package.
    image: abi::Ref,
    /// Validated initial entry address.
    entry: u64,
    /// Private allocations retained until the domain becomes observable.
    storage: Prepared,
}

/// Domain creation resources that must be published or cancelled.
export union Reservation: Once {
    /// One private domain and its reserved metadata capacity.
    Held(Creation),
}

/// Reserve domain capacity under metadata serialization while the caller stays live.
export unsafe fn reserve(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> Reservation throws (abi::Error)
{
    let permit = try capability::authority(table, authority, abi::Rights(abi::CREATE));
    if not backing::domainLive(backings, table.owner) or not backing::domainLive(backings, permit.object) {
        throw abi::Error::BadHandle;
    }

    };
    let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch err {
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let mut prepared = try prepare(&mut backings.pool, packages, root) catch err {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let object = slots::reference(&domainSlot);
    try backing::registerDomain(backings, object) catch err {
        discard(&mut backings.pool, prepared);
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    capability::initialize(&mut prepared.memory.table, object);
    try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
        try! backing::endDomain(backings, object);
        discard(&mut backings.pool, prepared);
    let initialSlot = slots::reference(&contextSlot);
    set store.contexts[initialSlot.index].userStack = range::Range { start: 0, end: 0 };
    let prepared = try allocate(&mut backings.pool, packages, root) catch err {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
        kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ | abi::WRITE),
    });
    let initial = slots::reference(&contextSlot);
    // MPIE enables interrupts after the first return into U-mode.
    let mut frame = trap::Frame { registers: [0; 32], pc: entry, status: 0x80, cause: 0, value: 0 };
    set frame.registers[1] = returnAddress();
    set frame.registers[3] = prepared.graph.table.ptr as u64;
    set store.contexts[initial.index] = Context {
        owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
        userStack: range::Range { start: 0, end: 0 }, frame,
    };
    set store.records[object.index] = Domain {
        state: Lifecycle::Pending, creator: table.owner, parent: table.owner, image: root, initial,
        allocation: prepared.allocation, memory: prepared.memory, graph: prepared.graph, events: queue,
    match handleSlot {
        case slots::Reservation::Held(handle) => match domainSlot {
            case slots::Reservation::Held(domain) => match contextSlot {
                case slots::Reservation::Held(context) => return Reservation::Held(Creation {
                    owner: table.owner, handle, domain, context,
                    rights: abi::Rights(MANAGEMENT_RIGHTS | (*permit.rights & (abi::CREATE | abi::ALLOCATE))),
                    image: root, entry, storage: prepared,
                }),
            },
        },
    }
}

/// Initialize private storage from resident packages while metadata can change.
export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
    match reservation {
        case Reservation::Held(creation) => {
            let mut storage = creation.storage;
            zero(storage.memory as u64, storage.allocation.count);
            zero(storage.kernelStack.start, storage.kernelFrames.count);
            try instances::fill(packages, &storage.graph.graph, storage.graph.base);
            capability::initialize(&mut storage.memory.table, creation.domain);
        },
    }
}

/// Check the unpublished capacity retained by one creation transaction.
fn require(store: &Store, table: &capability::Table, creation: &Creation) {
    assert table.owner == creation.owner;
    assert slots::matches(&table.slots[..], creation.handle, slots::State::Reserved);
    assert slots::matches(&store.slots[..], creation.domain, slots::State::Reserved);
    assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
}

/// Return private domain storage and reserved slots under metadata serialization.
export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            discard(&mut backings.pool, creation.storage);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
        },
    }
}

/// Publish an initialized domain and its Events under metadata serialization.
export unsafe fn publish(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation)
    -> abi::Handle throws (abi::Error)
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            if not backing::domainLive(backings, creation.owner) {
                cancel(store, backings, table, Reservation::Held(creation));
                throw abi::Error::BadHandle;
            }
            let object = creation.domain;
            let initial = creation.context;
            let mut prepared = creation.storage;
            try backing::registerDomain(backings, object) catch err {
                cancel(store, backings, table, Reservation::Held(creation)); throw err;
            };
            try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
                try! backing::endDomain(backings, object);
                cancel(store, backings, table, Reservation::Held(creation)); throw err;
            };
            let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
                kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ | abi::WRITE),
            });
            let graph = instances::commit(instances::Reservation::Held(prepared.graph));
            // MPIE enables interrupts after the first return into U-mode.
            let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
            set frame.registers[1] = returnAddress();
            set frame.registers[3] = graph.table.ptr as u64;
            set store.contexts[initial.index] = Context {
                owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
            };
            let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
            let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
            return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
                kind: abi::Kind::Domain, object: domain, rights: creation.rights,
            });
        },
    }
}

/// Create a private domain while the caller has exclusive access to metadata.
export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(store, backings, packages, table, authority, image);
    try prepare(packages, &pending) catch err {
        cancel(store, backings, table, pending); throw err;
    };
    let context = try! slots::commit(&mut store.contextSlots[..], contextSlot);
    let domain = try! slots::commit(&mut store.slots[..], domainSlot);
    return capability::publish(table, handleSlot, capability::Entry {
        kind: abi::Kind::Domain, object: domain,
        rights: abi::Rights(MANAGEMENT_RIGHTS | (*permit.rights & (abi::CREATE | abi::ALLOCATE))),
    });
    return try publish(store, backings, table, pending);
}

kernel/kernel/instances.rad +52 -9

    /// Package bases indexed by resident slot; absent packages have zero bases.
    table: *unsafe [u64],
}

/// Bounded traversal and conservative storage requirement for a package graph.
record Graph: Copy {
export record Graph: Copy {
    /// Reachable package references, each present once.
    packages: [abi::Ref; limits::PACKAGES],
    /// Number of initialized package references.
    count: u32,
    /// Bytes for the base table, private state, and worst-case alignment padding.
    bytes: u64,
}

/// Private storage and the resident graph selected at reservation time.
export record Allocation: Copy {
    /// Reachable immutable packages and their storage bounds.
    graph: Graph,
    /// Physical frames reserved for this instance.
    frames: frames::Run,
    /// Validated physical base for private initialization.
    base: u64,
}

/// Graph storage that must be committed or returned to its frame pool.
export union Reservation: Once {
    /// An unpublished private instance.
    Held(Allocation),
}

/// Map an allocated physical extent to typed storage at the call site.
fn memory(address: u64) -> *unsafe mut opaque;

/// Collect the reachable immutable packages with a bounded queue.
unsafe fn graph(store: &registry::Store, root: abi::Ref) -> Graph throws (abi::Error) {

    }
    return result;
}

/// Assign every base before initializing templates and their private pointers.
unsafe fn initialize(store: &registry::Store, graph: &Graph, base: u64) -> *unsafe [u64] throws (abi::Error) {
/// The caller owns a writable extent of at least graph.bytes bytes at base.
export unsafe fn fill(store: &registry::Store, graph: &Graph, base: u64) throws (abi::Error) {
    let table = @sliceOf(memory(base) as *unsafe mut u64, limits::PACKAGES);
    for i in 0..table.len { set table[i] = 0; }
    let mut next = base + limits::PACKAGES as u64 * 8;
    for object in &graph.packages[..graph.count] {
        let package = try! registry::get(store, object);

    for object in &graph.packages[..graph.count] {
        let package = try! registry::get(store, object);
        let bytes = @sliceOf(memory(table[object.index]) as *unsafe mut u8, package.memory);
        try shared::instantiate(&package, &table[..], bytes) catch { throw abi::Error::VerifyFailed; };
    }
    return &table[..];
}

/// Allocate and initialize one private graph without copying resident code.
/// The caller serializes frame reservations and keeps the registry immutable.
export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
/// Reserve one graph under metadata serialization while its packages stay resident.
export unsafe fn reserve(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Reservation throws (abi::Error) {
    let layout = try graph(store, root);
    let count = ((layout.bytes + limits::FRAME_SIZE - 1) / limits::FRAME_SIZE) as u32;
    let pending = try frames::reserve(pool, count);
    let run = frames::commit(pending);
    let extent = try! frames::extent(pool, run);
    let table = try initialize(store, &layout, extent.start) catch err {
        try! frames::release(pool, run); throw err;
    return Reservation::Held(Allocation { graph: layout, frames: run, base: extent.start });
}

/// Initialize private state from the reservation's immutable resident packages.
export unsafe fn initialize(store: &registry::Store, reservation: &Reservation) throws (abi::Error) {
    match reservation {
        case Reservation::Held(allocation) => try fill(store, &allocation.graph, allocation.base),
    }
}

/// Consume initialized private storage for publication in a domain.
export unsafe fn commit(reservation: Reservation) -> Instance {
    match reservation {
        case Reservation::Held(allocation) => return Instance {
            frames: allocation.frames,
            table: @sliceOf(memory(allocation.base) as *unsafe u64, limits::PACKAGES),
        },
    }
}

/// Return unpublished graph storage under frame-pool serialization.
export fn cancel(pool: &mut frames::Pool, reservation: Reservation) {
    match reservation { case Reservation::Held(allocation) => { try! frames::release(pool, allocation.frames); }, }
}

/// Allocate and initialize a private graph while the caller has exclusive metadata access.
export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
    let pending = try reserve(store, pool, root);
    try initialize(store, &pending) catch err {
        cancel(pool, pending); throw err;
    };
    return Instance { frames: run, table };
    return commit(pending);
}

kernel/kernel/loader.rad +189 -45

export union Lease: Once {
    /// Private temporary frames owned by the active load.
    Held(frames::Run),
}

/// Private input capacity retained throughout one load.
export record Input: Copy {
    /// Domain kept live while the source and reserved handle are in use.
    owner: abi::Ref,
    /// Reserved Image handle slot in the caller's table.
    handle: abi::Ref,
    /// Private workspace frames.
    frames: frames::Run,
    /// Physical workspace base retained by the lease.
    work: u64,
    /// Validated physical source address.
    address: u64,
    /// Number of source bytes to snapshot.
    length: u32,
}

/// Input capacity that must be cancelled or completed under metadata serialization.
export union Reservation: Once {
    /// One admitted load and its private compiler storage.
    Held(Input),
}

/// Global runtime-loader ownership.
export unsafe static STATE: State = State { busy: false, owner: abi::Ref { index: 0, generation: 0 },
    live: [false; limits::PACKAGES], resident: undefined };

/// Map a validated physical workspace extent to its compiler storage.

        },
        else => return abi::Error::VerifyFailed,
    }
}

/// Used page counts for unpublished native output.
record OutputSize: Copy {
/// Used native extents and validated metadata ready for publication.
export record Compiled: Copy {
    /// Pages occupied by generated code.
    code: u32,
    /// Pages occupied by the packed native catalog.
    metadata: u32,
    /// Immutable descriptor and resolved dependencies for the reserved package.
    admission: registry::Admission,
}

/// Compile and pack one candidate into caller-owned physical output extents.
unsafe fn generate(work: *mut Workspace, input: &binary::Package, packages: &registry::Store,
    slot: u32, code: u64, metadata: u64, length: u32) -> OutputSize throws (abi::Error)
    object: abi::Ref, code: u64, metadata: u64, length: u32) -> Compiled throws (abi::Error)
{
    let count = try registry::imports(packages, &input.dependencies[..], &mut work.imports[..]);
    let mut arena = alloc::new(&mut work.code[..]);
    let mut scratch = alloc::new(&mut work.scratch[..]);
    let package = try shared::compile(&*input, slot, code, &work.imports[..count],
    let package = try shared::compile(&*input, object.index, code, &work.imports[..count],
        shared::Storage {
            data: &mut work.data[..], symbols: &mut work.symbols[..], exports: &mut work.exports[..],
            template: &mut work.template[..], relocations: &mut work.relocations[..],
        }, &mut arena, &mut scratch) catch error { throw backend(error); };
    if package.code.len > CODE_BYTES / 4 { throw abi::Error::Exhausted; }

    let metadataMemory = @sliceOf(memory(metadata), METADATA_BYTES);
    let used = try catalog::pack(&[catalog::Entry { source: &work.input[..length], package }], metadata, metadataMemory)
        catch error { throw backend(error); };
    let mut codePages = (codeBytes + 4095) / 4096;
    if codePages == 0 { set codePages = 1; }
    return OutputSize { code: codePages, metadata: (used + 4095) / 4096 };
    let native = descriptor(metadata);
    let admission = try registry::admit(packages, object, native.source, native.package);
    return Compiled { code: codePages, metadata: (used + 4095) / 4096, admission };
}

/// Return unused output frames while preserving at least one frame for each extent.
fn trim(pool: &mut frames::Pool, run: frames::Run, count: u32) -> frames::Run {
    assert count > 0 and count <= run.count;
    if count < run.count { try! frames::release(pool, frames::Run { first: run.first + count, count: run.count - count }); }
    return frames::Run { first: run.first, count };
}

/// Snapshot, decode, compile, and publish a package while its caller remains live.
unsafe fn produce(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
    work: *mut Workspace, address: u64, length: u32) -> abi::Ref throws (abi::Error)
{
    try! mem::copy(&mut work.input[..length], @sliceOf(memory(address), length));
    let mut decoder = alloc::new(&mut work.decoded[..]);
    let input = try program::decode(&work.input[..length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
        if err == binary::Error::Storage { throw abi::Error::Exhausted; }
        throw abi::Error::VerifyFailed;
    };
    let existing = try registry::identify(packages, &input.name[..], &work.input[..length]);
    if let object = existing { return object; }
/// Snapshot and decode trusted input into the held private workspace.
/// The decoded package stays valid until this reservation is completed or cancelled.
export unsafe fn decode(reservation: &Reservation) -> binary::Package throws (abi::Error) {
    match reservation {
        case Reservation::Held(input) => {
            let mut work = workspace(input.work);
            try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
            let mut decoder = alloc::new(&mut work.decoded[..]);
            return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
                if err == binary::Error::Storage { throw abi::Error::Exhausted; }
                throw abi::Error::VerifyFailed;
            };
        },
    }
}

/// Reserved native output and its immutable physical extents.
export record Output: Copy {
    /// Reserved package slot selected for generated state references.
    object: abi::Ref,
    /// Private executable frames.
    code: frames::Run,
    /// Private source and descriptor frames.
    metadata: frames::Run,
    /// Validated physical executable base.
    codeBase: u64,
    /// Validated physical descriptor base.
    metadataBase: u64,
}

/// Native output that must be published or cancelled under serialization.
export union OutputReservation: Once {
    /// One unpublished package and its private output frames.
    Held(Output),
}

/// Reserve a package slot and native output under metadata serialization.
export fn reserveOutput(pool: &mut frames::Pool, packages: &mut registry::Store) -> OutputReservation throws (abi::Error) {
    let slot = try registry::reserve(packages);
    let object = slots::reference(&slot);
    let codeReservation = try frames::reserve(pool, CODE_BYTES / 4096) catch err { registry::cancel(packages, slot); throw err; };
    let code = frames::commit(codeReservation);
    let metadataReservation = try frames::reserve(pool, METADATA_BYTES / 4096) catch err {
        try! frames::release(pool, code); registry::cancel(packages, slot); throw err;
    };
    let metadata = frames::commit(metadataReservation);
    let codeRange = try! frames::extent(pool, code);
    let metadataRange = try! frames::extent(pool, metadata);
    let sizes = try generate(work, &input, packages, object.index, codeRange.start, metadataRange.start, length) catch err {
        try! frames::release(pool, metadata); try! frames::release(pool, code);
        registry::cancel(packages, slot); throw err;
    };
    let retainedCode = trim(pool, code, sizes.code);
    let retainedMetadata = trim(pool, metadata, sizes.metadata);
    let native = descriptor(metadataRange.start);
    sync::syncInstructions();
    let result = try registry::publish(packages, slot, native.source, native.package) catch err {
        try! frames::release(pool, retainedMetadata); try! frames::release(pool, retainedCode); throw err;
    };
    assert result == object;
    set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
    set state.live[result.index] = true;
    return result;
    match slot {
        case slots::Reservation::Held(object) => return OutputReservation::Held(Output {
            object, code, metadata, codeBase: codeRange.start, metadataBase: metadataRange.start,
        }),
    }
}

/// Load trusted RIL and install an Image capability after complete native publication.
/// The caller keeps its domain alive and serializes shared metadata transactions.
export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
    table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
/// Compile a decoded package into reserved private output using resident dependencies.
export unsafe fn compile(packages: &registry::Store, reservation: &Reservation, input: &binary::Package, output: &OutputReservation)
    -> Compiled throws (abi::Error)
{
    match reservation {
        case Reservation::Held(source) => match output {
            case OutputReservation::Held(target) => {
                let work = workspace(source.work);
                return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
            },
        },
    }
}

/// Return unpublished output frames and its package slot under serialization.
export fn cancelOutput(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation) {
    match reservation {
        case OutputReservation::Held(output) => {
            assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
            try! frames::release(pool, output.metadata);
            try! frames::release(pool, output.code);
            registry::cancel(packages, slots::Reservation::Held(output.object));
        },
    }
}

/// Publish completed native output and retain its used frames under serialization.
export fn publishOutput(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
    reservation: OutputReservation, sizes: Compiled) -> abi::Ref
{
    match reservation {
        case OutputReservation::Held(output) => {
            assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
            let retainedCode = trim(pool, output.code, sizes.code);
            let retainedMetadata = trim(pool, output.metadata, sizes.metadata);
            sync::syncInstructions();
            let result = registry::commit(packages, slots::Reservation::Held(output.object), sizes.admission);
            assert result == output.object;
            set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
            set state.live[result.index] = true;
            return result;
        },
    }
}

/// Compare the snapshot with immutable resident content while the workspace is held.
export unsafe fn identify(packages: &registry::Store, reservation: &Reservation, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
    match reservation {
        case Reservation::Held(source) => return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]),
    }
}

/// Reserve an Image handle and exclusive workspace under metadata serialization.
export unsafe fn reserve(state: &mut State, store: &mut pages::Store,
    table: &mut capability::Table, request: Request) -> Reservation throws (abi::Error)
{
    let address = try source(store, table, request);
    let handle = try slots::reserve(&mut table.slots[..]);
    let lease = try acquire(state, &mut store.backings.pool, table.owner) catch err {
        try! slots::cancel(&mut table.slots[..], handle); throw err;
    };
    let work = storage(&store.backings.pool, &lease);
    let object = try produce(state, &mut store.backings.pool, packages, work, address, request.length as u32) catch err {
        release(state, &mut store.backings.pool, lease);
        try! slots::cancel(&mut table.slots[..], handle); throw err;
    let work = storage(&store.backings.pool, &lease) as u64;
    match handle {
        case slots::Reservation::Held(handle) => match lease {
            case Lease::Held(frames) => return Reservation::Held(Input {
                owner: table.owner, handle, frames, work, address, length: request.length as u32,
            }),
        },
    }
}

/// Check workspace ownership and the caller's unpublished handle slot.
fn require(state: &State, table: &capability::Table, input: &Input) {
    assert state.busy and state.owner == input.owner and table.owner == input.owner;
    assert slots::matches(&table.slots[..], input.handle, slots::State::Reserved);
}

/// Return the workspace and reserved handle under metadata serialization.
export fn cancel(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation) {
    match reservation {
        case Reservation::Held(input) => {
            require(state, table, &input);
            release(state, &mut store.backings.pool, Lease::Held(input.frames));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(input.handle));
        },
    }
}

/// Install a resident package handle and return the workspace under serialization.
export fn finish(state: &mut State, store: &mut pages::Store, table: &mut capability::Table,
    reservation: Reservation, object: abi::Ref) -> abi::Handle throws (abi::Error)
{
    match reservation {
        case Reservation::Held(input) => {
            require(state, table, &input);
            if not backing::domainLive(&store.backings, input.owner) {
                cancel(state, store, table, Reservation::Held(input)); throw abi::Error::BadHandle;
            }
            let result = capability::publish(table, slots::Reservation::Held(input.handle), capability::Entry {
                kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
            });
            release(state, &mut store.backings.pool, Lease::Held(input.frames));
            return result;
        },
    }
}

/// Load trusted RIL while the caller has exclusive access to shared metadata.
export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
    table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(state, store, table, request);
    let input = try decode(&pending) catch err { cancel(state, store, table, pending); throw err; };
    let existing = try identify(packages, &pending, &input) catch err { cancel(state, store, table, pending); throw err; };
    if let object = existing { return try finish(state, store, table, pending, object); }
    let output = try reserveOutput(&mut store.backings.pool, packages) catch err { cancel(state, store, table, pending); throw err; };
    let sizes = try compile(packages, &pending, &input, &output) catch err {
        cancelOutput(&mut store.backings.pool, packages, output);
        cancel(state, store, table, pending); throw err;
    };
    let result = capability::publish(table, handle, capability::Entry {
        kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
    });
    release(state, &mut store.backings.pool, lease);
    return result;
    let object = publishOutput(state, &mut store.backings.pool, packages, output, sizes);
    return try finish(state, store, table, pending, object);
}

kernel/kernel/pages.rad +93 -16

        if cursor == before { return false; }
    }
    return true;
}

/// Allocate and zero frames before publishing a page and its capability.
/// Private frames and unpublished slots held while memory is cleared.
export record Allocation: Copy {
    /// Calling domain whose table contains the reserved handle.
    owner: abi::Ref,
    /// Unpublished capability slot.
    handle: abi::Ref,
    /// Unpublished page-object slot.
    page: abi::Ref,
    /// Unpublished backing-allocation slot.
    backing: abi::Ref,
    /// Frames excluded from allocation until publication or cancellation.
    run: frames::Run,
    /// Validated physical base for private initialization.
    base: u64,
}

/// Allocation capacity that must be published or cancelled under serialization.
export union Reservation: Once {
    /// Private memory and its three reserved metadata slots.
    Held(Allocation),
}

/// Reserve page capacity under metadata serialization for private initialization.
/// Allocation authority must name the calling domain. Failure cancels reservations.
export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> abi::Handle throws (abi::Error)
export fn reserve(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> Reservation throws (abi::Error)
{
    if count == 0 { throw abi::Error::InvalidArg; }
    if count > limits::FRAMES as u64 { throw abi::Error::OutOfMemory; }
    let owner = table.owner;
    let permission = try capability::authority(table, authority, abi::Rights(abi::ALLOCATE));
    if permission.object <> owner { throw abi::Error::Denied; }
    if not backing::domainLive(&store.backings, owner) { throw abi::Error::BadHandle; }
    let backings = &mut store.backings;
    if not backing::domainLive(backings, owner) { throw abi::Error::BadHandle; }
    let handleSlot = try slots::reserve(&mut table.slots[..]);
    let pageSlot = try slots::reserve(&mut store.slots[..]) catch err {
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let backingSlot = try slots::reserve(&mut store.backings.slots[..]) catch err {
    let backingSlot = try slots::reserve(&mut backings.slots[..]) catch err {
        try! slots::cancel(&mut store.slots[..], pageSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let memory = try frames::reserve(&mut store.backings.pool, count as u32) catch err {
        try! slots::cancel(&mut store.backings.slots[..], backingSlot);
    let memory = try frames::reserve(&mut backings.pool, count as u32) catch err {
        try! slots::cancel(&mut backings.slots[..], backingSlot);
        try! slots::cancel(&mut store.slots[..], pageSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let run = frames::commit(memory);
    let extent = try! frames::extent(&store.backings.pool, run);
    zero(extent.start, run.count);
    let allocation = backing::publish(&mut store.backings, backingSlot, run, owner);
    let object = slots::reference(&pageSlot);
    set store.records[object.index] = Page { backing: allocation, base: extent.start, count: run.count, origin: owner, handles: 1 };
    let page = try! slots::commit(&mut store.slots[..], pageSlot);
    return capability::publish(table, handleSlot, capability::Entry {
        kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
    });
    let extent = try! frames::extent(&backings.pool, run);
    match handleSlot {
        case slots::Reservation::Held(handle) => match pageSlot {
            case slots::Reservation::Held(page) => match backingSlot {
                case slots::Reservation::Held(backing) => return Reservation::Held(Allocation {
                    owner, handle, page, backing, run, base: extent.start,
                }),
            },
        },
    }
}

/// Clear private frames while shared metadata can be used by other harts.
export fn clear(reservation: &Reservation) {
    match reservation { case Reservation::Held(allocation) => zero(allocation.base, allocation.run.count), }
}

/// Check reserved generations before publishing or cancelling an allocation.
fn require(store: &Store, table: &capability::Table, allocation: &Allocation) {
    assert table.owner == allocation.owner;
    assert slots::matches(&table.slots[..], allocation.handle, slots::State::Reserved);
    assert slots::matches(&store.slots[..], allocation.page, slots::State::Reserved);
    assert slots::matches(&store.backings.slots[..], allocation.backing, slots::State::Reserved);
}

/// Publish cleared memory under metadata serialization while its owner stays live.
export unsafe fn publish(store: &mut Store, table: &mut capability::Table, reservation: Reservation) -> abi::Handle {
    match reservation {
        case Reservation::Held(allocation) => {
            require(store, table, &allocation);
            assert backing::domainLive(&store.backings, allocation.owner);
            let backing = backing::publish(&mut store.backings, slots::Reservation::Held(allocation.backing), allocation.run, allocation.owner);
            set store.records[allocation.page.index] = Page {
                backing, base: allocation.base, count: allocation.run.count, origin: allocation.owner, handles: 1,
            };
            let page = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(allocation.page));
            return capability::publish(table, slots::Reservation::Held(allocation.handle), capability::Entry {
                kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
            });
        },
    }
}

/// Return private allocation capacity under metadata serialization.
export fn cancel(store: &mut Store, table: &mut capability::Table, reservation: Reservation) {
    match reservation {
        case Reservation::Held(allocation) => {
            require(store, table, &allocation);
            try! frames::release(&mut store.backings.pool, allocation.run);
            try! slots::cancel(&mut store.backings.slots[..], slots::Reservation::Held(allocation.backing));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(allocation.page));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
        },
    }
}

/// Allocate zeroed pages while the caller has exclusive access to metadata.
export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(store, table, authority, count);
    clear(&pending);
    return publish(store, table, pending);
}

/// Split an exclusive page into two segments with the same backing and rights.
export fn split(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, leftCount: u64)
    -> abi::Handle throws (abi::Error)

kernel/kernel/registry.rad +39 -12

/// Return unpublished capacity after a failed load.
export fn cancel(store: &mut Store, reservation: slots::Reservation) {
    try! slots::cancel(&mut store.slots[..], reservation);
}

/// Validate dependencies without changing registry payloads.
unsafe fn validate(store: &Store, object: abi::Ref, package: &shared::Package) throws (abi::Error) {
/// Validated immutable package metadata ready for bounded publication.
export record Admission: Copy {
    /// Reserved package generation to receive the descriptor.
    object: abi::Ref,
    /// Persistent source and native descriptor.
    entry: catalog::Entry,
    /// Resolved resident dependencies in their declared order.
    dependencies: [abi::Ref; limits::PACKAGES],
    /// Number of initialized dependency references.
    count: u32,
}

/// Resolve admission against immutable packages while holding sole publisher ownership.
/// The caller keeps that ownership and the descriptor storage through commit or cancellation.
export unsafe fn admit(store: &Store, object: abi::Ref, source: *[u8], package: shared::Package) -> Admission throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Reserved) { throw abi::Error::BadHandle; }
    if try identify(store, &package.name[..], &source[..]) <> nil { throw abi::Error::Busy; }
    if package.name.len == 0 or package.slot <> object.index or package.dependencies.len > limits::PACKAGES {
        throw abi::Error::VerifyFailed;
    }
    let mut result = Admission { object, entry: catalog::Entry { source, package }, dependencies: undefined, count: package.dependencies.len };
    for dependency, i in package.dependencies {
        let target = find(store, &dependency[..]) else { throw abi::Error::VerifyFailed; };
        for prior in &package.dependencies[..i] {
            if mem::eq(prior, dependency) { throw abi::Error::VerifyFailed; }
        for prior in &result.dependencies[..i] {
            if prior == target { throw abi::Error::VerifyFailed; }
        }
        set result.dependencies[i] = target;
    }
    return result;
}

/// Publish admitted metadata under serialization using only bounded reference copies.
export fn commit(store: &mut Store, reservation: slots::Reservation, admission: Admission) -> abi::Ref {
    let object = slots::reference(&reservation);
    assert object == admission.object and slots::matches(&store.slots[..], object, slots::State::Reserved);
    assert admission.count <= limits::PACKAGES;
    for i in 0..admission.count {
        let target = admission.dependencies[i];
        assert slots::matches(&store.slots[..], target, slots::State::Live);
        set store.dependencies[object.index][i] = target;
    }
    set store.counts[object.index] = admission.count;
    set store.entries[object.index] = admission.entry;
    return try! slots::commit(&mut store.slots[..], reservation);
}

/// Publish persistent compiler output after successful admission checks.
/// All descriptor pointers and source bytes must remain immutable and resident.
/// Failure cancels the reservation; duplicate content returns its existing slot.

    let existing = try identify(store, &package.name[..], &source[..]) catch err {
        cancel(store, reservation); throw err;
    };
    if let object = existing { cancel(store, reservation); return object; }
    let object = slots::reference(&reservation);
    try validate(store, object, &package) catch err { cancel(store, reservation); throw err; };
    for dependency, i in package.dependencies {
        let target = find(store, &dependency[..]) else panic "resident dependency";
        set store.dependencies[object.index][i] = target;
    }
    set store.counts[object.index] = package.dependencies.len;
    set store.entries[object.index] = catalog::Entry { source, package };
    return try! slots::commit(&mut store.slots[..], reservation);
    let admission = try admit(store, object, source, package) catch err { cancel(store, reservation); throw err; };
    return commit(store, reservation, admission);
}

/// Read resident metadata through a generation-bearing package reference.
export fn get(store: &Store, object: abi::Ref) -> shared::Package throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) { throw abi::Error::BadHandle; }

kernel/kernel/sync.rad +60 -0

1	1		//! RV64 synchronization and ordered device access.
2	2
	3	+	/// Next metadata ticket, modulo 2^32. Each hart can hold at most one ticket.
	4	+	static NEXT: u32 = 0;
	5	+	/// Metadata ticket permitted to enter the critical section.
	6	+	static SERVING: u32 = 0;
	7	+	/// Largest observed instruction count inside the metadata lock.
	8	+	static MAXIMUM: u64 = 0;
	9	+
	10	+	/// Interrupt state and start counter retained by one metadata transaction.
	11	+	export record Section: Copy {
	12	+	/// Original machine interrupt-enable bit.
	13	+	interrupts: u64,
	14	+	/// Retired instruction count after lock acquisition.
	15	+	start: u64,
	16	+	/// Ticket held by the acquiring hart until release.
	17	+	ticket: u32,
	18	+	}
	19	+
	20	+	/// Lock ownership that must be released on its acquiring hart.
	21	+	export union Guard: Once {
	22	+	/// One serialized metadata transaction with machine interrupts masked.
	23	+	Held(Section),
	24	+	}
	25	+
	26	+	/// Mask machine interrupts and return the original enable bit.
	27	+	export fn maskInterrupts() -> u64;
	28	+	/// Restore the supplied machine interrupt-enable bit.
	29	+	export fn restoreInterrupts(value: u64);
	30	+	/// Read this hart's retired instruction counter.
	31	+	fn retired() -> u64;
	32	+
	33	+	/// Acquire shared metadata in ticket order with local interrupts masked.
	34	+	/// Calls must not nest. At most one ticket per online hart can be outstanding.
	35	+	export fn enter() -> Guard {
	36	+	let interrupts = maskInterrupts();
	37	+	let ticket = nextTicket(&mut NEXT);
	38	+	while loadAcquire32(&SERVING) <> ticket {}
	39	+	let start = retired();
	40	+	return Guard::Held(Section { interrupts, start, ticket });
	41	+	}
	42	+
	43	+	/// Publish metadata and restore the acquiring hart's interrupt state.
	44	+	export fn leave(guard: Guard) {
	45	+	match guard {
	46	+	case Guard::Held(section) => {
	47	+	let work = retired() - section.start;
	48	+	if work > MAXIMUM { set MAXIMUM = work; }
	49	+	storeRelease32(&mut SERVING, ((section.ticket as u64 + 1) & 0xffffffff) as u32);
	50	+	restoreInterrupts(section.interrupts);
	51	+	},
	52	+	}
	53	+	}
	54	+
	55	+	/// Read the maximum measured metadata work under serialization.
	56	+	export fn maximum() -> u64 {
	57	+	let guard = enter();
	58	+	let value = MAXIMUM;
	59	+	leave(guard);
	60	+	return value;
	61	+	}
	62	+
3	63		/// Allocate one wrapping ticket from a naturally aligned u32 counter.
4	64		export fn nextTicket(counter: &mut u32) -> u32;
5	65		/// Read a naturally aligned shared word with acquire ordering.
6	66		export fn loadAcquire(value: &u64) -> u64;
7	67		/// Publish a naturally aligned shared word with release ordering.

kernel/kernel/sync.ras +24 -0

.export @kernel::sync::fetchAdd;
.export @kernel::sync::syncInstructions;
.export @kernel::sync::deviceFence;
.export @kernel::sync::read32;
.export @kernel::sync::write32;
.export @kernel::sync::maskInterrupts;
.export @kernel::sync::restoreInterrupts;
.export @kernel::sync::retired;

// Preserve the previous interrupt-enable bit while masking machine interrupts.
@kernel::sync::maskInterrupts
    csrr %t0 mstatus;
    andi %a0 %t0 8;
    andi %t0 %t0 -9;
    csrw mstatus %t0;
    ret;

// Restore interrupt delivery while preserving other machine status fields.
@kernel::sync::restoreInterrupts
    csrr %t0 mstatus;
    andi %t0 %t0 -9;
    or %t0 %t0 %a0;
    csrw mstatus %t0;
    ret;

// Count local retired instructions independently of other harts' progress.
@kernel::sync::retired
    csrr %a0 instret;
    ret;

// Allocate a wrapping ticket and return its zero-extended u32 value.
@kernel::sync::nextTicket
    li %t0 1;
    amoadd.w.aqrl %a0 %t0 (%a0);

kernel/kernel/tests/calls.rad +1 -1

/// An unknown user operation advances PC and returns only a negative a0 error.
@test unsafe fn trapReply() throws (testing::TestError) {
    let owner = initialize();
    let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
    set frame.registers[17] = 0xffffffffffffffff;
    calls::handle(owner, &mut frame, 0);
    calls::handle(owner, &mut frame, 0, calls::invoke);
    try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc);
    for i in 0..32 {
        if i == 10 or i == 17 { continue; }
        try testing::expect(frame.registers[i] == 123);
    }

kernel/kernel/tests/dispatch.rad +17 -0

    try testing::expect(ended.context == nil and ended.deadline == 0xffffffffffffffff);
    let other = try! dispatch::select(&BUDGETS, &DOMAINS, 1, 10);
    try testing::expect(other.context == nil and other.deadline == 0xffffffffffffffff);
}

/// A context becomes eligible only after its previous hart releases ownership.
@test unsafe fn ownership() throws (testing::TestError) {
    initialize();
    set DOMAINS.contexts[0].hart = 1;
    let future = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
    assert future.context == nil and future.deadline == 10;
    let occupied = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
    assert occupied.context == nil and occupied.deadline == 20;
    set DOMAINS.contexts[0].hart = nil;
    let free = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
    assert free.context == TABLE.owner and free.deadline == 20;
    set DOMAINS.contexts[0].hart = 0;
    let local = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
    assert local == free;
}

/// Pending domains and stale context generations cannot become runnable through a budget.
@test unsafe fn lifetimes() throws (testing::TestError) {
    initialize();
    set DOMAINS.records[0].state = domains::Lifecycle::Pending;
    let pending = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

        stack: range::Range { start: 0x2000, end: 0x4000 },
    };
    let mut anchor = trap::Hart {
        stackTop: 0x4000, stackBottom: 0x2000, kernelGp: 0x20000, handler,
        savedT0: 0, savedT1: 0, savedSp: 0,
        dispatchTop: 0x4000, dispatchFrame: undefined,
    };
    let first = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 10);
    try testing::expect(frame.pc == 0x40 and anchor.stackTop == 0xc000);
    try testing::expect(DOMAINS.contexts[0].hart == 0);
    set frame.pc = 0x88;

kernel/kernel/tests/domains.rad +68 -1

    try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
        try testing::expect(error == abi::Error::OutOfMemory); set exhausted = true;
    };
    try testing::expect(exhausted and DOMAINS.contextSlots[child.initial.index + 1].state == slots::State::Free);
    set PAGES.backings.pool.count = count;
    let context = try! domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    let reserved = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    let index = child.initial.index + 1;
    assert DOMAINS.contextSlots[index].state == slots::State::Reserved;
    let mut overlap = false;
    try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
        assert error == abi::Error::Busy; set overlap = true;
    };
    assert overlap;
    let reservedFrames = DOMAINS.contexts[index].kernelFrames;
    domains::contextCancel(&mut DOMAINS, &mut PAGES, reserved);
    assert DOMAINS.contextSlots[index].state == slots::State::Free;
    for i in reservedFrames.first..reservedFrames.first + reservedFrames.count { assert PAGES.backings.pool.free[i]; }
    let pendingContext = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    let stackBase = DOMAINS.contexts[index].kernelStack.start;
    let base = (&RAM[0]) as u64;
    let firstWord = ((stackBase - base) / 8) as u32;
    let lastWord = firstWord + domains::KERNEL_STACK_PAGES * 512 - 1;
    set RAM[firstWord] = 123; set RAM[lastWord] = 456;
    domains::contextClear(&pendingContext);
    assert RAM[firstWord] == 0 and RAM[lastWord] == 0;
    let context = domains::contextPublish(&mut DOMAINS, pendingContext);
    let created = try! domains::context(&DOMAINS, authority.object, context);
    let initial = DOMAINS.contexts[child.initial.index];
    try testing::expect(context <> child.initial and created.owner == authority.object);
    try testing::expect(created.frame.registers[3] == initial.frame.registers[3]);
    try testing::expect(created.kernelStack.start >= initial.kernelStack.end or created.kernelStack.end <= initial.kernelStack.start);

        assert (error == abi::Error::BadHandle); set rejected += 1;
    };
    assert (rejected == 3);
}

/// Domain reservations retain private capacity across another complete creation.
@test unsafe fn stagedCreation() throws (testing::TestError) {
    initialize(abi::CREATE);
    let pending = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    assert DOMAINS.slots[1].state == slots::State::Reserved;
    assert DOMAINS.contextSlots[0].state == slots::State::Reserved;
    assert TABLE.slots[2].state == slots::State::Reserved;
    assert BACKINGS.domains[1] == 0 and DOMAINS.events.queues[1].generation == 0;
    let other = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    try! domains::prepare(&PACKAGES, &pending);
    let first = try! domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, pending);
    let a = try! capability::get(&TABLE, first);
    let b = try! capability::get(&TABLE, other);
    let left = try! domains::get(&DOMAINS, a.object);
    let right = try! domains::get(&DOMAINS, b.object);
    assert a.object.index == 1 and b.object.index == 2;
    assert left.graph.table[0] <> right.graph.table[0];
    assert left.state == domains::Lifecycle::Pending and right.state == domains::Lifecycle::Pending;
    initialize(abi::CREATE);
    let cancelled = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    domains::cancel(&mut DOMAINS, &mut BACKINGS, &mut TABLE, cancelled);
    assert DOMAINS.slots[1].state == slots::State::Free;
    assert DOMAINS.contextSlots[0].state == slots::State::Free;
    assert TABLE.slots[2].state == slots::State::Free;
    for i in 0..BACKINGS.pool.count { assert BACKINGS.pool.free[i]; }
    set PACKAGES.entries[0].package.relocations = &[shared::Relocation {
        offset: 0, count: 1, target: shared::DataRef { slot: 5, offset: 0 },
    }];
    let mut invalid = false;
    try domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE) catch error {
        assert error == abi::Error::VerifyFailed; set invalid = true;
    };
    assert invalid and DOMAINS.slots[1].state == slots::State::Free;
    for i in 0..BACKINGS.pool.count { assert BACKINGS.pool.free[i]; }
    initialize(abi::CREATE);
    let abandoned = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    try! domains::prepare(&PACKAGES, &abandoned);
    try! backing::endDomain(&mut BACKINGS, TABLE.owner);
    let mut dead = false;
    try domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, abandoned) catch error {
        assert error == abi::Error::BadHandle; set dead = true;
    };
    assert dead and DOMAINS.slots[1].state == slots::State::Free;
    assert TABLE.slots[2].state == slots::State::Free;
    for i in 0..BACKINGS.pool.count { assert BACKINGS.pool.free[i]; }
}

/// Creation installs only Events in the child and preserves selected authorizing rights.
@test unsafe fn pending() throws (testing::TestError) {
    initialize(abi::CREATE | abi::ALLOCATE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let authority = try! capability::get(&TABLE, handle);

kernel/kernel/tests/instances.rad +21 -0

fn word(address: u64) -> *mut u64 {
    let pointer = &RAM[0];
    return &mut RAM[((address - pointer as u64) / 8) as u32];
}

/// Private initialization retains its graph across other registry and frame transactions.
@test unsafe fn stagedGraph() throws (testing::TestError) {
    initialize();
    let root = register("root", &[], &[]);
    let pending = try! instances::reserve(&PACKAGES, &mut POOL, root);
    let later = register("later", &[], &[]);
    let other = try! instances::create(&PACKAGES, &mut POOL, later);
    set *word(other.table[later.index]) = 99;
    try! instances::initialize(&PACKAGES, &pending);
    let first = instances::commit(pending);
    assert first.table[root.index] <> 0 and first.table[later.index] == 0;
    assert *word(first.table[root.index]) == 7;
    assert *word(other.table[later.index]) == 99;
    assert first.frames.first + first.frames.count <= other.frames.first;
    try! frames::release(&mut POOL, first.frames);
    try! frames::release(&mut POOL, other.frames);
    let cancelled = try! instances::reserve(&PACKAGES, &mut POOL, root);
    instances::cancel(&mut POOL, cancelled);
    for i in 0..POOL.count { assert POOL.free[i]; }
}

/// A diamond has one dependency instance per domain and independent relocated pointers.
@test unsafe fn diamond() throws (testing::TestError) {
    initialize();
    let base = register("base", &[], &[]);
    unsafe static relocation: [shared::Relocation; 1] = undefined;

kernel/kernel/tests/loader.rad +62 -0

    let mut count: u32 = 0;
    for i in 0..PAGES.backings.pool.count { if PAGES.backings.pool.free[i] { set count += 1; } }
    return count;
}

/// Reserved input stays private across allocation, source mutation, and contention.
@test unsafe fn stagedInput() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();
    let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
    assert LOADER.busy and TABLE.slots[2].state == slots::State::Reserved;
    let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
    let decoded = try! loader::decode(&pending);
    assert decoded.program.fns.len == 1 and decoded.name.len == 6;
    let replacement = input(99, false);
    assert decoded.program.fns[0].blocks[0].instrs[0] == il::Instr::Ret { val: il::Val::Imm(43) };
    let mut busy = false;
    try loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy;
    loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
    assert not LOADER.busy and TABLE.slots[2].state == slots::State::Free;
    assert free() + 1 == before;
    assert (try! pages::get(&PAGES, (try! capability::get(&TABLE, other)).object)).count == 1;
}

/// Compiled output stays private until publication and cancellation restores capacity.
@test unsafe fn stagedOutput() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();
    let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
    let decoded = try! loader::decode(&pending);
    let output = try! loader::reserveOutput(&mut PAGES.backings.pool, &mut PACKAGES);
    assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
    let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
    let used = try! loader::compile(&PACKAGES, &pending, &decoded, &output);
    assert used.code > 0 and used.metadata > 0;
    assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
    loader::cancelOutput(&mut PAGES.backings.pool, &mut PACKAGES, output);
    loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
    assert free() + 1 == before and not LOADER.busy;
    assert PACKAGES.slots[0].state == slots::State::Free;
    let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
    assert (try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0))).index == 0;
}

/// A caller whose backing lifetime ended cannot receive a completed Image handle.
@test unsafe fn completionLifetime() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
    let object = try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0));
    let before = free();
    let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
    try! backing::endDomain(&mut PAGES.backings, TABLE.owner);
    let mut dead = false;
    try loader::finish(&mut LOADER, &mut PAGES, &mut TABLE, pending, object) catch error {
        assert error == abi::Error::BadHandle; set dead = true;
    };
    assert dead and not LOADER.busy and free() == before;
    assert TABLE.slots[3].state == slots::State::Free;
    assert slots::matches(&PACKAGES.slots[..], object, slots::State::Live);
}

/// Resident source bytes survive mutation of the source page and workspace release.
@test unsafe fn residentSnapshot() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();

kernel/kernel/tests/pages.rad +37 -0

/// Drop the test's handle and its object reference in the same serialized operation.
unsafe fn drop(handle: abi::Handle) {
    try! pages::drop(&mut STORE, &mut TABLE, handle);
}

/// Reserved frames stay private while another allocation changes shared metadata.
@test unsafe fn stagedAllocation() throws (testing::TestError) {
    initialize();
    let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 2);
    assert STORE.slots[0].state == slots::State::Reserved;
    assert TABLE.slots[1].state == slots::State::Reserved;
    assert not STORE.backings.pool.free[0] and not STORE.backings.pool.free[1];
    assert RAM[OFFSET] == 0xa5;
    let other = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 6);
    set RAM[OFFSET + 8192] = 0x7b;
    pages::clear(&pending);
    for i in OFFSET..OFFSET + 8192 { assert RAM[i] == 0; }
    assert RAM[OFFSET + 8192] == 0x7b;
    let handle = pages::publish(&mut STORE, &mut TABLE, pending);
    let entry = try! capability::get(&TABLE, handle);
    let page = try! pages::get(&STORE, entry.object);
    assert page.count == 2 and page.origin == TABLE.owner;
    drop(handle); drop(other);
    try! backing::endDomain(&mut STORE.backings, TABLE.owner);
    for i in 0..8 { assert STORE.backings.pool.free[i]; }
}

/// Cancelling unpublished storage returns all capacity without exposing its bytes.
@test unsafe fn cancelAllocation() throws (testing::TestError) {
    initialize();
    let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 8);
    pages::cancel(&mut STORE, &mut TABLE, pending);
    assert STORE.slots[0].state == slots::State::Free;
    assert STORE.backings.slots[0].state == slots::State::Free;
    assert TABLE.slots[1].state == slots::State::Free;
    for i in 0..8 { assert STORE.backings.pool.free[i]; }
    assert RAM[OFFSET] == 0xa5;
    let full = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
    let entry = try! capability::get(&TABLE, full);
    assert (try! pages::get(&STORE, entry.object)).count == 8;
}

/// Transfers keep both domains exposed without duplicating page handles.
@test unsafe fn transferExposure() throws (testing::TestError) {
    initialize();
    let receiver = abi::Ref { index: 1, generation: 1 };
    capability::initialize(&mut TARGET, receiver);

kernel/kernel/tests/registry.rad +24 -0

    let pending = try registry::reserve(&mut STORE);
    let slot = slots::reference(&pending);
    return try registry::publish(&mut STORE, pending, bytes, package(name, dependencies, slot.index));
}

/// Admission resolves dependencies before its reserved package becomes visible.
@test unsafe fn stagedAdmission() throws (testing::TestError) {
    registry::initialize(&mut STORE);
    let base = try! register("base", &[], "base bytes");
    let pending = try! registry::reserve(&mut STORE);
    let object = slots::reference(&pending);
    let admitted = try! registry::admit(&STORE, object, "child bytes", package("child", &["base"], object.index));
    assert STORE.slots[object.index].state == slots::State::Reserved;
    assert registry::find(&STORE, &"child"[..]) == nil;
    assert admitted.dependencies[0] == base;
    let published = registry::commit(&mut STORE, pending, admitted);
    assert published == object and STORE.counts[object.index] == 1;
    assert STORE.dependencies[object.index][0] == base;
    let rejected = try! registry::reserve(&mut STORE);
    let slot = slots::reference(&rejected);
    let mut duplicate = false;
    try registry::admit(&STORE, slot, "bad bytes", package("bad", &["base", "base"], slot.index)) catch error {
        assert error == abi::Error::VerifyFailed; set duplicate = true;
    };
    assert duplicate and STORE.slots[slot.index].state == slots::State::Reserved;
    registry::cancel(&mut STORE, rejected);
    assert STORE.slots[slot.index].state == slots::State::Free;
}

/// Identical bytes reuse one package; a name cannot identify different content.
@test unsafe fn identity() throws (testing::TestError) {
    registry::initialize(&mut STORE);
    let first = try! register("a", &[], "binary a");
    let second = try! register("a", &[], "binary a");

kernel/kernel/tests/trap.rad +1 -1

use kernel::trap;

/// The assembly frame includes x0-x31 and all four trap CSRs.
@test fn layout() throws (testing::TestError) {
    try testing::expect(@sizeOf(trap::Frame) == 288);
    try testing::expect(@sizeOf(trap::Hart) == 56);
    assert @sizeOf(trap::Hart) == 352;
    try testing::expect(trap::FRAME_SIZE % 16 == 0);
}

/// Interrupts retain their source identity; exceptions retain their fault class.
@test fn classification() throws (testing::TestError) {

kernel/kernel/trap.rad +5 -1

    savedT0: u64,
    /// Temporary preservation of x6 during entry.
    savedT1: u64,
    /// Interrupted stack pointer, before stack selection.
    savedSp: u64,
    /// Firmware stack top used while releasing and selecting context ownership.
    dispatchTop: u64,
    /// Hart-owned frame used after the interrupted context's stack is released.
    dispatchFrame: Frame,
}

/// Trap classes consumed by kernel dispatch.
export union Cause: Copy {
    /// Machine software interrupt.


/// Install trap entry for the current hart with machine interrupts disabled.
export fn install(hart: *mut Hart);
/// Restore a complete integer context through mret. The current mscratch must
/// name an installed Hart, and the frame must have machine interrupts disabled.
export fn resume(frame: &Frame) -> !;
export fn resume(frame: *Frame) -> !;

/// Disable machine interrupts and stop the current hart.
export fn halt() -> !;

lib/std/arch/rv64/asm.rad +2 -1

    { name: "tp",   reg: rv64::TP },
    { name: "zero", reg: rv64::ZERO },
];

/// Sorted CSR-name lookup table used by the assembler parser.
export constant CSRS: [CsrEntry; 9] = [
export constant CSRS: [CsrEntry; 10] = [
    { name: "instret",  csr: 0xC02 },
    { name: "mcause",   csr: 0x342 },
    { name: "mepc",     csr: 0x341 },
    { name: "mhartid",  csr: 0xF14 },
    { name: "mie",      csr: 0x304 },
    { name: "mip",      csr: 0x344 },

lib/std/arch/rv64/asm/tests.rad +6 -0

static ASM_DATA_STORAGE: [u8; 1024] = [0; 1024];
unsafe static ASM_STRING_POOL: strings::Pool = strings::Pool { table: undefined, count: 0 };
static PRINT_ARENA_STORAGE: [u8; 1024] = [0; 1024];
static PRINT_BUFFER: [u8; 128] = [0; 128];

/// The retired-instruction CSR assembles to its exact architectural encoding.
@test unsafe fn retiredCounter() throws (testing::TestError) {
    let program = try assembleSource(".text;\ncsrr %a0 instret;\n");
    assert program.text.len == 1 and program.text[0] == 0xc0202573;
}

/// Source buffer for comment capacity tests.
static SOURCE: [u8; 4096] = [0; 4096];

/// Long comments consume source space while leaving symbol and fixup demand small.
@test unsafe fn commentStorage() throws (testing::TestError) {

test/dispatch/run +9 -5

#!/bin/sh
# Preempt and resume U-mode and M-mode work through production dispatch.
set -eu
emulator=${RAD_EMULATOR:-emulator}
fixture=${1:-dispatch}
harts=${2:-1}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp test/dispatch/spin.rad "$work/spin.rad"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"
cp test/dispatch/kernel/dispatchcheck.rad "$work/kernel/"
cp "test/$fixture/kernel/dispatchcheck.rad" "$work/kernel/"
length=$(wc -c < "$work/spin.ril")
printf '//! Binary user loop.\n/// Complete trusted input package.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/dispatchinput.rad"
od -An -v -tu1 "$work/spin.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/dispatch/machine.ras kernel/kernel/*.ras > "$work/dispatch.ras"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/dispatch.rv64"
printf 'dispatch: user calls, preemption, idle gap, and retained machine continuation passed\n'
for count in $harts; do
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
    printf '%s: %s-hart execution passed\n' "$fixture" "$count"
done

test/smp/kernel/dispatchcheck.rad added +149 -0

1	+	//! One domain executing separate user contexts on every online hart.
2	+	use std::mem;
3	+	use std::arch::rv64::shared;
4	+	use kernel::abi;
5	+	use kernel::slots;
6	+	use kernel::backing;
7	+	use kernel::pages;
8	+	use kernel::capability;
9	+	use kernel::registry;
10	+	use kernel::loader;
11	+	use kernel::domains;
12	+	use kernel::budgets;
13	+	use kernel::dispatch;
14	+	use kernel::boot;
15	+	use kernel::sync;
16	+	use kernel::dispatchinput;
17	+
18	+	/// Bootstrap resource authority.
19	+	unsafe static TABLE: capability::Table = undefined;
20	+	/// Release/acquire barrier after fixture publication.
21	+	static READY: u64 = 0;
22	+	/// Shared user domain generation.
23	+	unsafe static USER: abi::Ref = undefined;
24	+	/// Per-hart user execution contexts.
25	+	unsafe static CONTEXTS: [abi::Ref; 8] = undefined;
26	+	/// Mapped progress counters in the user's private state.
27	+	static COUNTERS: u64 = 0;
28	+	/// Validated physical memory mapping.
29	+	fn memory(address: u64) -> *mut u8;
30	+	/// Current kernel package-state table.
31	+	fn kernelGp() -> u64;
32	+	/// M-mode entry of the completion checker.
33	+	fn completion() -> u64;
34	+	/// Write one diagnostic byte to the fixture UART.
35	+	fn put(byte: u8);
36	+	/// Finish the machine test successfully.
37	+	fn finish();
38	+
39	+	/// Publish fixture state on the last initialized hart and wait on every hart.
40	+	export unsafe fn start(last: bool) {
41	+	if last { setup(); sync::storeRelease(&mut READY, 1); }
42	+	while sync::loadAcquire(&READY) == 0 {}
43	+	}
44	+
45	+	/// Create one context per online hart before any dispatch can access the fixture.
46	+	unsafe fn setup() {
47	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
48	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
49	+	capability::initialize(&mut TABLE, owner);
50	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
51	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
52	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
53	+	});
54	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
55	+	let sourceEntry = try! capability::get(&TABLE, source);
56	+	let page = try! pages::get(&pages::STORE, sourceEntry.object);
57	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
58	+	let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
59	+	let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
60	+	loader::Request { authority, source, offset: 0, length: length as u64 });
61	+	let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
62	+	let cap = try! capability::get(&TABLE, handle);
63	+	set USER = cap.object;
64	+	let mut user = try! domains::get(&domains::STORE, USER);
65	+	let self = try! capability::install(&mut user.memory.table, capability::Entry {
66	+	kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE),
67	+	});
68	+	let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);
69	+	let case shared::Target::Function(entry) = target else panic "worker entry";
70	+	let counterTarget = try! registry::exported(&registry::STORE, user.image, &"spin::count"[..]);
71	+	let case shared::Target::Data(data) = counterTarget else panic "counter state";
72	+	set COUNTERS = user.graph.table[data.slot] + data.offset as u64;
73	+	for hart in 0..8 {
74	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
75	+	let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
76	+	let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
77	+	let permission = try! capability::get(&TABLE, stack);
78	+	let page = try! pages::get(&pages::STORE, permission.object);
79	+	let argument = @sliceOf(memory(page.base), 8);
80	+	for i in 0..8 { set argument[i] = ((hart as u64) >> (i as u64 * 8)) as u8; }
81	+	if hart == 0 {
82	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, page.base + 4096, page.base, 8);
83	+	set CONTEXTS[hart] = user.initial;
84	+	} else {
85	+	set CONTEXTS[hart] = try! domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
86	+	&TABLE, handle, abi::ContextStart { entry, stack: page.base + 4096, args: page.base, size: 8 });
87	+	}
88	+	}
89	+	let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
90	+	let checkerCap = try! capability::get(&TABLE, checkerHandle);
91	+	let checker = try! domains::get(&domains::STORE, checkerCap.object);
92	+	set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;
93	+	set domains::STORE.contexts[checker.initial.index].frame.pc = completion();
94	+	set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;
95	+	set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;
96	+	set domains::STORE.contexts[checker.initial.index].frame.registers[3] = kernelGp();
97	+	let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
98	+	let start = clock + 1000000;
99	+	let mut harts: u64 = 0;
100	+	for hart in 0..8 { if (boot::PLATFORM.harts & (1 << hart)) <> 0 { set harts += 1; } }
101	+	let end = start + 10000000 * harts * harts;
102	+	for hart in 0..8 {
103	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
104	+	let budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);
105	+	let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);
106	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
107	+	budgets::Binding { budget, domain: handle, context: CONTEXTS[hart] }, clock);
108	+	if hart == 0 {
109	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
110	+	budgets::Binding { budget: final, domain: checkerHandle, context: checker.initial }, clock);
111	+	}
112	+	}
113	+	}
114	+
115	+	/// Check shared progress and context isolation after the user windows finish.
116	+	export unsafe fn verify() {
117	+	let guard = sync::enter();
118	+	let mut counts: [u64; 8] = [0; 8];
119	+	for hart in 0..8 {
120	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
121	+	let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);
122	+	set counts[hart] = value;
123	+	let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);
124	+	assert context.owner == USER and (context.frame.status & 0x1800) == 0;
125	+	assert context.hart == nil or context.hart == hart;
126	+	}
127	+	sync::leave(guard);
128	+	for byte in "smp completed transactions:" { put(byte); }
129	+	for hart in 0..8 {
130	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 { continue; }
131	+	put(32); number(counts[hart]);
132	+	}
133	+	put(10);
134	+	for hart in 0..8 {
135	+	if (boot::PLATFORM.harts & (1 << hart)) <> 0 { assert counts[hart] == 16; }
136	+	}
137	+	let maximum = sync::maximum();
138	+	assert maximum > 0 and maximum < 1000000;
139	+	for byte in "smp metadata instructions: 0x" { put(byte); }
140	+	number(maximum);
141	+	put(10);
142	+	finish();
143	+	}
144	+
145	+	/// Print one full-width hexadecimal diagnostic value.
146	+	fn number(value: u64) {
147	+	let digits = "0123456789abcdef";
148	+	for i in 0..16 { put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]); }
149	+	}

test/smp/machine.ras added +31 -0

1	+	//! Start all initialized harts after the fixture publishes its shared domains.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::start;
5	+	call @kernel::boot::run;
6	+	ebreak;
7	+	.export @kernel::dispatchcheck::memory;
8	+	.export @kernel::dispatchcheck::kernelGp;
9	+	.export @kernel::dispatchcheck::completion;
10	+	.export @kernel::dispatchcheck::finish;
11	+	.export @kernel::dispatchcheck::put;
12	+	@kernel::dispatchcheck::memory
13	+	ret;
14	+	@kernel::dispatchcheck::kernelGp
15	+	mv %a0 %gp;
16	+	ret;
17	+	@kernel::dispatchcheck::completion
18	+	la %a0 @complete;
19	+	ret;
20	+	@complete
21	+	call @kernel::dispatchcheck::verify;
22	+	ebreak;
23	+	@kernel::dispatchcheck::put
24	+	li %t0 0x10000000;
25	+	sb %a0 0(%t0);
26	+	ret;
27	+	@kernel::dispatchcheck::finish
28	+	li %t0 0x10001000;
29	+	li %t1 0x5555;
30	+	sw %t1 0(%t0);
31	+	ebreak;

test/smp/spin.rad added +31 -0

1	+	//! Concurrent user contexts with shared private state and separate call buffers.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Completed allocation transactions, indexed by the executing hart.
6	+	export static count: [u64; 8] = [0; 8];
7	+
8	+	/// Check hart identity and shared-table allocation through repeated direct calls.
9	+	fn run(hart: u64) {
10	+	let original = sys::currentContext();
11	+	assert original.hart == hart;
12	+	for i in 0..16 {
13	+	let page = try! sys::pageAllocate(abi::Handle(0), 1);
14	+	let info = sys::queryPage(page);
15	+	assert info.count == 1 and info.base <> 0;
16	+	try! sys::capabilityDrop(page);
17	+	let current = sys::currentContext();
18	+	assert current.context == original.context and current.hart == hart;
19	+	set count[hart as u32] += 1;
20	+	}
21	+	while true {}
22	+	}
23	+
24	+	/// Start an additional context with its physical hart index in validated memory.
25	+	export fn worker(args: *u64, size: u64) {
26	+	assert size == 8;
27	+	run(*args);
28	+	}
29	+
30	+	/// The initial context owns hart zero's window.
31	+	@default fn main(env: *opaque) { run(0); }

test/sync/machine.ras +21 -0

//! Exercise code publication and contended metadata updates on each hart.
.constant EXPECTED HARTS * 100;
.text;
@entry
    li %t0 0x1888;
    csrw mstatus %t0;
    csrr %s7 mstatus;
    call @kernel::sync::maskInterrupts;
    li %t0 8;
    bne %a0 %t0 @fail;
    csrr %t0 mstatus;
    andi %t1 %s7 -9;
    bne %t0 %t1 @fail;
    call @kernel::sync::restoreInterrupts;
    csrr %t0 mstatus;
    bne %t0 %s7 @fail;
    li %a0 0;
    call @kernel::sync::restoreInterrupts;
    csrr %t0 mstatus;
    andi %t1 %s7 -9;
    bne %t0 %t1 @fail;
    call @kernel::sync::retired;
    mv %s5 %a0;
    call @kernel::sync::retired;
    bgeu %s5 %a0 @fail;
    csrr %s6 mhartid;
    li %s0 0x40010000;
    slli %s0 %s0 1;
    bnez %s6 @waitCode;
    li %t0 1;

206	206
207	207		# Native preemption across user code, idle, and retained machine work.
208	208		.PHONY: dispatch-test
209	209		dispatch-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
210	210		@RAD_EMULATOR="$(EMU)" sh test/dispatch/run
	211	+
	212	+	# Concurrent user contexts and metadata transactions on every online hart.
	213	+	.PHONY: smp-test
	214	+	smp-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	215	+	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run smp "1 2 8"

216	217		budgets::Binding { budget: handle, domain: abi::Handle(arguments[1]), context: abi::reference(arguments[2]) }, now),
217	218		else => throw abi::Error::InvalidArg,
218	219		}
219	220		}
220	221
221		-	/// Complete a user ecall and preserve all registers except its scalar result.
222		-	export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64) {
	222	+	/// Check a retained domain's lifetime while metadata is serialized.
	223	+	unsafe fn live(owner: abi::Ref) -> bool {
	224	+	return slots::matches(&domains::STORE.slots[..], owner, slots::State::Live)
	225	+	and domains::STORE.records[owner.index].state <> domains::Lifecycle::Dead;
	226	+	}
	227	+
	228	+	/// Load a package with metadata locked only around resource transactions.
	229	+	unsafe fn runtimeLoad(domain: &mut domains::Domain, request: loader::Request, guard: sync::Guard) -> u64 throws (abi::Error) {
	230	+	let owner = domain.memory.table.owner;
	231	+	let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
	232	+	catch error { sync::leave(guard); throw error; };
	233	+	sync::leave(guard);
	234	+	let input = try loader::decode(&pending) catch error {
	235	+	let guard = sync::enter();
	236	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	237	+	sync::leave(guard); throw error;
	238	+	};
	239	+	let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
	240	+	let guard = sync::enter();
	241	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	242	+	sync::leave(guard); throw error;
	243	+	};
	244	+	let allocationGuard = sync::enter();
	245	+	if not live(owner) {
	246	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	247	+	sync::leave(allocationGuard); throw abi::Error::BadHandle;
	248	+	}
	249	+	if let object = existing {
	250	+	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	251	+	catch error { sync::leave(allocationGuard); throw error; };
	252	+	sync::leave(allocationGuard); return *result;
	253	+	}
	254	+	let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
	255	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	256	+	sync::leave(allocationGuard); throw error;
	257	+	};
	258	+	sync::leave(allocationGuard);
	259	+	let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
	260	+	let guard = sync::enter();
	261	+	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
	262	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	263	+	sync::leave(guard); throw error;
	264	+	};
	265	+	let publicationGuard = sync::enter();
	266	+	if not live(owner) {
	267	+	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
	268	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	269	+	sync::leave(publicationGuard); throw abi::Error::BadHandle;
	270	+	}
	271	+	let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
	272	+	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	273	+	catch error { sync::leave(publicationGuard); throw error; };
	274	+	sync::leave(publicationGuard);
	275	+	return *result;
	276	+	}
	277	+
	278	+	/// Execute a call with short metadata transactions and private bulk initialization.
	279	+	/// The running context keeps its domain storage live until dispatch releases it.
	280	+	export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
	281	+	assert arguments.len == 4;
	282	+	let guard = sync::enter();
	283	+	let mut domain = try domains::get(&domains::STORE, owner) catch error { sync::leave(guard); throw error; };
	284	+	if domain.state <> domains::Lifecycle::Active { sync::leave(guard); throw abi::Error::BadHandle; }
	285	+	let handle = abi::Handle(arguments[0]);
	286	+	match operation {
	287	+	case 20 => {
	288	+	let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
	289	+	&mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error { sync::leave(guard); throw error; };
	290	+	sync::leave(guard);
	291	+	try domains::prepare(&registry::STORE, &pending) catch error {
	292	+	let guard = sync::enter();
	293	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	294	+	sync::leave(guard); throw error;
	295	+	};
	296	+	let guard = sync::enter();
	297	+	if not live(owner) {
	298	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	299	+	sync::leave(guard); throw abi::Error::BadHandle;
	300	+	}
	301	+	let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
	302	+	catch error { sync::leave(guard); throw error; };
	303	+	sync::leave(guard); return *result;
	304	+	},
	305	+	case 30 => {
	306	+	let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
	307	+	catch error { sync::leave(guard); throw error; };
	308	+	sync::leave(guard);
	309	+	pages::clear(&pending);
	310	+	let guard = sync::enter();
	311	+	if not live(owner) {
	312	+	pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
	313	+	sync::leave(guard); throw abi::Error::BadHandle;
	314	+	}
	315	+	let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
	316	+	sync::leave(guard); return *result;
	317	+	},
	318	+	case 51 => return try runtimeLoad(&mut domain, loader::Request {
	319	+	authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
	320	+	}, guard),
	321	+	case 60 => {
	322	+	if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
	323	+	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
	324	+	sync::leave(guard); throw abi::Error::InvalidArg;
	325	+	}
	326	+	let words = @sliceOf(memory(arguments[1]), 4);
	327	+	let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
	328	+	let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
	329	+	&domain.memory.table, handle, start) catch error { sync::leave(guard); throw error; };
	330	+	let mut target: abi::Ref = undefined;
	331	+	match &pending { case domains::ContextReservation::Held(allocation) => { set target = allocation.owner; }, }
	332	+	sync::leave(guard);
	333	+	domains::contextClear(&pending);
	334	+	let guard = sync::enter();
	335	+	if not live(owner) or not live(target) {
	336	+	domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
	337	+	sync::leave(guard); throw abi::Error::BadHandle;
	338	+	}
	339	+	let result = domains::contextPublish(&mut domains::STORE, pending);
	340	+	sync::leave(guard); return abi::id(result);
	341	+	},
	342	+	else => {
	343	+	let result = try invoke(owner, operation, arguments, now) catch error { sync::leave(guard); throw error; };
	344	+	sync::leave(guard); return result;
	345	+	},
	346	+	}
	347	+	}
	348	+
	349	+	/// Complete a user ecall through the supplied execution boundary.
	350	+	export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
	351	+	execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
	352	+	{
223	353		assert trap::fromUser(frame) and frame.cause == 8;
224	354		assert trap::advanceCall(frame);
225	355		let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
226		-	let result = try invoke(owner, frame.registers[17], &args[..], now) catch error {
	356	+	let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
227	357		set frame.registers[10] = (0 as u64) - error as u64;
228	358		return;
229	359		};
230	360		set frame.registers[10] = result;
231	361		}

173	186		if not runnable(contexts, window.owner, context) { continue; }
174	187		if window.start > now {
175	188		if window.start < result.deadline { set result.deadline = window.start; }
176	189		continue;
177	190		}
	191	+	if let owner = contexts.contexts[context.index].hart {
	192	+	if owner <> hart {
	193	+	if window.end < result.deadline { set result.deadline = window.end; }
	194	+	continue;
	195	+	}
	196	+	}
178	197		assert result.budget == nil;
179	198		set result.budget = abi::Ref { index: i, generation: windows.slots[i].generation };
180	199		set result.context = context;
181	200		if window.end < result.deadline { set result.deadline = window.end; }
182	201		}

210	210		}
211	211		}
212	212		throw abi::Error::InvalidArg;
213	213		}
214	214
215		-	/// Reject physical overlap with any live context's reserved user stack.
	215	+	/// Reject physical overlap with published or reserved user stacks.
216	216		fn stackAvailable(store: &Store, candidate: range::Range) throws (abi::Error) {
217	217		for i in 0..limits::CONTEXTS {
218		-	if store.contextSlots[i].state <> slots::State::Live { continue; }
	218	+	if store.contextSlots[i].state <> slots::State::Live and store.contextSlots[i].state <> slots::State::Reserved { continue; }
219	219		let used = store.contexts[i].userStack;
220	220		if used.start < used.end and candidate.start < used.end and used.start < candidate.end {
221	221		throw abi::Error::Busy;
222	222		}
223	223		}

261	261		}
262	262		}
263	263		return false;
264	264		}
265	265
266		-	/// Create an additional context with private kernel storage and shared domain state.
267		-	/// The caller serializes domain, capability, and physical allocation metadata.
268		-	export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
269		-	table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
	266	+	/// Private kernel storage retained by an unpublished context.
	267	+	export record ContextAllocation: Copy {
	268	+	/// Reserved context slot with an initialized user-stack claim.
	269	+	object: abi::Ref,
	270	+	/// Domain kept live until publication or cancellation.
	271	+	owner: abi::Ref,
	272	+	/// Reserved kernel-stack frames.
	273	+	frames: frames::Run,
	274	+	/// Validated physical base for private stack clearing.
	275	+	base: u64,
	276	+	}
	277	+
	278	+	/// Context storage that must be published or cancelled under serialization.
	279	+	export union ContextReservation: Once {
	280	+	/// One reserved context and its private kernel storage.
	281	+	Held(ContextAllocation),
	282	+	}
	283	+
	284	+	/// Reserve an additional context and both stack claims under metadata serialization.
	285	+	export unsafe fn contextReserve(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
	286	+	table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> ContextReservation throws (abi::Error)
270	287		{
271	288		let permission = try capability::authority(table, authority, abi::Rights(abi::EXECUTE));
272	289		let domain = try get(store, permission.object);
273	290		if domain.state == Lifecycle::Dead { throw abi::Error::BadHandle; }
274	291		if start.entry == 0 or not exportedEntry(&domain, packages, start.entry)

21	21
22	22		# Verify the emulator binary exists.
23	23		EMU_PATH := $(shell command -v $(EMU) 2>/dev/null)
24	24
25	25		default: emulator $(RAD_BIN)
26		-	test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test
	26	+	test: emulator seed-test std-test bin-test kernel-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test
27	27
28	28		seed-test:
29	29		@seed/test
30	30
31	31		# Emulator command check

98	98		assert stackTop == PLATFORM.stacks[hart as u32].end and (stackTop & 15) == 0;
99	99		let stack = PLATFORM.stacks[hart as u32];
100	100		set HARTS[hart as u32] = trap::Hart {
101	101		stackTop: stack.end, stackBottom: stack.start, kernelGp: stateTable as u64, handler: unexpected,
102	102		savedT0: 0, savedT1: 0, savedSp: 0,
	103	+	dispatchTop: stack.end, dispatchFrame: undefined,
103	104		};
104	105		trap::install(&mut HARTS[hart as u32]);
105	106		try! dispatch::initialize(&PLATFORM, hart as u32, stateTable as u64);
106	107		let mut count: u64 = 0;
107	108		for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
108	109		return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
109	110		}
110	111
111		-	/// Enter dispatch after this hart completes platform initialization.
112		-	export unsafe fn run() { dispatch::start(&mut HARTS[dispatch::hart()]); }
	112	+	/// Enter dispatch after every online hart publishes its initialized state.
	113	+	export unsafe fn run() -> ! {
	114	+	let mut count: u64 = 0;
	115	+	for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
	116	+	while sync::loadAcquire(&ARRIVED) < count {}
	117	+	sync::syncInstructions();
	118	+	dispatch::start(&mut HARTS[dispatch::hart()]);
	119	+	}

12	12		use super::events;
13	13		use super::transactions;
14	14		use super::loader;
15	15		use super::dispatch;
16	16		use super::timers;
	17	+	use super::sync;
17	18
18	19		/// Validate resident object generations for capabilities without page ownership.
19	20		unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
20	21		match entry.kind {
21	22		case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },

8	8		use super::platform;
9	9		use super::trap;
10	10		use super::range;
11	11		use super::calls;
12	12		use super::timers;
	13	+	use super::sync;
13	14
14	15		/// Dispatch ownership and idle state retained by one physical hart.
15	16		export record State: Copy {
16	17		/// Physical hart whose windows this state selects.
17	18		hart: u32,

44	45		/// Runtime slots initialized before each hart enables timer interrupts.
45	46		unsafe static HARTS: [Runtime; limits::HARTS] = undefined;
46	47
47	48		/// Read the executing physical hart identifier.
48	49		export fn hart() -> u32;
	50	+	/// Read the current machine stack pointer.
	51	+	fn stackPointer() -> u64;
49	52		/// Address of the M-mode idle loop.
50	53		fn idleAddress() -> u64;
51	54		/// Enable machine timer delivery while global interrupts remain disabled.
52	55		fn enable();
53	56

62	65		state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
63	66		};
64	67		arm(device, 0xffffffffffffffff);
65	68		}
66	69
67		-	/// Reevaluate authority after a timer interrupt with shared metadata serialized.
68		-	export unsafe fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
	70	+	/// Tail-enter dispatch on the firmware stack and resume the selected frame.
	71	+	export fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> !;
	72	+
	73	+	/// Select authority from hart-owned storage with shared metadata serialized.
	74	+	export unsafe fn choose(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
69	75		let id = hart();
70	76		assert id < limits::HARTS;
	77	+	let stack = HARTS[id].state.stack;
	78	+	let pointer = stackPointer();
	79	+	assert pointer >= stack.start and pointer < stack.end;
	80	+	let guard = sync::enter();
71	81		let clock = now(HARTS[id].timer);
72	82		let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
73	83		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
74	84		let mut deadline = choice.deadline;
75	85		if timeout < deadline { set deadline = timeout; }
76	86		arm(HARTS[id].timer, deadline);
	87	+	sync::leave(guard);
77	88		}
78	89
79	90		/// Read the running context's identity and domain-relative CPU authority.
80	91		export unsafe fn current() -> abi::CurrentContextInfo {
81	92		let id = hart();

85	96		let window = try! budgets::get(&budgets::STORE, budget);
86	97		return abi::CurrentContextInfo { context: abi::id(context), hart: id as u64, budget: *window.handle };
87	98		}
88	99
89	100		/// Complete the current user's call and select authority at the return boundary.
90		-	export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
	101	+	export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
91	102		let id = hart();
92	103		assert id < limits::HARTS;
93		-	let context = HARTS[id].state.current else { trap::halt(); return; };
	104	+	let context = HARTS[id].state.current else trap::halt();
	105	+	let guard = sync::enter();
94	106		assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
95	107		let owner = domains::STORE.contexts[context.index].owner;
96	108		if frame.registers[17] == abi::Operation::ContextReturn as u64 {
97	109		assert trap::advanceCall(frame);
98	110		try! domains::returned(&mut domains::STORE, owner, context);
	111	+	sync::leave(guard);
99	112		} else {
100		-	calls::handle(owner, frame, now(HARTS[id].timer));
	113	+	sync::leave(guard);
	114	+	calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
101	115		}
102	116		interrupt(frame, anchor);
103	117		}
104	118
105	119		/// Enter the first authorized context or the hart's retained idle frame.
106	120		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
107	121		let id = hart();
108	122		assert id < limits::HARTS;
109	123		let mut frame = HARTS[id].state.idle;
110		-	interrupt(&mut frame, anchor);
111	124		enable();
112		-	trap::resume(&frame);
	125	+	interrupt(&mut frame, anchor);
113	126		}
114	127
115	128		/// Read one validated, naturally aligned 64-bit device register.
116	129		fn read(address: u64) -> u64;
117	130		/// Write one validated, naturally aligned 64-bit device register.

1	1		//! Naturally aligned RV64 CLINT access with device ordering.
2	2		.text;
3	3		.export @kernel::dispatch::read;
4	4		.export @kernel::dispatch::write;
5	5		.export @kernel::dispatch::hart;
	6	+	.export @kernel::dispatch::stackPointer;
6	7		.export @kernel::dispatch::idleAddress;
7	8		.export @kernel::dispatch::enable;
	9	+	.export @kernel::dispatch::interrupt;
	10	+
	11	+	// Retain the frame outside the context stack before publishing its release.
	12	+	@kernel::dispatch::interrupt
	13	+	mv %t0 %a0;
	14	+	addi %t1 %a1 64;
	15	+	li %t2 36;
	16	+	@retainDispatchFrame
	17	+	ld %t3 0(%t0);
	18	+	sd %t3 0(%t1);
	19	+	addi %t0 %t0 8;
	20	+	addi %t1 %t1 8;
	21	+	addi %t2 %t2 -1;
	22	+	bnez %t2 @retainDispatchFrame;
	23	+	ld %sp 56(%a1);
	24	+	addi %a0 %a1 64;
	25	+	call @kernel::dispatch::choose;
	26	+	csrr %a0 mscratch;
	27	+	addi %a0 %a0 64;
	28	+	j @kernel::trap::resume;
8	29
9	30		// The executing hart selects its private dispatch runtime slot.
10	31		@kernel::dispatch::hart
11	32		csrr %a0 mhartid;
12	33		ret;
13	34
	35	+	// Dispatch releases context stacks only from the hart's firmware stack.
	36	+	@kernel::dispatch::stackPointer
	37	+	mv %a0 %sp;
	38	+	ret;
	39	+
14	40		// Saved idle frames enter this loop with machine interrupts enabled by mret.
15	41		@kernel::dispatch::idleAddress
16	42		la %a0 @dispatchIdle;
17	43		ret;
18	44		@dispatchIdle

281	298		let pending = try frames::reserve(&mut memory.backings.pool, KERNEL_STACK_PAGES) catch error {
282	299		try! slots::cancel(&mut store.contextSlots[..], slot); throw error;
283	300		};
284	301		let kernelFrames = frames::commit(pending);
285	302		let kernelStack = try! frames::extent(&memory.backings.pool, kernelFrames);
286		-	zero(kernelStack.start, kernelFrames.count);
287	303		let mut frame = trap::Frame { registers: [0; 32], pc: start.entry, status: 0x80, cause: 0, value: 0 };
288	304		set frame.registers[1] = returnAddress();
289	305		set frame.registers[2] = start.stack;
290	306		set frame.registers[3] = domain.graph.table.ptr as u64;
291	307		set frame.registers[10] = start.args;
292	308		set frame.registers[11] = start.size;
293	309		let object = slots::reference(&slot);
294	310		set store.contexts[object.index] = Context {
295	311		owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
296	312		};
297		-	return try! slots::commit(&mut store.contextSlots[..], slot);
	313	+	match slot {
	314	+	case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
	315	+	object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
	316	+	}),
	317	+	}
	318	+	}
	319	+
	320	+	/// Clear an unpublished context's private kernel stack.
	321	+	export fn contextClear(reservation: &ContextReservation) {
	322	+	match reservation { case ContextReservation::Held(allocation) => zero(allocation.base, allocation.frames.count), }
	323	+	}
	324	+
	325	+	/// Check the reserved context and its retained physical allocation.
	326	+	fn contextRequired(store: &Store, allocation: &ContextAllocation) {
	327	+	assert slots::matches(&store.contextSlots[..], allocation.object, slots::State::Reserved);
	328	+	let context = store.contexts[allocation.object.index];
	329	+	assert context.owner == allocation.owner and context.kernelFrames == allocation.frames;
	330	+	assert context.hart == nil;
	331	+	}
	332	+
	333	+	/// Publish an initialized context while its domain stays live under serialization.
	334	+	export fn contextPublish(store: &mut Store, reservation: ContextReservation) -> abi::Ref {
	335	+	match reservation {
	336	+	case ContextReservation::Held(allocation) => {
	337	+	contextRequired(store, &allocation);
	338	+	assert slots::matches(&store.slots[..], allocation.owner, slots::State::Live);
	339	+	return try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
	340	+	},
	341	+	}
	342	+	}
	343	+
	344	+	/// Return an unpublished context's stack and slot under metadata serialization.
	345	+	export fn contextCancel(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation) {
	346	+	match reservation {
	347	+	case ContextReservation::Held(allocation) => {
	348	+	contextRequired(store, &allocation);
	349	+	try! frames::release(&mut memory.backings.pool, allocation.frames);
	350	+	try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
	351	+	},
	352	+	}
	353	+	}
	354	+
	355	+	/// Create a context while the caller has exclusive access to shared metadata.
	356	+	export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
	357	+	table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
	358	+	{
	359	+	let pending = try contextReserve(store, memory, packages, table, authority, start);
	360	+	contextClear(&pending);
	361	+	return contextPublish(store, pending);
298	362		}
299	363
300	364		/// Memory allocated before the domain and its context become observable.
301		-	record Prepared: Copy {
	365	+	export record Prepared: Copy {
302	366		/// Frames for capabilities and the event ring.
303	367		allocation: frames::Run,
304	368		/// Mapped metadata at the allocation's start.
305	369		memory: *unsafe mut Memory,
306		-	/// Independent package-state graph.
307		-	graph: instances::Instance,
	370	+	/// Reserved private package-state graph.
	371	+	graph: instances::Allocation,
308	372		/// Retained stack allocation for the initial context.
309	373		kernelFrames: frames::Run,
310	374		/// Initial context's mapped stack bounds.
311	375		kernelStack: range::Range,
312	376		}
313	377
314	378		/// Allocate private storage and return all earlier allocations on failure.
315		-	unsafe fn prepare(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
	379	+	unsafe fn allocate(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
316	380		let pending = try frames::reserve(pool, (@sizeOf(Memory) + 4095) / 4096);
317	381		let allocation = frames::commit(pending);
318		-	let graph = try instances::create(packages, pool, image) catch err {
	382	+	let graph = try instances::reserve(packages, pool, image) catch err {
319	383		try! frames::release(pool, allocation); throw err;
320	384		};
321	385		let pendingStack = try frames::reserve(pool, KERNEL_STACK_PAGES) catch err {
322		-	try! frames::release(pool, graph.frames);
	386	+	instances::cancel(pool, graph);
323	387		try! frames::release(pool, allocation); throw err;
324	388		};
325	389		let kernelFrames = frames::commit(pendingStack);
326	390		let kernelStack = try! frames::extent(pool, kernelFrames);
327	391		let extent = try! frames::extent(pool, allocation);
328		-	zero(extent.start, allocation.count);
329		-	zero(kernelStack.start, kernelFrames.count);
330		-	return Prepared { allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack };
	392	+	match graph {
	393	+	case instances::Reservation::Held(graph) => return Prepared {
	394	+	allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack,
	395	+	},
	396	+	}
331	397		}
332	398
333	399		/// Return storage that has never been exposed to a published domain.
334	400		fn discard(pool: &mut frames::Pool, prepared: Prepared) {
335	401		try! frames::release(pool, prepared.kernelFrames);
336	402		try! frames::release(pool, prepared.graph.frames);
337	403		try! frames::release(pool, prepared.allocation);
338	404		}
339	405
340		-	/// Create an unscheduled domain with Events and one initial integer context.
341		-	/// The caller serializes publication and keeps the authorizing domain alive.
342		-	export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
343		-	table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
	406	+	/// Unpublished domain metadata retained during private initialization.
	407	+	export record Creation: Copy {
	408	+	/// Calling domain kept live until publication or cancellation.
	409	+	owner: abi::Ref,
	410	+	/// Reserved capability slot in the calling domain.
	411	+	handle: abi::Ref,
	412	+	/// Reserved domain slot.
	413	+	domain: abi::Ref,
	414	+	/// Reserved initial-context slot.
	415	+	context: abi::Ref,
	416	+	/// Management and inherited authority selected at admission.
	417	+	rights: abi::Rights,
	418	+	/// Resident root package.
	419	+	image: abi::Ref,
	420	+	/// Validated initial entry address.
	421	+	entry: u64,
	422	+	/// Private allocations retained until the domain becomes observable.
	423	+	storage: Prepared,
	424	+	}
	425	+
	426	+	/// Domain creation resources that must be published or cancelled.
	427	+	export union Reservation: Once {
	428	+	/// One private domain and its reserved metadata capacity.
	429	+	Held(Creation),
	430	+	}
	431	+
	432	+	/// Reserve domain capacity under metadata serialization while the caller stays live.
	433	+	export unsafe fn reserve(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
	434	+	table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> Reservation throws (abi::Error)
344	435		{
345	436		let permit = try capability::authority(table, authority, abi::Rights(abi::CREATE));
346	437		if not backing::domainLive(backings, table.owner) or not backing::domainLive(backings, permit.object) {
347	438		throw abi::Error::BadHandle;
348	439		}

355	446		};
356	447		let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch err {
357	448		try! slots::cancel(&mut store.slots[..], domainSlot);
358	449		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
359	450		};
360		-	let mut prepared = try prepare(&mut backings.pool, packages, root) catch err {
361		-	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
362		-	try! slots::cancel(&mut store.slots[..], domainSlot);
363		-	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
364		-	};
365		-	let object = slots::reference(&domainSlot);
366		-	try backing::registerDomain(backings, object) catch err {
367		-	discard(&mut backings.pool, prepared);
368		-	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
369		-	try! slots::cancel(&mut store.slots[..], domainSlot);
370		-	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
371		-	};
372		-	capability::initialize(&mut prepared.memory.table, object);
373		-	try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
374		-	try! backing::endDomain(backings, object);
375		-	discard(&mut backings.pool, prepared);
	451	+	let initialSlot = slots::reference(&contextSlot);
	452	+	set store.contexts[initialSlot.index].userStack = range::Range { start: 0, end: 0 };
	453	+	let prepared = try allocate(&mut backings.pool, packages, root) catch err {
376	454		try! slots::cancel(&mut store.contextSlots[..], contextSlot);
377	455		try! slots::cancel(&mut store.slots[..], domainSlot);
378	456		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
379	457		};
380		-	let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
381		-	kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ \| abi::WRITE),
382		-	});
383		-	let initial = slots::reference(&contextSlot);
384		-	// MPIE enables interrupts after the first return into U-mode.
385		-	let mut frame = trap::Frame { registers: [0; 32], pc: entry, status: 0x80, cause: 0, value: 0 };
386		-	set frame.registers[1] = returnAddress();
387		-	set frame.registers[3] = prepared.graph.table.ptr as u64;
388		-	set store.contexts[initial.index] = Context {
389		-	owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
390		-	userStack: range::Range { start: 0, end: 0 }, frame,
391		-	};
392		-	set store.records[object.index] = Domain {
393		-	state: Lifecycle::Pending, creator: table.owner, parent: table.owner, image: root, initial,
394		-	allocation: prepared.allocation, memory: prepared.memory, graph: prepared.graph, events: queue,
	458	+	match handleSlot {
	459	+	case slots::Reservation::Held(handle) => match domainSlot {
	460	+	case slots::Reservation::Held(domain) => match contextSlot {
	461	+	case slots::Reservation::Held(context) => return Reservation::Held(Creation {
	462	+	owner: table.owner, handle, domain, context,
	463	+	rights: abi::Rights(MANAGEMENT_RIGHTS \| (*permit.rights & (abi::CREATE \| abi::ALLOCATE))),
	464	+	image: root, entry, storage: prepared,
	465	+	}),
	466	+	},
	467	+	},
	468	+	}
	469	+	}
	470	+
	471	+	/// Initialize private storage from resident packages while metadata can change.
	472	+	export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
	473	+	match reservation {
	474	+	case Reservation::Held(creation) => {
	475	+	let mut storage = creation.storage;
	476	+	zero(storage.memory as u64, storage.allocation.count);
	477	+	zero(storage.kernelStack.start, storage.kernelFrames.count);
	478	+	try instances::fill(packages, &storage.graph.graph, storage.graph.base);
	479	+	capability::initialize(&mut storage.memory.table, creation.domain);
	480	+	},
	481	+	}
	482	+	}
	483	+
	484	+	/// Check the unpublished capacity retained by one creation transaction.
	485	+	fn require(store: &Store, table: &capability::Table, creation: &Creation) {
	486	+	assert table.owner == creation.owner;
	487	+	assert slots::matches(&table.slots[..], creation.handle, slots::State::Reserved);
	488	+	assert slots::matches(&store.slots[..], creation.domain, slots::State::Reserved);
	489	+	assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
	490	+	}
	491	+
	492	+	/// Return private domain storage and reserved slots under metadata serialization.
	493	+	export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
	494	+	match reservation {
	495	+	case Reservation::Held(creation) => {
	496	+	require(store, table, &creation);
	497	+	discard(&mut backings.pool, creation.storage);
	498	+	try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
	499	+	try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
	500	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
	501	+	},
	502	+	}
	503	+	}
	504	+
	505	+	/// Publish an initialized domain and its Events under metadata serialization.
	506	+	export unsafe fn publish(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation)
	507	+	-> abi::Handle throws (abi::Error)
	508	+	{
	509	+	match reservation {
	510	+	case Reservation::Held(creation) => {
	511	+	require(store, table, &creation);
	512	+	if not backing::domainLive(backings, creation.owner) {
	513	+	cancel(store, backings, table, Reservation::Held(creation));
	514	+	throw abi::Error::BadHandle;
	515	+	}
	516	+	let object = creation.domain;
	517	+	let initial = creation.context;
	518	+	let mut prepared = creation.storage;
	519	+	try backing::registerDomain(backings, object) catch err {
	520	+	cancel(store, backings, table, Reservation::Held(creation)); throw err;
	521	+	};
	522	+	try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
	523	+	try! backing::endDomain(backings, object);
	524	+	cancel(store, backings, table, Reservation::Held(creation)); throw err;
	525	+	};
	526	+	let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
	527	+	kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ \| abi::WRITE),
	528	+	});
	529	+	let graph = instances::commit(instances::Reservation::Held(prepared.graph));
	530	+	// MPIE enables interrupts after the first return into U-mode.
	531	+	let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
	532	+	set frame.registers[1] = returnAddress();
	533	+	set frame.registers[3] = graph.table.ptr as u64;
	534	+	set store.contexts[initial.index] = Context {
	535	+	owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
	536	+	userStack: range::Range { start: 0, end: 0 }, frame,
	537	+	};
	538	+	set store.records[object.index] = Domain {
	539	+	state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
	540	+	allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
	541	+	};
	542	+	let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
	543	+	let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
	544	+	return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
	545	+	kind: abi::Kind::Domain, object: domain, rights: creation.rights,
	546	+	});
	547	+	},
	548	+	}
	549	+	}
	550	+
	551	+	/// Create a private domain while the caller has exclusive access to metadata.
	552	+	export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
	553	+	table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
	554	+	{
	555	+	let pending = try reserve(store, backings, packages, table, authority, image);
	556	+	try prepare(packages, &pending) catch err {
	557	+	cancel(store, backings, table, pending); throw err;
395	558		};
396		-	let context = try! slots::commit(&mut store.contextSlots[..], contextSlot);
397		-	let domain = try! slots::commit(&mut store.slots[..], domainSlot);
398		-	return capability::publish(table, handleSlot, capability::Entry {
399		-	kind: abi::Kind::Domain, object: domain,
400		-	rights: abi::Rights(MANAGEMENT_RIGHTS \| (*permit.rights & (abi::CREATE \| abi::ALLOCATE))),
401		-	});
	559	+	return try publish(store, backings, table, pending);
402	560		}

13	13		/// Package bases indexed by resident slot; absent packages have zero bases.
14	14		table: *unsafe [u64],
15	15		}
16	16
17	17		/// Bounded traversal and conservative storage requirement for a package graph.
18		-	record Graph: Copy {
	18	+	export record Graph: Copy {
19	19		/// Reachable package references, each present once.
20	20		packages: [abi::Ref; limits::PACKAGES],
21	21		/// Number of initialized package references.
22	22		count: u32,
23	23		/// Bytes for the base table, private state, and worst-case alignment padding.
24	24		bytes: u64,
25	25		}
26	26
	27	+	/// Private storage and the resident graph selected at reservation time.
	28	+	export record Allocation: Copy {
	29	+	/// Reachable immutable packages and their storage bounds.
	30	+	graph: Graph,
	31	+	/// Physical frames reserved for this instance.
	32	+	frames: frames::Run,
	33	+	/// Validated physical base for private initialization.
	34	+	base: u64,
	35	+	}
	36	+
	37	+	/// Graph storage that must be committed or returned to its frame pool.
	38	+	export union Reservation: Once {
	39	+	/// An unpublished private instance.
	40	+	Held(Allocation),
	41	+	}
	42	+
27	43		/// Map an allocated physical extent to typed storage at the call site.
28	44		fn memory(address: u64) -> *unsafe mut opaque;
29	45
30	46		/// Collect the reachable immutable packages with a bounded queue.
31	47		unsafe fn graph(store: &registry::Store, root: abi::Ref) -> Graph throws (abi::Error) {

55	71		}
56	72		return result;
57	73		}
58	74
59	75		/// Assign every base before initializing templates and their private pointers.
60		-	unsafe fn initialize(store: &registry::Store, graph: &Graph, base: u64) -> *unsafe [u64] throws (abi::Error) {
	76	+	/// The caller owns a writable extent of at least graph.bytes bytes at base.
	77	+	export unsafe fn fill(store: &registry::Store, graph: &Graph, base: u64) throws (abi::Error) {
61	78		let table = @sliceOf(memory(base) as *unsafe mut u64, limits::PACKAGES);
62	79		for i in 0..table.len { set table[i] = 0; }
63	80		let mut next = base + limits::PACKAGES as u64 * 8;
64	81		for object in &graph.packages[..graph.count] {
65	82		let package = try! registry::get(store, object);

71	88		for object in &graph.packages[..graph.count] {
72	89		let package = try! registry::get(store, object);
73	90		let bytes = @sliceOf(memory(table[object.index]) as *unsafe mut u8, package.memory);
74	91		try shared::instantiate(&package, &table[..], bytes) catch { throw abi::Error::VerifyFailed; };
75	92		}
76		-	return &table[..];
77	93		}
78	94
79		-	/// Allocate and initialize one private graph without copying resident code.
80		-	/// The caller serializes frame reservations and keeps the registry immutable.
81		-	export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
	95	+	/// Reserve one graph under metadata serialization while its packages stay resident.
	96	+	export unsafe fn reserve(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Reservation throws (abi::Error) {
82	97		let layout = try graph(store, root);
83	98		let count = ((layout.bytes + limits::FRAME_SIZE - 1) / limits::FRAME_SIZE) as u32;
84	99		let pending = try frames::reserve(pool, count);
85	100		let run = frames::commit(pending);
86	101		let extent = try! frames::extent(pool, run);
87		-	let table = try initialize(store, &layout, extent.start) catch err {
88		-	try! frames::release(pool, run); throw err;
	102	+	return Reservation::Held(Allocation { graph: layout, frames: run, base: extent.start });
	103	+	}
	104	+
	105	+	/// Initialize private state from the reservation's immutable resident packages.
	106	+	export unsafe fn initialize(store: &registry::Store, reservation: &Reservation) throws (abi::Error) {
	107	+	match reservation {
	108	+	case Reservation::Held(allocation) => try fill(store, &allocation.graph, allocation.base),
	109	+	}
	110	+	}
	111	+
	112	+	/// Consume initialized private storage for publication in a domain.
	113	+	export unsafe fn commit(reservation: Reservation) -> Instance {
	114	+	match reservation {
	115	+	case Reservation::Held(allocation) => return Instance {
	116	+	frames: allocation.frames,
	117	+	table: @sliceOf(memory(allocation.base) as *unsafe u64, limits::PACKAGES),
	118	+	},
	119	+	}
	120	+	}
	121	+
	122	+	/// Return unpublished graph storage under frame-pool serialization.
	123	+	export fn cancel(pool: &mut frames::Pool, reservation: Reservation) {
	124	+	match reservation { case Reservation::Held(allocation) => { try! frames::release(pool, allocation.frames); }, }
	125	+	}
	126	+
	127	+	/// Allocate and initialize a private graph while the caller has exclusive metadata access.
	128	+	export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
	129	+	let pending = try reserve(store, pool, root);
	130	+	try initialize(store, &pending) catch err {
	131	+	cancel(pool, pending); throw err;
89	132		};
90		-	return Instance { frames: run, table };
	133	+	return commit(pending);
91	134		}

85	85		export union Lease: Once {
86	86		/// Private temporary frames owned by the active load.
87	87		Held(frames::Run),
88	88		}
89	89
	90	+	/// Private input capacity retained throughout one load.
	91	+	export record Input: Copy {
	92	+	/// Domain kept live while the source and reserved handle are in use.
	93	+	owner: abi::Ref,
	94	+	/// Reserved Image handle slot in the caller's table.
	95	+	handle: abi::Ref,
	96	+	/// Private workspace frames.
	97	+	frames: frames::Run,
	98	+	/// Physical workspace base retained by the lease.
	99	+	work: u64,
	100	+	/// Validated physical source address.
	101	+	address: u64,
	102	+	/// Number of source bytes to snapshot.
	103	+	length: u32,
	104	+	}
	105	+
	106	+	/// Input capacity that must be cancelled or completed under metadata serialization.
	107	+	export union Reservation: Once {
	108	+	/// One admitted load and its private compiler storage.
	109	+	Held(Input),
	110	+	}
	111	+
90	112		/// Global runtime-loader ownership.
91	113		export unsafe static STATE: State = State { busy: false, owner: abi::Ref { index: 0, generation: 0 },
92	114		live: [false; limits::PACKAGES], resident: undefined };
93	115
94	116		/// Map a validated physical workspace extent to its compiler storage.

161	183		},
162	184		else => return abi::Error::VerifyFailed,
163	185		}
164	186		}
165	187
166		-	/// Used page counts for unpublished native output.
167		-	record OutputSize: Copy {
	188	+	/// Used native extents and validated metadata ready for publication.
	189	+	export record Compiled: Copy {
168	190		/// Pages occupied by generated code.
169	191		code: u32,
170	192		/// Pages occupied by the packed native catalog.
171	193		metadata: u32,
	194	+	/// Immutable descriptor and resolved dependencies for the reserved package.
	195	+	admission: registry::Admission,
172	196		}
173	197
174	198		/// Compile and pack one candidate into caller-owned physical output extents.
175	199		unsafe fn generate(work: *mut Workspace, input: &binary::Package, packages: &registry::Store,
176		-	slot: u32, code: u64, metadata: u64, length: u32) -> OutputSize throws (abi::Error)
	200	+	object: abi::Ref, code: u64, metadata: u64, length: u32) -> Compiled throws (abi::Error)
177	201		{
178	202		let count = try registry::imports(packages, &input.dependencies[..], &mut work.imports[..]);
179	203		let mut arena = alloc::new(&mut work.code[..]);
180	204		let mut scratch = alloc::new(&mut work.scratch[..]);
181		-	let package = try shared::compile(&*input, slot, code, &work.imports[..count],
	205	+	let package = try shared::compile(&*input, object.index, code, &work.imports[..count],
182	206		shared::Storage {
183	207		data: &mut work.data[..], symbols: &mut work.symbols[..], exports: &mut work.exports[..],
184	208		template: &mut work.template[..], relocations: &mut work.relocations[..],
185	209		}, &mut arena, &mut scratch) catch error { throw backend(error); };
186	210		if package.code.len > CODE_BYTES / 4 { throw abi::Error::Exhausted; }

190	214		let metadataMemory = @sliceOf(memory(metadata), METADATA_BYTES);
191	215		let used = try catalog::pack(&[catalog::Entry { source: &work.input[..length], package }], metadata, metadataMemory)
192	216		catch error { throw backend(error); };
193	217		let mut codePages = (codeBytes + 4095) / 4096;
194	218		if codePages == 0 { set codePages = 1; }
195		-	return OutputSize { code: codePages, metadata: (used + 4095) / 4096 };
	219	+	let native = descriptor(metadata);
	220	+	let admission = try registry::admit(packages, object, native.source, native.package);
	221	+	return Compiled { code: codePages, metadata: (used + 4095) / 4096, admission };
196	222		}
197	223
198	224		/// Return unused output frames while preserving at least one frame for each extent.
199	225		fn trim(pool: &mut frames::Pool, run: frames::Run, count: u32) -> frames::Run {
200	226		assert count > 0 and count <= run.count;
201	227		if count < run.count { try! frames::release(pool, frames::Run { first: run.first + count, count: run.count - count }); }
202	228		return frames::Run { first: run.first, count };
203	229		}
204	230
205		-	/// Snapshot, decode, compile, and publish a package while its caller remains live.
206		-	unsafe fn produce(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
207		-	work: *mut Workspace, address: u64, length: u32) -> abi::Ref throws (abi::Error)
208		-	{
209		-	try! mem::copy(&mut work.input[..length], @sliceOf(memory(address), length));
210		-	let mut decoder = alloc::new(&mut work.decoded[..]);
211		-	let input = try program::decode(&work.input[..length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
212		-	if err == binary::Error::Storage { throw abi::Error::Exhausted; }
213		-	throw abi::Error::VerifyFailed;
214		-	};
215		-	let existing = try registry::identify(packages, &input.name[..], &work.input[..length]);
216		-	if let object = existing { return object; }
	231	+	/// Snapshot and decode trusted input into the held private workspace.
	232	+	/// The decoded package stays valid until this reservation is completed or cancelled.
	233	+	export unsafe fn decode(reservation: &Reservation) -> binary::Package throws (abi::Error) {
	234	+	match reservation {
	235	+	case Reservation::Held(input) => {
	236	+	let mut work = workspace(input.work);
	237	+	try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
	238	+	let mut decoder = alloc::new(&mut work.decoded[..]);
	239	+	return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
	240	+	if err == binary::Error::Storage { throw abi::Error::Exhausted; }
	241	+	throw abi::Error::VerifyFailed;
	242	+	};
	243	+	},
	244	+	}
	245	+	}
	246	+
	247	+	/// Reserved native output and its immutable physical extents.
	248	+	export record Output: Copy {
	249	+	/// Reserved package slot selected for generated state references.
	250	+	object: abi::Ref,
	251	+	/// Private executable frames.
	252	+	code: frames::Run,
	253	+	/// Private source and descriptor frames.
	254	+	metadata: frames::Run,
	255	+	/// Validated physical executable base.
	256	+	codeBase: u64,
	257	+	/// Validated physical descriptor base.
	258	+	metadataBase: u64,
	259	+	}
	260	+
	261	+	/// Native output that must be published or cancelled under serialization.
	262	+	export union OutputReservation: Once {
	263	+	/// One unpublished package and its private output frames.
	264	+	Held(Output),
	265	+	}
	266	+
	267	+	/// Reserve a package slot and native output under metadata serialization.
	268	+	export fn reserveOutput(pool: &mut frames::Pool, packages: &mut registry::Store) -> OutputReservation throws (abi::Error) {
217	269		let slot = try registry::reserve(packages);
218	270		let object = slots::reference(&slot);
219	271		let codeReservation = try frames::reserve(pool, CODE_BYTES / 4096) catch err { registry::cancel(packages, slot); throw err; };
220	272		let code = frames::commit(codeReservation);
221	273		let metadataReservation = try frames::reserve(pool, METADATA_BYTES / 4096) catch err {
222	274		try! frames::release(pool, code); registry::cancel(packages, slot); throw err;
223	275		};
224	276		let metadata = frames::commit(metadataReservation);
225	277		let codeRange = try! frames::extent(pool, code);
226	278		let metadataRange = try! frames::extent(pool, metadata);
227		-	let sizes = try generate(work, &input, packages, object.index, codeRange.start, metadataRange.start, length) catch err {
228		-	try! frames::release(pool, metadata); try! frames::release(pool, code);
229		-	registry::cancel(packages, slot); throw err;
230		-	};
231		-	let retainedCode = trim(pool, code, sizes.code);
232		-	let retainedMetadata = trim(pool, metadata, sizes.metadata);
233		-	let native = descriptor(metadataRange.start);
234		-	sync::syncInstructions();
235		-	let result = try registry::publish(packages, slot, native.source, native.package) catch err {
236		-	try! frames::release(pool, retainedMetadata); try! frames::release(pool, retainedCode); throw err;
237		-	};
238		-	assert result == object;
239		-	set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
240		-	set state.live[result.index] = true;
241		-	return result;
	279	+	match slot {
	280	+	case slots::Reservation::Held(object) => return OutputReservation::Held(Output {
	281	+	object, code, metadata, codeBase: codeRange.start, metadataBase: metadataRange.start,
	282	+	}),
	283	+	}
242	284		}
243	285
244		-	/// Load trusted RIL and install an Image capability after complete native publication.
245		-	/// The caller keeps its domain alive and serializes shared metadata transactions.
246		-	export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
247		-	table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
	286	+	/// Compile a decoded package into reserved private output using resident dependencies.
	287	+	export unsafe fn compile(packages: &registry::Store, reservation: &Reservation, input: &binary::Package, output: &OutputReservation)
	288	+	-> Compiled throws (abi::Error)
	289	+	{
	290	+	match reservation {
	291	+	case Reservation::Held(source) => match output {
	292	+	case OutputReservation::Held(target) => {
	293	+	let work = workspace(source.work);
	294	+	return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
	295	+	},
	296	+	},
	297	+	}
	298	+	}
	299	+
	300	+	/// Return unpublished output frames and its package slot under serialization.
	301	+	export fn cancelOutput(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation) {
	302	+	match reservation {
	303	+	case OutputReservation::Held(output) => {
	304	+	assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
	305	+	try! frames::release(pool, output.metadata);
	306	+	try! frames::release(pool, output.code);
	307	+	registry::cancel(packages, slots::Reservation::Held(output.object));
	308	+	},
	309	+	}
	310	+	}
	311	+
	312	+	/// Publish completed native output and retain its used frames under serialization.
	313	+	export fn publishOutput(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
	314	+	reservation: OutputReservation, sizes: Compiled) -> abi::Ref
	315	+	{
	316	+	match reservation {
	317	+	case OutputReservation::Held(output) => {
	318	+	assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
	319	+	let retainedCode = trim(pool, output.code, sizes.code);
	320	+	let retainedMetadata = trim(pool, output.metadata, sizes.metadata);
	321	+	sync::syncInstructions();
	322	+	let result = registry::commit(packages, slots::Reservation::Held(output.object), sizes.admission);
	323	+	assert result == output.object;
	324	+	set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
	325	+	set state.live[result.index] = true;
	326	+	return result;
	327	+	},
	328	+	}
	329	+	}
	330	+
	331	+	/// Compare the snapshot with immutable resident content while the workspace is held.
	332	+	export unsafe fn identify(packages: &registry::Store, reservation: &Reservation, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
	333	+	match reservation {
	334	+	case Reservation::Held(source) => return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]),
	335	+	}
	336	+	}
	337	+
	338	+	/// Reserve an Image handle and exclusive workspace under metadata serialization.
	339	+	export unsafe fn reserve(state: &mut State, store: &mut pages::Store,
	340	+	table: &mut capability::Table, request: Request) -> Reservation throws (abi::Error)
248	341		{
249	342		let address = try source(store, table, request);
250	343		let handle = try slots::reserve(&mut table.slots[..]);
251	344		let lease = try acquire(state, &mut store.backings.pool, table.owner) catch err {
252	345		try! slots::cancel(&mut table.slots[..], handle); throw err;
253	346		};
254		-	let work = storage(&store.backings.pool, &lease);
255		-	let object = try produce(state, &mut store.backings.pool, packages, work, address, request.length as u32) catch err {
256		-	release(state, &mut store.backings.pool, lease);
257		-	try! slots::cancel(&mut table.slots[..], handle); throw err;
	347	+	let work = storage(&store.backings.pool, &lease) as u64;
	348	+	match handle {
	349	+	case slots::Reservation::Held(handle) => match lease {
	350	+	case Lease::Held(frames) => return Reservation::Held(Input {
	351	+	owner: table.owner, handle, frames, work, address, length: request.length as u32,
	352	+	}),
	353	+	},
	354	+	}
	355	+	}
	356	+
	357	+	/// Check workspace ownership and the caller's unpublished handle slot.
	358	+	fn require(state: &State, table: &capability::Table, input: &Input) {
	359	+	assert state.busy and state.owner == input.owner and table.owner == input.owner;
	360	+	assert slots::matches(&table.slots[..], input.handle, slots::State::Reserved);
	361	+	}
	362	+
	363	+	/// Return the workspace and reserved handle under metadata serialization.
	364	+	export fn cancel(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation) {
	365	+	match reservation {
	366	+	case Reservation::Held(input) => {
	367	+	require(state, table, &input);
	368	+	release(state, &mut store.backings.pool, Lease::Held(input.frames));
	369	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(input.handle));
	370	+	},
	371	+	}
	372	+	}
	373	+
	374	+	/// Install a resident package handle and return the workspace under serialization.
	375	+	export fn finish(state: &mut State, store: &mut pages::Store, table: &mut capability::Table,
	376	+	reservation: Reservation, object: abi::Ref) -> abi::Handle throws (abi::Error)
	377	+	{
	378	+	match reservation {
	379	+	case Reservation::Held(input) => {
	380	+	require(state, table, &input);
	381	+	if not backing::domainLive(&store.backings, input.owner) {
	382	+	cancel(state, store, table, Reservation::Held(input)); throw abi::Error::BadHandle;
	383	+	}
	384	+	let result = capability::publish(table, slots::Reservation::Held(input.handle), capability::Entry {
	385	+	kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
	386	+	});
	387	+	release(state, &mut store.backings.pool, Lease::Held(input.frames));
	388	+	return result;
	389	+	},
	390	+	}
	391	+	}
	392	+
	393	+	/// Load trusted RIL while the caller has exclusive access to shared metadata.
	394	+	export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
	395	+	table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
	396	+	{
	397	+	let pending = try reserve(state, store, table, request);
	398	+	let input = try decode(&pending) catch err { cancel(state, store, table, pending); throw err; };
	399	+	let existing = try identify(packages, &pending, &input) catch err { cancel(state, store, table, pending); throw err; };
	400	+	if let object = existing { return try finish(state, store, table, pending, object); }
	401	+	let output = try reserveOutput(&mut store.backings.pool, packages) catch err { cancel(state, store, table, pending); throw err; };
	402	+	let sizes = try compile(packages, &pending, &input, &output) catch err {
	403	+	cancelOutput(&mut store.backings.pool, packages, output);
	404	+	cancel(state, store, table, pending); throw err;
258	405		};
259		-	let result = capability::publish(table, handle, capability::Entry {
260		-	kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
261		-	});
262		-	release(state, &mut store.backings.pool, lease);
263		-	return result;
	406	+	let object = publishOutput(state, &mut store.backings.pool, packages, output, sizes);
	407	+	return try finish(state, store, table, pending, object);
264	408		}

78	78		if cursor == before { return false; }
79	79		}
80	80		return true;
81	81		}
82	82
83		-	/// Allocate and zero frames before publishing a page and its capability.
	83	+	/// Private frames and unpublished slots held while memory is cleared.
	84	+	export record Allocation: Copy {
	85	+	/// Calling domain whose table contains the reserved handle.
	86	+	owner: abi::Ref,
	87	+	/// Unpublished capability slot.
	88	+	handle: abi::Ref,
	89	+	/// Unpublished page-object slot.
	90	+	page: abi::Ref,
	91	+	/// Unpublished backing-allocation slot.
	92	+	backing: abi::Ref,
	93	+	/// Frames excluded from allocation until publication or cancellation.
	94	+	run: frames::Run,
	95	+	/// Validated physical base for private initialization.
	96	+	base: u64,
	97	+	}
	98	+
	99	+	/// Allocation capacity that must be published or cancelled under serialization.
	100	+	export union Reservation: Once {
	101	+	/// Private memory and its three reserved metadata slots.
	102	+	Held(Allocation),
	103	+	}
	104	+
	105	+	/// Reserve page capacity under metadata serialization for private initialization.
84	106		/// Allocation authority must name the calling domain. Failure cancels reservations.
85		-	export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
86		-	-> abi::Handle throws (abi::Error)
	107	+	export fn reserve(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
	108	+	-> Reservation throws (abi::Error)
87	109		{
88	110		if count == 0 { throw abi::Error::InvalidArg; }
89	111		if count > limits::FRAMES as u64 { throw abi::Error::OutOfMemory; }
90	112		let owner = table.owner;
91	113		let permission = try capability::authority(table, authority, abi::Rights(abi::ALLOCATE));
92	114		if permission.object <> owner { throw abi::Error::Denied; }
93		-	if not backing::domainLive(&store.backings, owner) { throw abi::Error::BadHandle; }
	115	+	let backings = &mut store.backings;
	116	+	if not backing::domainLive(backings, owner) { throw abi::Error::BadHandle; }
94	117		let handleSlot = try slots::reserve(&mut table.slots[..]);
95	118		let pageSlot = try slots::reserve(&mut store.slots[..]) catch err {
96	119		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
97	120		};
98		-	let backingSlot = try slots::reserve(&mut store.backings.slots[..]) catch err {
	121	+	let backingSlot = try slots::reserve(&mut backings.slots[..]) catch err {
99	122		try! slots::cancel(&mut store.slots[..], pageSlot);
100	123		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
101	124		};
102		-	let memory = try frames::reserve(&mut store.backings.pool, count as u32) catch err {
103		-	try! slots::cancel(&mut store.backings.slots[..], backingSlot);
	125	+	let memory = try frames::reserve(&mut backings.pool, count as u32) catch err {
	126	+	try! slots::cancel(&mut backings.slots[..], backingSlot);
104	127		try! slots::cancel(&mut store.slots[..], pageSlot);
105	128		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
106	129		};
107	130		let run = frames::commit(memory);
108		-	let extent = try! frames::extent(&store.backings.pool, run);
109		-	zero(extent.start, run.count);
110		-	let allocation = backing::publish(&mut store.backings, backingSlot, run, owner);
111		-	let object = slots::reference(&pageSlot);
112		-	set store.records[object.index] = Page { backing: allocation, base: extent.start, count: run.count, origin: owner, handles: 1 };
113		-	let page = try! slots::commit(&mut store.slots[..], pageSlot);
114		-	return capability::publish(table, handleSlot, capability::Entry {
115		-	kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
116		-	});
	131	+	let extent = try! frames::extent(&backings.pool, run);
	132	+	match handleSlot {
	133	+	case slots::Reservation::Held(handle) => match pageSlot {
	134	+	case slots::Reservation::Held(page) => match backingSlot {
	135	+	case slots::Reservation::Held(backing) => return Reservation::Held(Allocation {
	136	+	owner, handle, page, backing, run, base: extent.start,
	137	+	}),
	138	+	},
	139	+	},
	140	+	}
	141	+	}
	142	+
	143	+	/// Clear private frames while shared metadata can be used by other harts.
	144	+	export fn clear(reservation: &Reservation) {
	145	+	match reservation { case Reservation::Held(allocation) => zero(allocation.base, allocation.run.count), }
	146	+	}
	147	+
	148	+	/// Check reserved generations before publishing or cancelling an allocation.
	149	+	fn require(store: &Store, table: &capability::Table, allocation: &Allocation) {
	150	+	assert table.owner == allocation.owner;
	151	+	assert slots::matches(&table.slots[..], allocation.handle, slots::State::Reserved);
	152	+	assert slots::matches(&store.slots[..], allocation.page, slots::State::Reserved);
	153	+	assert slots::matches(&store.backings.slots[..], allocation.backing, slots::State::Reserved);
	154	+	}
	155	+
	156	+	/// Publish cleared memory under metadata serialization while its owner stays live.
	157	+	export unsafe fn publish(store: &mut Store, table: &mut capability::Table, reservation: Reservation) -> abi::Handle {
	158	+	match reservation {
	159	+	case Reservation::Held(allocation) => {
	160	+	require(store, table, &allocation);
	161	+	assert backing::domainLive(&store.backings, allocation.owner);
	162	+	let backing = backing::publish(&mut store.backings, slots::Reservation::Held(allocation.backing), allocation.run, allocation.owner);
	163	+	set store.records[allocation.page.index] = Page {
	164	+	backing, base: allocation.base, count: allocation.run.count, origin: allocation.owner, handles: 1,
	165	+	};
	166	+	let page = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(allocation.page));
	167	+	return capability::publish(table, slots::Reservation::Held(allocation.handle), capability::Entry {
	168	+	kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
	169	+	});
	170	+	},
	171	+	}
	172	+	}
	173	+
	174	+	/// Return private allocation capacity under metadata serialization.
	175	+	export fn cancel(store: &mut Store, table: &mut capability::Table, reservation: Reservation) {
	176	+	match reservation {
	177	+	case Reservation::Held(allocation) => {
	178	+	require(store, table, &allocation);
	179	+	try! frames::release(&mut store.backings.pool, allocation.run);
	180	+	try! slots::cancel(&mut store.backings.slots[..], slots::Reservation::Held(allocation.backing));
	181	+	try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(allocation.page));
	182	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
	183	+	},
	184	+	}
	185	+	}
	186	+
	187	+	/// Allocate zeroed pages while the caller has exclusive access to metadata.
	188	+	export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
	189	+	-> abi::Handle throws (abi::Error)
	190	+	{
	191	+	let pending = try reserve(store, table, authority, count);
	192	+	clear(&pending);
	193	+	return publish(store, table, pending);
117	194		}
118	195
119	196		/// Split an exclusive page into two segments with the same backing and rights.
120	197		export fn split(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, leftCount: u64)
121	198		-> abi::Handle throws (abi::Error)

55	55		/// Return unpublished capacity after a failed load.
56	56		export fn cancel(store: &mut Store, reservation: slots::Reservation) {
57	57		try! slots::cancel(&mut store.slots[..], reservation);
58	58		}
59	59
60		-	/// Validate dependencies without changing registry payloads.
61		-	unsafe fn validate(store: &Store, object: abi::Ref, package: &shared::Package) throws (abi::Error) {
	60	+	/// Validated immutable package metadata ready for bounded publication.
	61	+	export record Admission: Copy {
	62	+	/// Reserved package generation to receive the descriptor.
	63	+	object: abi::Ref,
	64	+	/// Persistent source and native descriptor.
	65	+	entry: catalog::Entry,
	66	+	/// Resolved resident dependencies in their declared order.
	67	+	dependencies: [abi::Ref; limits::PACKAGES],
	68	+	/// Number of initialized dependency references.
	69	+	count: u32,
	70	+	}
	71	+
	72	+	/// Resolve admission against immutable packages while holding sole publisher ownership.
	73	+	/// The caller keeps that ownership and the descriptor storage through commit or cancellation.
	74	+	export unsafe fn admit(store: &Store, object: abi::Ref, source: *[u8], package: shared::Package) -> Admission throws (abi::Error) {
	75	+	if not slots::matches(&store.slots[..], object, slots::State::Reserved) { throw abi::Error::BadHandle; }
	76	+	if try identify(store, &package.name[..], &source[..]) <> nil { throw abi::Error::Busy; }
62	77		if package.name.len == 0 or package.slot <> object.index or package.dependencies.len > limits::PACKAGES {
63	78		throw abi::Error::VerifyFailed;
64	79		}
	80	+	let mut result = Admission { object, entry: catalog::Entry { source, package }, dependencies: undefined, count: package.dependencies.len };
65	81		for dependency, i in package.dependencies {
66	82		let target = find(store, &dependency[..]) else { throw abi::Error::VerifyFailed; };
67		-	for prior in &package.dependencies[..i] {
68		-	if mem::eq(prior, dependency) { throw abi::Error::VerifyFailed; }
	83	+	for prior in &result.dependencies[..i] {
	84	+	if prior == target { throw abi::Error::VerifyFailed; }
69	85		}
	86	+	set result.dependencies[i] = target;
	87	+	}
	88	+	return result;
	89	+	}
	90	+
	91	+	/// Publish admitted metadata under serialization using only bounded reference copies.
	92	+	export fn commit(store: &mut Store, reservation: slots::Reservation, admission: Admission) -> abi::Ref {
	93	+	let object = slots::reference(&reservation);
	94	+	assert object == admission.object and slots::matches(&store.slots[..], object, slots::State::Reserved);
	95	+	assert admission.count <= limits::PACKAGES;
	96	+	for i in 0..admission.count {
	97	+	let target = admission.dependencies[i];
	98	+	assert slots::matches(&store.slots[..], target, slots::State::Live);
	99	+	set store.dependencies[object.index][i] = target;
70	100		}
	101	+	set store.counts[object.index] = admission.count;
	102	+	set store.entries[object.index] = admission.entry;
	103	+	return try! slots::commit(&mut store.slots[..], reservation);
71	104		}
72	105
73	106		/// Publish persistent compiler output after successful admission checks.
74	107		/// All descriptor pointers and source bytes must remain immutable and resident.
75	108		/// Failure cancels the reservation; duplicate content returns its existing slot.

79	112		let existing = try identify(store, &package.name[..], &source[..]) catch err {
80	113		cancel(store, reservation); throw err;
81	114		};
82	115		if let object = existing { cancel(store, reservation); return object; }
83	116		let object = slots::reference(&reservation);
84		-	try validate(store, object, &package) catch err { cancel(store, reservation); throw err; };
85		-	for dependency, i in package.dependencies {
86		-	let target = find(store, &dependency[..]) else panic "resident dependency";
87		-	set store.dependencies[object.index][i] = target;
88		-	}
89		-	set store.counts[object.index] = package.dependencies.len;
90		-	set store.entries[object.index] = catalog::Entry { source, package };
91		-	return try! slots::commit(&mut store.slots[..], reservation);
	117	+	let admission = try admit(store, object, source, package) catch err { cancel(store, reservation); throw err; };
	118	+	return commit(store, reservation, admission);
92	119		}
93	120
94	121		/// Read resident metadata through a generation-bearing package reference.
95	122		export fn get(store: &Store, object: abi::Ref) -> shared::Package throws (abi::Error) {
96	123		if not slots::matches(&store.slots[..], object, slots::State::Live) { throw abi::Error::BadHandle; }

8	8		.export @kernel::sync::fetchAdd;
9	9		.export @kernel::sync::syncInstructions;
10	10		.export @kernel::sync::deviceFence;
11	11		.export @kernel::sync::read32;
12	12		.export @kernel::sync::write32;
	13	+	.export @kernel::sync::maskInterrupts;
	14	+	.export @kernel::sync::restoreInterrupts;
	15	+	.export @kernel::sync::retired;
	16	+
	17	+	// Preserve the previous interrupt-enable bit while masking machine interrupts.
	18	+	@kernel::sync::maskInterrupts
	19	+	csrr %t0 mstatus;
	20	+	andi %a0 %t0 8;
	21	+	andi %t0 %t0 -9;
	22	+	csrw mstatus %t0;
	23	+	ret;
	24	+
	25	+	// Restore interrupt delivery while preserving other machine status fields.
	26	+	@kernel::sync::restoreInterrupts
	27	+	csrr %t0 mstatus;
	28	+	andi %t0 %t0 -9;
	29	+	or %t0 %t0 %a0;
	30	+	csrw mstatus %t0;
	31	+	ret;
	32	+
	33	+	// Count local retired instructions independently of other harts' progress.
	34	+	@kernel::sync::retired
	35	+	csrr %a0 instret;
	36	+	ret;
13	37
14	38		// Allocate a wrapping ticket and return its zero-extended u32 value.
15	39		@kernel::sync::nextTicket
16	40		li %t0 1;
17	41		amoadd.w.aqrl %a0 %t0 (%a0);

164	164		/// An unknown user operation advances PC and returns only a negative a0 error.
165	165		@test unsafe fn trapReply() throws (testing::TestError) {
166	166		let owner = initialize();
167	167		let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
168	168		set frame.registers[17] = 0xffffffffffffffff;
169		-	calls::handle(owner, &mut frame, 0);
	169	+	calls::handle(owner, &mut frame, 0, calls::invoke);
170	170		try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc);
171	171		for i in 0..32 {
172	172		if i == 10 or i == 17 { continue; }
173	173		try testing::expect(frame.registers[i] == 123);
174	174		}

57	57		try testing::expect(ended.context == nil and ended.deadline == 0xffffffffffffffff);
58	58		let other = try! dispatch::select(&BUDGETS, &DOMAINS, 1, 10);
59	59		try testing::expect(other.context == nil and other.deadline == 0xffffffffffffffff);
60	60		}
61	61
	62	+	/// A context becomes eligible only after its previous hart releases ownership.
	63	+	@test unsafe fn ownership() throws (testing::TestError) {
	64	+	initialize();
	65	+	set DOMAINS.contexts[0].hart = 1;
	66	+	let future = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
	67	+	assert future.context == nil and future.deadline == 10;
	68	+	let occupied = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
	69	+	assert occupied.context == nil and occupied.deadline == 20;
	70	+	set DOMAINS.contexts[0].hart = nil;
	71	+	let free = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
	72	+	assert free.context == TABLE.owner and free.deadline == 20;
	73	+	set DOMAINS.contexts[0].hart = 0;
	74	+	let local = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
	75	+	assert local == free;
	76	+	}
	77	+
62	78		/// Pending domains and stale context generations cannot become runnable through a budget.
63	79		@test unsafe fn lifetimes() throws (testing::TestError) {
64	80		initialize();
65	81		set DOMAINS.records[0].state = domains::Lifecycle::Pending;
66	82		let pending = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

122	138		stack: range::Range { start: 0x2000, end: 0x4000 },
123	139		};
124	140		let mut anchor = trap::Hart {
125	141		stackTop: 0x4000, stackBottom: 0x2000, kernelGp: 0x20000, handler,
126	142		savedT0: 0, savedT1: 0, savedSp: 0,
	143	+	dispatchTop: 0x4000, dispatchFrame: undefined,
127	144		};
128	145		let first = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 10);
129	146		try testing::expect(frame.pc == 0x40 and anchor.stackTop == 0xc000);
130	147		try testing::expect(DOMAINS.contexts[0].hart == 0);
131	148		set frame.pc = 0x88;

129	129		try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
130	130		try testing::expect(error == abi::Error::OutOfMemory); set exhausted = true;
131	131		};
132	132		try testing::expect(exhausted and DOMAINS.contextSlots[child.initial.index + 1].state == slots::State::Free);
133	133		set PAGES.backings.pool.count = count;
134		-	let context = try! domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
	134	+	let reserved = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
	135	+	let index = child.initial.index + 1;
	136	+	assert DOMAINS.contextSlots[index].state == slots::State::Reserved;
	137	+	let mut overlap = false;
	138	+	try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
	139	+	assert error == abi::Error::Busy; set overlap = true;
	140	+	};
	141	+	assert overlap;
	142	+	let reservedFrames = DOMAINS.contexts[index].kernelFrames;
	143	+	domains::contextCancel(&mut DOMAINS, &mut PAGES, reserved);
	144	+	assert DOMAINS.contextSlots[index].state == slots::State::Free;
	145	+	for i in reservedFrames.first..reservedFrames.first + reservedFrames.count { assert PAGES.backings.pool.free[i]; }
	146	+	let pendingContext = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
	147	+	let stackBase = DOMAINS.contexts[index].kernelStack.start;
	148	+	let base = (&RAM[0]) as u64;
	149	+	let firstWord = ((stackBase - base) / 8) as u32;
	150	+	let lastWord = firstWord + domains::KERNEL_STACK_PAGES * 512 - 1;
	151	+	set RAM[firstWord] = 123; set RAM[lastWord] = 456;
	152	+	domains::contextClear(&pendingContext);
	153	+	assert RAM[firstWord] == 0 and RAM[lastWord] == 0;
	154	+	let context = domains::contextPublish(&mut DOMAINS, pendingContext);
135	155		let created = try! domains::context(&DOMAINS, authority.object, context);
136	156		let initial = DOMAINS.contexts[child.initial.index];
137	157		try testing::expect(context <> child.initial and created.owner == authority.object);
138	158		try testing::expect(created.frame.registers[3] == initial.frame.registers[3]);
139	159		try testing::expect(created.kernelStack.start >= initial.kernelStack.end or created.kernelStack.end <= initial.kernelStack.start);

244	264		assert (error == abi::Error::BadHandle); set rejected += 1;
245	265		};
246	266		assert (rejected == 3);
247	267		}
248	268
	269	+	/// Domain reservations retain private capacity across another complete creation.
	270	+	@test unsafe fn stagedCreation() throws (testing::TestError) {
	271	+	initialize(abi::CREATE);
	272	+	let pending = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	273	+	assert DOMAINS.slots[1].state == slots::State::Reserved;
	274	+	assert DOMAINS.contextSlots[0].state == slots::State::Reserved;
	275	+	assert TABLE.slots[2].state == slots::State::Reserved;
	276	+	assert BACKINGS.domains[1] == 0 and DOMAINS.events.queues[1].generation == 0;
	277	+	let other = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	278	+	try! domains::prepare(&PACKAGES, &pending);
	279	+	let first = try! domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, pending);
	280	+	let a = try! capability::get(&TABLE, first);
	281	+	let b = try! capability::get(&TABLE, other);
	282	+	let left = try! domains::get(&DOMAINS, a.object);
	283	+	let right = try! domains::get(&DOMAINS, b.object);
	284	+	assert a.object.index == 1 and b.object.index == 2;
	285	+	assert left.graph.table[0] <> right.graph.table[0];
	286	+	assert left.state == domains::Lifecycle::Pending and right.state == domains::Lifecycle::Pending;
	287	+	initialize(abi::CREATE);
	288	+	let cancelled = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	289	+	domains::cancel(&mut DOMAINS, &mut BACKINGS, &mut TABLE, cancelled);
	290	+	assert DOMAINS.slots[1].state == slots::State::Free;
	291	+	assert DOMAINS.contextSlots[0].state == slots::State::Free;
	292	+	assert TABLE.slots[2].state == slots::State::Free;
	293	+	for i in 0..BACKINGS.pool.count { assert BACKINGS.pool.free[i]; }
	294	+	set PACKAGES.entries[0].package.relocations = &[shared::Relocation {
	295	+	offset: 0, count: 1, target: shared::DataRef { slot: 5, offset: 0 },
	296	+	}];
	297	+	let mut invalid = false;
	298	+	try domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE) catch error {
	299	+	assert error == abi::Error::VerifyFailed; set invalid = true;
	300	+	};
	301	+	assert invalid and DOMAINS.slots[1].state == slots::State::Free;
	302	+	for i in 0..BACKINGS.pool.count { assert BACKINGS.pool.free[i]; }
	303	+	initialize(abi::CREATE);
	304	+	let abandoned = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	305	+	try! domains::prepare(&PACKAGES, &abandoned);
	306	+	try! backing::endDomain(&mut BACKINGS, TABLE.owner);
	307	+	let mut dead = false;
	308	+	try domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, abandoned) catch error {
	309	+	assert error == abi::Error::BadHandle; set dead = true;
	310	+	};
	311	+	assert dead and DOMAINS.slots[1].state == slots::State::Free;
	312	+	assert TABLE.slots[2].state == slots::State::Free;
	313	+	for i in 0..BACKINGS.pool.count { assert BACKINGS.pool.free[i]; }
	314	+	}
	315	+
249	316		/// Creation installs only Events in the child and preserves selected authorizing rights.
250	317		@test unsafe fn pending() throws (testing::TestError) {
251	318		initialize(abi::CREATE \| abi::ALLOCATE);
252	319		let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
253	320		let authority = try! capability::get(&TABLE, handle);

48	48		fn word(address: u64) -> *mut u64 {
49	49		let pointer = &RAM[0];
50	50		return &mut RAM[((address - pointer as u64) / 8) as u32];
51	51		}
52	52
	53	+	/// Private initialization retains its graph across other registry and frame transactions.
	54	+	@test unsafe fn stagedGraph() throws (testing::TestError) {
	55	+	initialize();
	56	+	let root = register("root", &[], &[]);
	57	+	let pending = try! instances::reserve(&PACKAGES, &mut POOL, root);
	58	+	let later = register("later", &[], &[]);
	59	+	let other = try! instances::create(&PACKAGES, &mut POOL, later);
	60	+	set *word(other.table[later.index]) = 99;
	61	+	try! instances::initialize(&PACKAGES, &pending);
	62	+	let first = instances::commit(pending);
	63	+	assert first.table[root.index] <> 0 and first.table[later.index] == 0;
	64	+	assert *word(first.table[root.index]) == 7;
	65	+	assert *word(other.table[later.index]) == 99;
	66	+	assert first.frames.first + first.frames.count <= other.frames.first;
	67	+	try! frames::release(&mut POOL, first.frames);
	68	+	try! frames::release(&mut POOL, other.frames);
	69	+	let cancelled = try! instances::reserve(&PACKAGES, &mut POOL, root);
	70	+	instances::cancel(&mut POOL, cancelled);
	71	+	for i in 0..POOL.count { assert POOL.free[i]; }
	72	+	}
	73	+
53	74		/// A diamond has one dependency instance per domain and independent relocated pointers.
54	75		@test unsafe fn diamond() throws (testing::TestError) {
55	76		initialize();
56	77		let base = register("base", &[], &[]);
57	78		unsafe static relocation: [shared::Relocation; 1] = undefined;

83	83		let mut count: u32 = 0;
84	84		for i in 0..PAGES.backings.pool.count { if PAGES.backings.pool.free[i] { set count += 1; } }
85	85		return count;
86	86		}
87	87
	88	+	/// Reserved input stays private across allocation, source mutation, and contention.
	89	+	@test unsafe fn stagedInput() throws (testing::TestError) {
	90	+	initialize();
	91	+	let request = input(43, false);
	92	+	let before = free();
	93	+	let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
	94	+	assert LOADER.busy and TABLE.slots[2].state == slots::State::Reserved;
	95	+	let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
	96	+	let decoded = try! loader::decode(&pending);
	97	+	assert decoded.program.fns.len == 1 and decoded.name.len == 6;
	98	+	let replacement = input(99, false);
	99	+	assert decoded.program.fns[0].blocks[0].instrs[0] == il::Instr::Ret { val: il::Val::Imm(43) };
	100	+	let mut busy = false;
	101	+	try loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request) catch error {
	102	+	assert error == abi::Error::Busy; set busy = true;
	103	+	};
	104	+	assert busy;
	105	+	loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
	106	+	assert not LOADER.busy and TABLE.slots[2].state == slots::State::Free;
	107	+	assert free() + 1 == before;
	108	+	assert (try! pages::get(&PAGES, (try! capability::get(&TABLE, other)).object)).count == 1;
	109	+	}
	110	+
	111	+	/// Compiled output stays private until publication and cancellation restores capacity.
	112	+	@test unsafe fn stagedOutput() throws (testing::TestError) {
	113	+	initialize();
	114	+	let request = input(43, false);
	115	+	let before = free();
	116	+	let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
	117	+	let decoded = try! loader::decode(&pending);
	118	+	let output = try! loader::reserveOutput(&mut PAGES.backings.pool, &mut PACKAGES);
	119	+	assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
	120	+	let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
	121	+	let used = try! loader::compile(&PACKAGES, &pending, &decoded, &output);
	122	+	assert used.code > 0 and used.metadata > 0;
	123	+	assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
	124	+	loader::cancelOutput(&mut PAGES.backings.pool, &mut PACKAGES, output);
	125	+	loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
	126	+	assert free() + 1 == before and not LOADER.busy;
	127	+	assert PACKAGES.slots[0].state == slots::State::Free;
	128	+	let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
	129	+	assert (try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0))).index == 0;
	130	+	}
	131	+
	132	+	/// A caller whose backing lifetime ended cannot receive a completed Image handle.
	133	+	@test unsafe fn completionLifetime() throws (testing::TestError) {
	134	+	initialize();
	135	+	let request = input(43, false);
	136	+	let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
	137	+	let object = try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0));
	138	+	let before = free();
	139	+	let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
	140	+	try! backing::endDomain(&mut PAGES.backings, TABLE.owner);
	141	+	let mut dead = false;
	142	+	try loader::finish(&mut LOADER, &mut PAGES, &mut TABLE, pending, object) catch error {
	143	+	assert error == abi::Error::BadHandle; set dead = true;
	144	+	};
	145	+	assert dead and not LOADER.busy and free() == before;
	146	+	assert TABLE.slots[3].state == slots::State::Free;
	147	+	assert slots::matches(&PACKAGES.slots[..], object, slots::State::Live);
	148	+	}
	149	+
88	150		/// Resident source bytes survive mutation of the source page and workspace release.
89	151		@test unsafe fn residentSnapshot() throws (testing::TestError) {
90	152		initialize();
91	153		let request = input(43, false);
92	154		let before = free();

41	41		/// Drop the test's handle and its object reference in the same serialized operation.
42	42		unsafe fn drop(handle: abi::Handle) {
43	43		try! pages::drop(&mut STORE, &mut TABLE, handle);
44	44		}
45	45
	46	+	/// Reserved frames stay private while another allocation changes shared metadata.
	47	+	@test unsafe fn stagedAllocation() throws (testing::TestError) {
	48	+	initialize();
	49	+	let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 2);
	50	+	assert STORE.slots[0].state == slots::State::Reserved;
	51	+	assert TABLE.slots[1].state == slots::State::Reserved;
	52	+	assert not STORE.backings.pool.free[0] and not STORE.backings.pool.free[1];
	53	+	assert RAM[OFFSET] == 0xa5;
	54	+	let other = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 6);
	55	+	set RAM[OFFSET + 8192] = 0x7b;
	56	+	pages::clear(&pending);
	57	+	for i in OFFSET..OFFSET + 8192 { assert RAM[i] == 0; }
	58	+	assert RAM[OFFSET + 8192] == 0x7b;
	59	+	let handle = pages::publish(&mut STORE, &mut TABLE, pending);
	60	+	let entry = try! capability::get(&TABLE, handle);
	61	+	let page = try! pages::get(&STORE, entry.object);
	62	+	assert page.count == 2 and page.origin == TABLE.owner;
	63	+	drop(handle); drop(other);
	64	+	try! backing::endDomain(&mut STORE.backings, TABLE.owner);
	65	+	for i in 0..8 { assert STORE.backings.pool.free[i]; }
	66	+	}
	67	+
	68	+	/// Cancelling unpublished storage returns all capacity without exposing its bytes.
	69	+	@test unsafe fn cancelAllocation() throws (testing::TestError) {
	70	+	initialize();
	71	+	let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 8);
	72	+	pages::cancel(&mut STORE, &mut TABLE, pending);
	73	+	assert STORE.slots[0].state == slots::State::Free;
	74	+	assert STORE.backings.slots[0].state == slots::State::Free;
	75	+	assert TABLE.slots[1].state == slots::State::Free;
	76	+	for i in 0..8 { assert STORE.backings.pool.free[i]; }
	77	+	assert RAM[OFFSET] == 0xa5;
	78	+	let full = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
	79	+	let entry = try! capability::get(&TABLE, full);
	80	+	assert (try! pages::get(&STORE, entry.object)).count == 8;
	81	+	}
	82	+
46	83		/// Transfers keep both domains exposed without duplicating page handles.
47	84		@test unsafe fn transferExposure() throws (testing::TestError) {
48	85		initialize();
49	86		let receiver = abi::Ref { index: 1, generation: 1 };
50	87		capability::initialize(&mut TARGET, receiver);

27	27		let pending = try registry::reserve(&mut STORE);
28	28		let slot = slots::reference(&pending);
29	29		return try registry::publish(&mut STORE, pending, bytes, package(name, dependencies, slot.index));
30	30		}
31	31
	32	+	/// Admission resolves dependencies before its reserved package becomes visible.
	33	+	@test unsafe fn stagedAdmission() throws (testing::TestError) {
	34	+	registry::initialize(&mut STORE);
	35	+	let base = try! register("base", &[], "base bytes");
	36	+	let pending = try! registry::reserve(&mut STORE);
	37	+	let object = slots::reference(&pending);
	38	+	let admitted = try! registry::admit(&STORE, object, "child bytes", package("child", &["base"], object.index));
	39	+	assert STORE.slots[object.index].state == slots::State::Reserved;
	40	+	assert registry::find(&STORE, &"child"[..]) == nil;
	41	+	assert admitted.dependencies[0] == base;
	42	+	let published = registry::commit(&mut STORE, pending, admitted);
	43	+	assert published == object and STORE.counts[object.index] == 1;
	44	+	assert STORE.dependencies[object.index][0] == base;
	45	+	let rejected = try! registry::reserve(&mut STORE);
	46	+	let slot = slots::reference(&rejected);
	47	+	let mut duplicate = false;
	48	+	try registry::admit(&STORE, slot, "bad bytes", package("bad", &["base", "base"], slot.index)) catch error {
	49	+	assert error == abi::Error::VerifyFailed; set duplicate = true;
	50	+	};
	51	+	assert duplicate and STORE.slots[slot.index].state == slots::State::Reserved;
	52	+	registry::cancel(&mut STORE, rejected);
	53	+	assert STORE.slots[slot.index].state == slots::State::Free;
	54	+	}
	55	+
32	56		/// Identical bytes reuse one package; a name cannot identify different content.
33	57		@test unsafe fn identity() throws (testing::TestError) {
34	58		registry::initialize(&mut STORE);
35	59		let first = try! register("a", &[], "binary a");
36	60		let second = try! register("a", &[], "binary a");

4	4		use kernel::trap;
5	5
6	6		/// The assembly frame includes x0-x31 and all four trap CSRs.
7	7		@test fn layout() throws (testing::TestError) {
8	8		try testing::expect(@sizeOf(trap::Frame) == 288);
9		-	try testing::expect(@sizeOf(trap::Hart) == 56);
	9	+	assert @sizeOf(trap::Hart) == 352;
10	10		try testing::expect(trap::FRAME_SIZE % 16 == 0);
11	11		}
12	12
13	13		/// Interrupts retain their source identity; exceptions retain their fault class.
14	14		@test fn classification() throws (testing::TestError) {