radiance · commit

kernel: Start secondary harts

ba7c33175d691a55b5e48ba7b6804e421b4c3469a02ee8914c5e67604f67c28d

Serialize shared metadata with interrupt-masked linear guards. Keep
initialization and compilation outside the lock, exchange contexts on
hart-owned stacks, and gate dispatch on complete hart initialization.

Verify concurrent contexts of one domain on 1, 2, and 8 harts,
repeated capability transactions, register isolation, and measured
metadata work. Full library, kernel, compiler, and machine suites
pass.

Assisted-by: Codex:gpt-6

Alexis Sellier committed 13 days ago 1 parent 34415325

kernel/kernel/boot.rad +12 -2

    assert stackTop == PLATFORM.stacks[hart as u32].end and (stackTop & 15) == 0;
    let stack = PLATFORM.stacks[hart as u32];
    set HARTS[hart as u32] = trap::Hart {
        stackTop: stack.end, stackBottom: stack.start, kernelGp: stateTable as u64, handler: unexpected,
        savedT0: 0, savedT1: 0, savedSp: 0,
        dispatchTop: stack.end, dispatchFrame: undefined,
    };
    trap::install(&mut HARTS[hart as u32]);
    try! dispatch::initialize(&PLATFORM, hart as u32, stateTable as u64);
    let mut count: u64 = 0;
    for id in 0..limits::HARTS {

        }
    }
    return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
}

/// Enter dispatch after this hart completes platform initialization.
export unsafe fn run() {
/// Enter dispatch after every online hart publishes its initialized state.
export unsafe fn run() -> ! {
    let mut count: u64 = 0;
    for id in 0..limits::HARTS {
        if (PLATFORM.harts & (1 << id)) <> 0 {
            set count += 1;
        }
    }
    while sync::loadAcquire(&ARRIVED) < count {
    }
    sync::syncInstructions();
    dispatch::start(&mut HARTS[dispatch::hart()]);
}

kernel/kernel/calls.rad +167 -3

use super::events;
use super::transactions;
use super::loader;
use super::dispatch;
use super::timers;
use super::sync;

/// Validate resident object generations for capabilities without page ownership.
unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => {

            budgets::Binding { budget: handle, domain: abi::Handle(arguments[1]), context: abi::reference(arguments[2]) }, now),
        else => throw abi::Error::InvalidArg,
    }
}

/// Complete a user ecall and preserve all registers except its scalar result.
export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64) {
/// Check a retained domain's lifetime while metadata is serialized.
unsafe fn live(owner: abi::Ref) -> bool {
    return slots::matches(&domains::STORE.slots[..], owner, slots::State::Live)
        and domains::STORE.records[owner.index].state <> domains::Lifecycle::Dead;
}

/// Load a package with metadata locked only around resource transactions.
unsafe fn runtimeLoad(domain: &mut domains::Domain, request: loader::Request, guard: sync::Guard) -> u64 throws (abi::Error) {
    let owner = domain.memory.table.owner;
    let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
        catch error {
            sync::leave(guard);
            throw error;
        };
    sync::leave(guard);
    let input = try loader::decode(&pending) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(guard); throw error;
    };
    let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
        let guard = sync::enter();
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(guard); throw error;
    };
    let allocationGuard = sync::enter();
    if not live(owner) {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(allocationGuard); throw abi::Error::BadHandle;
    }
    if let object = existing {
        let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
            catch error {
                sync::leave(allocationGuard);
                throw error;
            };
        sync::leave(allocationGuard); return *result;
    }
    let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(allocationGuard); throw error;
    };
    sync::leave(allocationGuard);
    let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
        let guard = sync::enter();
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(guard); throw error;
    };
    let publicationGuard = sync::enter();
    if not live(owner) {
        loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
        loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
        sync::leave(publicationGuard); throw abi::Error::BadHandle;
    }
    let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
    let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
        catch error {
            sync::leave(publicationGuard);
            throw error;
        };
    sync::leave(publicationGuard);
    return *result;
}

/// Execute a call with short metadata transactions and private bulk initialization.
/// The running context keeps its domain storage live until dispatch releases it.
export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
    assert arguments.len == 4;
    let guard = sync::enter();
    let mut domain = try domains::get(&domains::STORE, owner) catch error {
        sync::leave(guard);
        throw error;
    };
    if domain.state <> domains::Lifecycle::Active {
        sync::leave(guard);
        throw abi::Error::BadHandle;
    }
    let handle = abi::Handle(arguments[0]);
    match operation {
        case 20 => {
            let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
                &mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error {
                    sync::leave(guard);
                    throw error;
                };
            sync::leave(guard);
            try domains::prepare(&registry::STORE, &pending) catch error {
                let guard = sync::enter();
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                sync::leave(guard); throw error;
            };
            let guard = sync::enter();
            if not live(owner) {
                domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
                catch error {
                    sync::leave(guard);
                    throw error;
                };
            sync::leave(guard); return *result;
        },
        case 30 => {
            let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
                catch error {
                    sync::leave(guard);
                    throw error;
                };
            sync::leave(guard);
            pages::clear(&pending);
            let guard = sync::enter();
            if not live(owner) {
                pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
            sync::leave(guard); return *result;
        },
        case 51 => return try runtimeLoad(&mut domain, loader::Request {
            authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
        }, guard),
        case 60 => {
            if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
                sync::leave(guard); throw abi::Error::InvalidArg;
            }
            let words = @sliceOf(memory(arguments[1]), 4);
            let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
            let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
                &domain.memory.table, handle, start) catch error {
                    sync::leave(guard);
                    throw error;
                };
            let mut target: abi::Ref = undefined;
            match &pending {
                case domains::ContextReservation::Held(allocation) => {
                    set target = allocation.owner;
                },
            }
            sync::leave(guard);
            domains::contextClear(&pending);
            let guard = sync::enter();
            if not live(owner) or not live(target) {
                domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
                sync::leave(guard); throw abi::Error::BadHandle;
            }
            let result = domains::contextPublish(&mut domains::STORE, pending);
            sync::leave(guard); return abi::id(result);
        },
        else => {
            let result = try invoke(owner, operation, arguments, now) catch error {
                sync::leave(guard);
                throw error;
            };
            sync::leave(guard); return result;
        },
    }
}

/// Complete a user ecall through the supplied execution boundary.
export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
    execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
{
    assert trap::fromUser(frame) and frame.cause == 8;
    assert trap::advanceCall(frame);
    let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
    let result = try invoke(owner, frame.registers[17], &args[..], now) catch error {
    let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
        set frame.registers[10] = (0 as u64) - error as u64;
        return;
    };
    set frame.registers[10] = result;
}

kernel/kernel/dispatch.rad +28 -10

use super::platform;
use super::trap;
use super::range;
use super::calls;
use super::timers;
use super::sync;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

/// Runtime slots initialized before each hart enables timer interrupts.
unsafe static HARTS: [Runtime; limits::HARTS] = undefined;

/// Read the executing physical hart identifier.
export fn hart() -> u32;
/// Read the current machine stack pointer.
fn stackPointer() -> u64;
/// Address of the M-mode idle loop.
fn idleAddress() -> u64;
/// Enable machine timer delivery while global interrupts remain disabled.
fn enable();


        state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
    };
    arm(device, 0xffffffffffffffff);
}

/// Reevaluate authority after a timer interrupt with shared metadata serialized.
export unsafe fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
/// Tail-enter dispatch on the firmware stack and resume the selected frame.
export fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> !;

/// Select authority from hart-owned storage with shared metadata serialized.
export unsafe fn choose(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
    let id = hart();
    assert id < limits::HARTS;
    let stack = HARTS[id].state.stack;
    let pointer = stackPointer();
    assert pointer >= stack.start and pointer < stack.end;
    let guard = sync::enter();
    let clock = now(HARTS[id].timer);
    let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    let mut deadline = choice.deadline;
    if timeout < deadline {
        set deadline = timeout;
    }
    arm(HARTS[id].timer, deadline);
    sync::leave(guard);
}

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();

    let window = try! budgets::get(&budgets::STORE, budget);
    return abi::CurrentContextInfo { context: abi::id(context), hart: id as u64, budget: *window.handle };
}

/// Complete the current user's call and select authority at the return boundary.
export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else {
        trap::halt();
        return;
    };
    let context = HARTS[id].state.current else trap::halt();
    let guard = sync::enter();
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    let owner = domains::STORE.contexts[context.index].owner;
    if frame.registers[17] == abi::Operation::ContextReturn as u64 {
        assert trap::advanceCall(frame);
        try! domains::returned(&mut domains::STORE, owner, context);
        sync::leave(guard);
    } else {
        calls::handle(owner, frame, now(HARTS[id].timer));
        sync::leave(guard);
        calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
    }
    interrupt(frame, anchor);
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let mut frame = HARTS[id].state.idle;
    interrupt(&mut frame, anchor);
    enable();
    trap::resume(&frame);
    interrupt(&mut frame, anchor);
}

/// Read one validated, naturally aligned 64-bit device register.
fn read(address: u64) -> u64;
/// Write one validated, naturally aligned 64-bit device register.

            if window.start < result.deadline {
                set result.deadline = window.start;
            }
            continue;
        }
        if let owner = contexts.contexts[context.index].hart {
            if owner <> hart {
                if window.end < result.deadline {
                    set result.deadline = window.end;
                }
                continue;
            }
        }
        assert result.budget == nil;
        set result.budget = abi::Ref { index: i, generation: windows.slots[i].generation };
        set result.context = context;
        if window.end < result.deadline {
            set result.deadline = window.end;

kernel/kernel/dispatch.ras +26 -0

//! Naturally aligned RV64 CLINT access with device ordering.
.text;
.export @kernel::dispatch::read;
.export @kernel::dispatch::write;
.export @kernel::dispatch::hart;
.export @kernel::dispatch::stackPointer;
.export @kernel::dispatch::idleAddress;
.export @kernel::dispatch::enable;
.export @kernel::dispatch::interrupt;

// Retain the frame outside the context stack before publishing its release.
@kernel::dispatch::interrupt
    mv %t0 %a0;
    addi %t1 %a1 64;
    li %t2 36;
@retainDispatchFrame
    ld %t3 0(%t0);
    sd %t3 0(%t1);
    addi %t0 %t0 8;
    addi %t1 %t1 8;
    addi %t2 %t2 -1;
    bnez %t2 @retainDispatchFrame;
    ld %sp 56(%a1);
    addi %a0 %a1 64;
    call @kernel::dispatch::choose;
    csrr %a0 mscratch;
    addi %a0 %a0 64;
    j @kernel::trap::resume;

// The executing hart selects its private dispatch runtime slot.
@kernel::dispatch::hart
    csrr %a0 mhartid;
    ret;

// Dispatch releases context stacks only from the hart's firmware stack.
@kernel::dispatch::stackPointer
    mv %a0 %sp;
    ret;

// Saved idle frames enter this loop with machine interrupts enabled by mret.
@kernel::dispatch::idleAddress
    la %a0 @dispatchIdle;
    ret;
@dispatchIdle

kernel/kernel/domains.rad +216 -58

        }
    }
    throw abi::Error::InvalidArg;
}

/// Reject physical overlap with any live context's reserved user stack.
/// Reject physical overlap with published or reserved user stacks.
fn stackAvailable(store: &Store, candidate: range::Range) throws (abi::Error) {
    for i in 0..limits::CONTEXTS {
        if store.contextSlots[i].state <> slots::State::Live {
        if store.contextSlots[i].state <> slots::State::Live and store.contextSlots[i].state <> slots::State::Reserved {
            continue;
        }
        let used = store.contexts[i].userStack;
        if used.start < used.end and candidate.start < used.end and used.start < candidate.end {
            throw abi::Error::Busy;

        }
    }
    return false;
}

/// Create an additional context with private kernel storage and shared domain state.
/// The caller serializes domain, capability, and physical allocation metadata.
export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
    table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
/// Private kernel storage retained by an unpublished context.
export record ContextAllocation: Copy {
    /// Reserved context slot with an initialized user-stack claim.
    object: abi::Ref,
    /// Domain kept live until publication or cancellation.
    owner: abi::Ref,
    /// Reserved kernel-stack frames.
    frames: frames::Run,
    /// Validated physical base for private stack clearing.
    base: u64,
}

/// Context storage that must be published or cancelled under serialization.
export union ContextReservation: Once {
    /// One reserved context and its private kernel storage.
    Held(ContextAllocation),
}

/// Reserve an additional context and both stack claims under metadata serialization.
export unsafe fn contextReserve(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
    table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> ContextReservation throws (abi::Error)
{
    let permission = try capability::authority(table, authority, abi::Rights(abi::EXECUTE));
    let domain = try get(store, permission.object);
    if domain.state == Lifecycle::Dead {
        throw abi::Error::BadHandle;

    let pending = try frames::reserve(&mut memory.backings.pool, KERNEL_STACK_PAGES) catch error {
        try! slots::cancel(&mut store.contextSlots[..], slot); throw error;
    };
    let kernelFrames = frames::commit(pending);
    let kernelStack = try! frames::extent(&memory.backings.pool, kernelFrames);
    zero(kernelStack.start, kernelFrames.count);
    let mut frame = trap::Frame { registers: [0; 32], pc: start.entry, status: 0x80, cause: 0, value: 0 };
    set frame.registers[1] = returnAddress();
    set frame.registers[2] = start.stack;
    set frame.registers[3] = domain.graph.table.ptr as u64;
    set frame.registers[10] = start.args;
    set frame.registers[11] = start.size;
    let object = slots::reference(&slot);
    set store.contexts[object.index] = Context {
        owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
    };
    return try! slots::commit(&mut store.contextSlots[..], slot);
    match slot {
        case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
            object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
        }),
    }
}

/// Clear an unpublished context's private kernel stack.
export fn contextClear(reservation: &ContextReservation) {
    match reservation { case ContextReservation::Held(allocation) => zero(allocation.base, allocation.frames.count), }
}

/// Check the reserved context and its retained physical allocation.
fn contextRequired(store: &Store, allocation: &ContextAllocation) {
    assert slots::matches(&store.contextSlots[..], allocation.object, slots::State::Reserved);
    let context = store.contexts[allocation.object.index];
    assert context.owner == allocation.owner and context.kernelFrames == allocation.frames;
    assert context.hart == nil;
}

/// Publish an initialized context while its domain stays live under serialization.
export fn contextPublish(store: &mut Store, reservation: ContextReservation) -> abi::Ref {
    match reservation {
        case ContextReservation::Held(allocation) => {
            contextRequired(store, &allocation);
            assert slots::matches(&store.slots[..], allocation.owner, slots::State::Live);
            return try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
        },
    }
}

/// Return an unpublished context's stack and slot under metadata serialization.
export fn contextCancel(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation) {
    match reservation {
        case ContextReservation::Held(allocation) => {
            contextRequired(store, &allocation);
            try! frames::release(&mut memory.backings.pool, allocation.frames);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
        },
    }
}

/// Create a context while the caller has exclusive access to shared metadata.
export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
    table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
{
    let pending = try contextReserve(store, memory, packages, table, authority, start);
    contextClear(&pending);
    return contextPublish(store, pending);
}

/// Memory allocated before the domain and its context become observable.
record Prepared: Copy {
export record Prepared: Copy {
    /// Frames for capabilities and the event ring.
    allocation: frames::Run,
    /// Mapped metadata at the allocation's start.
    memory: *unsafe mut Memory,
    /// Independent package-state graph.
    graph: instances::Instance,
    /// Reserved private package-state graph.
    graph: instances::Allocation,
    /// Retained stack allocation for the initial context.
    kernelFrames: frames::Run,
    /// Initial context's mapped stack bounds.
    kernelStack: range::Range,
}

/// Allocate private storage and return all earlier allocations on failure.
unsafe fn prepare(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
unsafe fn allocate(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
    let pending = try frames::reserve(pool, (@sizeOf(Memory) + 4095) / 4096);
    let allocation = frames::commit(pending);
    let graph = try instances::create(packages, pool, image) catch err {
    let graph = try instances::reserve(packages, pool, image) catch err {
        try! frames::release(pool, allocation); throw err;
    };
    let pendingStack = try frames::reserve(pool, KERNEL_STACK_PAGES) catch err {
        try! frames::release(pool, graph.frames);
        instances::cancel(pool, graph);
        try! frames::release(pool, allocation); throw err;
    };
    let kernelFrames = frames::commit(pendingStack);
    let kernelStack = try! frames::extent(pool, kernelFrames);
    let extent = try! frames::extent(pool, allocation);
    zero(extent.start, allocation.count);
    zero(kernelStack.start, kernelFrames.count);
    return Prepared { allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack };
    match graph {
        case instances::Reservation::Held(graph) => return Prepared {
            allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack,
        },
    }
}

/// Return storage that has never been exposed to a published domain.
fn discard(pool: &mut frames::Pool, prepared: Prepared) {
    try! frames::release(pool, prepared.kernelFrames);
    try! frames::release(pool, prepared.graph.frames);
    try! frames::release(pool, prepared.allocation);
}

/// Create an unscheduled domain with Events and one initial integer context.
/// The caller serializes publication and keeps the authorizing domain alive.
export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
/// Unpublished domain metadata retained during private initialization.
export record Creation: Copy {
    /// Calling domain kept live until publication or cancellation.
    owner: abi::Ref,
    /// Reserved capability slot in the calling domain.
    handle: abi::Ref,
    /// Reserved domain slot.
    domain: abi::Ref,
    /// Reserved initial-context slot.
    context: abi::Ref,
    /// Management and inherited authority selected at admission.
    rights: abi::Rights,
    /// Resident root package.
    image: abi::Ref,
    /// Validated initial entry address.
    entry: u64,
    /// Private allocations retained until the domain becomes observable.
    storage: Prepared,
}

/// Domain creation resources that must be published or cancelled.
export union Reservation: Once {
    /// One private domain and its reserved metadata capacity.
    Held(Creation),
}

/// Reserve domain capacity under metadata serialization while the caller stays live.
export unsafe fn reserve(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> Reservation throws (abi::Error)
{
    let permit = try capability::authority(table, authority, abi::Rights(abi::CREATE));
    if not backing::domainLive(backings, table.owner) or not backing::domainLive(backings, permit.object) {
        throw abi::Error::BadHandle;
    }

    };
    let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch err {
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let mut prepared = try prepare(&mut backings.pool, packages, root) catch err {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let object = slots::reference(&domainSlot);
    try backing::registerDomain(backings, object) catch err {
        discard(&mut backings.pool, prepared);
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    capability::initialize(&mut prepared.memory.table, object);
    try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
        try! backing::endDomain(backings, object);
        discard(&mut backings.pool, prepared);
    let initialSlot = slots::reference(&contextSlot);
    set store.contexts[initialSlot.index].userStack = range::Range { start: 0, end: 0 };
    let prepared = try allocate(&mut backings.pool, packages, root) catch err {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
        kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ | abi::WRITE),
    });
    let initial = slots::reference(&contextSlot);
    // MPIE enables interrupts after the first return into U-mode.
    let mut frame = trap::Frame { registers: [0; 32], pc: entry, status: 0x80, cause: 0, value: 0 };
    set frame.registers[1] = returnAddress();
    set frame.registers[3] = prepared.graph.table.ptr as u64;
    set store.contexts[initial.index] = Context {
        owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
        userStack: range::Range { start: 0, end: 0 }, frame,
    };
    set store.records[object.index] = Domain {
        state: Lifecycle::Pending, creator: table.owner, parent: table.owner, image: root, initial,
        allocation: prepared.allocation, memory: prepared.memory, graph: prepared.graph, events: queue,
    match handleSlot {
        case slots::Reservation::Held(handle) => match domainSlot {
            case slots::Reservation::Held(domain) => match contextSlot {
                case slots::Reservation::Held(context) => return Reservation::Held(Creation {
                    owner: table.owner, handle, domain, context,
                    rights: abi::Rights(MANAGEMENT_RIGHTS | (*permit.rights & (abi::CREATE | abi::ALLOCATE))),
                    image: root, entry, storage: prepared,
                }),
            },
        },
    }
}

/// Initialize private storage from resident packages while metadata can change.
export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
    match reservation {
        case Reservation::Held(creation) => {
            let mut storage = creation.storage;
            zero(storage.memory as u64, storage.allocation.count);
            zero(storage.kernelStack.start, storage.kernelFrames.count);
            try instances::fill(packages, &storage.graph.graph, storage.graph.base);
            capability::initialize(&mut storage.memory.table, creation.domain);
        },
    }
}

/// Check the unpublished capacity retained by one creation transaction.
fn require(store: &Store, table: &capability::Table, creation: &Creation) {
    assert table.owner == creation.owner;
    assert slots::matches(&table.slots[..], creation.handle, slots::State::Reserved);
    assert slots::matches(&store.slots[..], creation.domain, slots::State::Reserved);
    assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
}

/// Return private domain storage and reserved slots under metadata serialization.
export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            discard(&mut backings.pool, creation.storage);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
        },
    }
}

/// Publish an initialized domain and its Events under metadata serialization.
export unsafe fn publish(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation)
    -> abi::Handle throws (abi::Error)
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            if not backing::domainLive(backings, creation.owner) {
                cancel(store, backings, table, Reservation::Held(creation));
                throw abi::Error::BadHandle;
            }
            let object = creation.domain;
            let initial = creation.context;
            let mut prepared = creation.storage;
            try backing::registerDomain(backings, object) catch err {
                cancel(store, backings, table, Reservation::Held(creation)); throw err;
            };
            try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
                try! backing::endDomain(backings, object);
                cancel(store, backings, table, Reservation::Held(creation)); throw err;
            };
            let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
                kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ | abi::WRITE),
            });
            let graph = instances::commit(instances::Reservation::Held(prepared.graph));
            // MPIE enables interrupts after the first return into U-mode.
            let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
            set frame.registers[1] = returnAddress();
            set frame.registers[3] = graph.table.ptr as u64;
            set store.contexts[initial.index] = Context {
                owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
            };
            let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
            let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
            return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
                kind: abi::Kind::Domain, object: domain, rights: creation.rights,
            });
        },
    }
}

/// Create a private domain while the caller has exclusive access to metadata.
export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(store, backings, packages, table, authority, image);
    try prepare(packages, &pending) catch err {
        cancel(store, backings, table, pending); throw err;
    };
    let context = try! slots::commit(&mut store.contextSlots[..], contextSlot);
    let domain = try! slots::commit(&mut store.slots[..], domainSlot);
    return capability::publish(table, handleSlot, capability::Entry {
        kind: abi::Kind::Domain, object: domain,
        rights: abi::Rights(MANAGEMENT_RIGHTS | (*permit.rights & (abi::CREATE | abi::ALLOCATE))),
    });
    return try publish(store, backings, table, pending);
}

kernel/kernel/instances.rad +56 -9

    /// Package bases indexed by resident slot; absent packages have zero bases.
    table: *unsafe [u64],
}

/// Bounded traversal and conservative storage requirement for a package graph.
record Graph: Copy {
export record Graph: Copy {
    /// Reachable package references, each present once.
    packages: [abi::Ref; limits::PACKAGES],
    /// Number of initialized package references.
    count: u32,
    /// Bytes for the base table, private state, and worst-case alignment padding.
    bytes: u64,
}

/// Private storage and the resident graph selected at reservation time.
export record Allocation: Copy {
    /// Reachable immutable packages and their storage bounds.
    graph: Graph,
    /// Physical frames reserved for this instance.
    frames: frames::Run,
    /// Validated physical base for private initialization.
    base: u64,
}

/// Graph storage that must be committed or returned to its frame pool.
export union Reservation: Once {
    /// An unpublished private instance.
    Held(Allocation),
}

/// Map an allocated physical extent to typed storage at the call site.
fn memory(address: u64) -> *unsafe mut opaque;

/// Collect the reachable immutable packages with a bounded queue.
unsafe fn graph(store: &registry::Store, root: abi::Ref) -> Graph throws (abi::Error) {

    }
    return result;
}

/// Assign every base before initializing templates and their private pointers.
unsafe fn initialize(store: &registry::Store, graph: &Graph, base: u64) -> *unsafe [u64] throws (abi::Error) {
/// The caller owns a writable extent of at least graph.bytes bytes at base.
export unsafe fn fill(store: &registry::Store, graph: &Graph, base: u64) throws (abi::Error) {
    let table = @sliceOf(memory(base) as *unsafe mut u64, limits::PACKAGES);
    for i in 0..table.len {
        set table[i] = 0;
    }
    let mut next = base + limits::PACKAGES as u64 * 8;

        let bytes = @sliceOf(memory(table[object.index]) as *unsafe mut u8, package.memory);
        try shared::instantiate(&package, &table[..], bytes) catch {
            throw abi::Error::VerifyFailed;
        };
    }
    return &table[..];
}

/// Allocate and initialize one private graph without copying resident code.
/// The caller serializes frame reservations and keeps the registry immutable.
export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
/// Reserve one graph under metadata serialization while its packages stay resident.
export unsafe fn reserve(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Reservation throws (abi::Error) {
    let layout = try graph(store, root);
    let count = ((layout.bytes + limits::FRAME_SIZE - 1) / limits::FRAME_SIZE) as u32;
    let pending = try frames::reserve(pool, count);
    let run = frames::commit(pending);
    let extent = try! frames::extent(pool, run);
    let table = try initialize(store, &layout, extent.start) catch err {
        try! frames::release(pool, run); throw err;
    return Reservation::Held(Allocation { graph: layout, frames: run, base: extent.start });
}

/// Initialize private state from the reservation's immutable resident packages.
export unsafe fn initialize(store: &registry::Store, reservation: &Reservation) throws (abi::Error) {
    match reservation {
        case Reservation::Held(allocation) => try fill(store, &allocation.graph, allocation.base),
    }
}

/// Consume initialized private storage for publication in a domain.
export unsafe fn commit(reservation: Reservation) -> Instance {
    match reservation {
        case Reservation::Held(allocation) => return Instance {
            frames: allocation.frames,
            table: @sliceOf(memory(allocation.base) as *unsafe u64, limits::PACKAGES),
        },
    }
}

/// Return unpublished graph storage under frame-pool serialization.
export fn cancel(pool: &mut frames::Pool, reservation: Reservation) {
    match reservation {
        case Reservation::Held(allocation) => {
            try! frames::release(pool, allocation.frames);
        },
    }
}

/// Allocate and initialize a private graph while the caller has exclusive metadata access.
export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
    let pending = try reserve(store, pool, root);
    try initialize(store, &pending) catch err {
        cancel(pool, pending); throw err;
    };
    return Instance { frames: run, table };
    return commit(pending);
}

kernel/kernel/loader.rad +201 -48

export union Lease: Once {
    /// Private temporary frames owned by the active load.
    Held(frames::Run),
}

/// Private input capacity retained throughout one load.
export record Input: Copy {
    /// Domain kept live while the source and reserved handle are in use.
    owner: abi::Ref,
    /// Reserved Image handle slot in the caller's table.
    handle: abi::Ref,
    /// Private workspace frames.
    frames: frames::Run,
    /// Physical workspace base retained by the lease.
    work: u64,
    /// Validated physical source address.
    address: u64,
    /// Number of source bytes to snapshot.
    length: u32,
}

/// Input capacity that must be cancelled or completed under metadata serialization.
export union Reservation: Once {
    /// One admitted load and its private compiler storage.
    Held(Input),
}

/// Global runtime-loader ownership.
export unsafe static STATE: State = State { busy: false, owner: abi::Ref { index: 0, generation: 0 },
    live: [false; limits::PACKAGES], resident: undefined };

/// Map a validated physical workspace extent to its compiler storage.

        },
        else => return abi::Error::VerifyFailed,
    }
}

/// Used page counts for unpublished native output.
record OutputSize: Copy {
/// Used native extents and validated metadata ready for publication.
export record Compiled: Copy {
    /// Pages occupied by generated code.
    code: u32,
    /// Pages occupied by the packed native catalog.
    metadata: u32,
    /// Immutable descriptor and resolved dependencies for the reserved package.
    admission: registry::Admission,
}

/// Compile and pack one candidate into caller-owned physical output extents.
unsafe fn generate(work: *mut Workspace, input: &binary::Package, packages: &registry::Store,
    slot: u32, code: u64, metadata: u64, length: u32) -> OutputSize throws (abi::Error)
    object: abi::Ref, code: u64, metadata: u64, length: u32) -> Compiled throws (abi::Error)
{
    let count = try registry::imports(packages, &input.dependencies[..], &mut work.imports[..]);
    let mut arena = alloc::new(&mut work.code[..]);
    let mut scratch = alloc::new(&mut work.scratch[..]);
    let package = try shared::compile(&*input, slot, code, &work.imports[..count],
    let package = try shared::compile(&*input, object.index, code, &work.imports[..count],
        shared::Storage {
            data: &mut work.data[..], symbols: &mut work.symbols[..], exports: &mut work.exports[..],
            template: &mut work.template[..], relocations: &mut work.relocations[..],
        }, &mut arena, &mut scratch) catch error {
            throw backend(error);

        };
    let mut codePages = (codeBytes + 4095) / 4096;
    if codePages == 0 {
        set codePages = 1;
    }
    return OutputSize { code: codePages, metadata: (used + 4095) / 4096 };
    let native = descriptor(metadata);
    let admission = try registry::admit(packages, object, native.source, native.package);
    return Compiled { code: codePages, metadata: (used + 4095) / 4096, admission };
}

/// Return unused output frames while preserving at least one frame for each extent.
fn trim(pool: &mut frames::Pool, run: frames::Run, count: u32) -> frames::Run {
    assert count > 0 and count <= run.count;

        try! frames::release(pool, frames::Run { first: run.first + count, count: run.count - count });
    }
    return frames::Run { first: run.first, count };
}

/// Snapshot, decode, compile, and publish a package while its caller remains live.
unsafe fn produce(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
    work: *mut Workspace, address: u64, length: u32) -> abi::Ref throws (abi::Error)
{
    try! mem::copy(&mut work.input[..length], @sliceOf(memory(address), length));
    let mut decoder = alloc::new(&mut work.decoded[..]);
    let input = try program::decode(&work.input[..length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
        if err == binary::Error::Storage {
            throw abi::Error::Exhausted;
        }
        throw abi::Error::VerifyFailed;
    };
    let existing = try registry::identify(packages, &input.name[..], &work.input[..length]);
    if let object = existing {
        return object;
/// Snapshot and decode trusted input into the held private workspace.
/// The decoded package stays valid until this reservation is completed or cancelled.
export unsafe fn decode(reservation: &Reservation) -> binary::Package throws (abi::Error) {
    match reservation {
        case Reservation::Held(input) => {
            let mut work = workspace(input.work);
            try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
            let mut decoder = alloc::new(&mut work.decoded[..]);
            return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
                if err == binary::Error::Storage {
                    throw abi::Error::Exhausted;
                }
                throw abi::Error::VerifyFailed;
            };
        },
    }
}

/// Reserved native output and its immutable physical extents.
export record Output: Copy {
    /// Reserved package slot selected for generated state references.
    object: abi::Ref,
    /// Private executable frames.
    code: frames::Run,
    /// Private source and descriptor frames.
    metadata: frames::Run,
    /// Validated physical executable base.
    codeBase: u64,
    /// Validated physical descriptor base.
    metadataBase: u64,
}

/// Native output that must be published or cancelled under serialization.
export union OutputReservation: Once {
    /// One unpublished package and its private output frames.
    Held(Output),
}

/// Reserve a package slot and native output under metadata serialization.
export fn reserveOutput(pool: &mut frames::Pool, packages: &mut registry::Store) -> OutputReservation throws (abi::Error) {
    let slot = try registry::reserve(packages);
    let object = slots::reference(&slot);
    let codeReservation = try frames::reserve(pool, CODE_BYTES / 4096) catch err {
        registry::cancel(packages, slot);
        throw err;

        try! frames::release(pool, code); registry::cancel(packages, slot); throw err;
    };
    let metadata = frames::commit(metadataReservation);
    let codeRange = try! frames::extent(pool, code);
    let metadataRange = try! frames::extent(pool, metadata);
    let sizes = try generate(work, &input, packages, object.index, codeRange.start, metadataRange.start, length) catch err {
        try! frames::release(pool, metadata); try! frames::release(pool, code);
        registry::cancel(packages, slot); throw err;
    };
    let retainedCode = trim(pool, code, sizes.code);
    let retainedMetadata = trim(pool, metadata, sizes.metadata);
    let native = descriptor(metadataRange.start);
    sync::syncInstructions();
    let result = try registry::publish(packages, slot, native.source, native.package) catch err {
        try! frames::release(pool, retainedMetadata); try! frames::release(pool, retainedCode); throw err;
    };
    assert result == object;
    set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
    set state.live[result.index] = true;
    return result;
    match slot {
        case slots::Reservation::Held(object) => return OutputReservation::Held(Output {
            object, code, metadata, codeBase: codeRange.start, metadataBase: metadataRange.start,
        }),
    }
}

/// Load trusted RIL and install an Image capability after complete native publication.
/// The caller keeps its domain alive and serializes shared metadata transactions.
export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
    table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
/// Compile a decoded package into reserved private output using resident dependencies.
export unsafe fn compile(packages: &registry::Store, reservation: &Reservation, input: &binary::Package, output: &OutputReservation)
    -> Compiled throws (abi::Error)
{
    match reservation {
        case Reservation::Held(source) => match output {
            case OutputReservation::Held(target) => {
                let work = workspace(source.work);
                return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
            },
        },
    }
}

/// Return unpublished output frames and its package slot under serialization.
export fn cancelOutput(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation) {
    match reservation {
        case OutputReservation::Held(output) => {
            assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
            try! frames::release(pool, output.metadata);
            try! frames::release(pool, output.code);
            registry::cancel(packages, slots::Reservation::Held(output.object));
        },
    }
}

/// Publish completed native output and retain its used frames under serialization.
export fn publishOutput(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
    reservation: OutputReservation, sizes: Compiled) -> abi::Ref
{
    match reservation {
        case OutputReservation::Held(output) => {
            assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
            let retainedCode = trim(pool, output.code, sizes.code);
            let retainedMetadata = trim(pool, output.metadata, sizes.metadata);
            sync::syncInstructions();
            let result = registry::commit(packages, slots::Reservation::Held(output.object), sizes.admission);
            assert result == output.object;
            set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
            set state.live[result.index] = true;
            return result;
        },
    }
}

/// Compare the snapshot with immutable resident content while the workspace is held.
export unsafe fn identify(packages: &registry::Store, reservation: &Reservation, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
    match reservation {
        case Reservation::Held(source) => return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]),
    }
}

/// Reserve an Image handle and exclusive workspace under metadata serialization.
export unsafe fn reserve(state: &mut State, store: &mut pages::Store,
    table: &mut capability::Table, request: Request) -> Reservation throws (abi::Error)
{
    let address = try source(store, table, request);
    let handle = try slots::reserve(&mut table.slots[..]);
    let lease = try acquire(state, &mut store.backings.pool, table.owner) catch err {
        try! slots::cancel(&mut table.slots[..], handle); throw err;
    };
    let work = storage(&store.backings.pool, &lease);
    let object = try produce(state, &mut store.backings.pool, packages, work, address, request.length as u32) catch err {
        release(state, &mut store.backings.pool, lease);
        try! slots::cancel(&mut table.slots[..], handle); throw err;
    let work = storage(&store.backings.pool, &lease) as u64;
    match handle {
        case slots::Reservation::Held(handle) => match lease {
            case Lease::Held(frames) => return Reservation::Held(Input {
                owner: table.owner, handle, frames, work, address, length: request.length as u32,
            }),
        },
    }
}

/// Check workspace ownership and the caller's unpublished handle slot.
fn require(state: &State, table: &capability::Table, input: &Input) {
    assert state.busy and state.owner == input.owner and table.owner == input.owner;
    assert slots::matches(&table.slots[..], input.handle, slots::State::Reserved);
}

/// Return the workspace and reserved handle under metadata serialization.
export fn cancel(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation) {
    match reservation {
        case Reservation::Held(input) => {
            require(state, table, &input);
            release(state, &mut store.backings.pool, Lease::Held(input.frames));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(input.handle));
        },
    }
}

/// Install a resident package handle and return the workspace under serialization.
export fn finish(state: &mut State, store: &mut pages::Store, table: &mut capability::Table,
    reservation: Reservation, object: abi::Ref) -> abi::Handle throws (abi::Error)
{
    match reservation {
        case Reservation::Held(input) => {
            require(state, table, &input);
            if not backing::domainLive(&store.backings, input.owner) {
                cancel(state, store, table, Reservation::Held(input)); throw abi::Error::BadHandle;
            }
            let result = capability::publish(table, slots::Reservation::Held(input.handle), capability::Entry {
                kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
            });
            release(state, &mut store.backings.pool, Lease::Held(input.frames));
            return result;
        },
    }
}

/// Load trusted RIL while the caller has exclusive access to shared metadata.
export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
    table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(state, store, table, request);
    let input = try decode(&pending) catch err {
        cancel(state, store, table, pending);
        throw err;
    };
    let existing = try identify(packages, &pending, &input) catch err {
        cancel(state, store, table, pending);
        throw err;
    };
    if let object = existing {
        return try finish(state, store, table, pending, object);
    }
    let output = try reserveOutput(&mut store.backings.pool, packages) catch err {
        cancel(state, store, table, pending);
        throw err;
    };
    let sizes = try compile(packages, &pending, &input, &output) catch err {
        cancelOutput(&mut store.backings.pool, packages, output);
        cancel(state, store, table, pending); throw err;
    };
    let result = capability::publish(table, handle, capability::Entry {
        kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
    });
    release(state, &mut store.backings.pool, lease);
    return result;
    let object = publishOutput(state, &mut store.backings.pool, packages, output, sizes);
    return try finish(state, store, table, pending, object);
}

kernel/kernel/pages.rad +93 -16

        }
    }
    return true;
}

/// Allocate and zero frames before publishing a page and its capability.
/// Private frames and unpublished slots held while memory is cleared.
export record Allocation: Copy {
    /// Calling domain whose table contains the reserved handle.
    owner: abi::Ref,
    /// Unpublished capability slot.
    handle: abi::Ref,
    /// Unpublished page-object slot.
    page: abi::Ref,
    /// Unpublished backing-allocation slot.
    backing: abi::Ref,
    /// Frames excluded from allocation until publication or cancellation.
    run: frames::Run,
    /// Validated physical base for private initialization.
    base: u64,
}

/// Allocation capacity that must be published or cancelled under serialization.
export union Reservation: Once {
    /// Private memory and its three reserved metadata slots.
    Held(Allocation),
}

/// Reserve page capacity under metadata serialization for private initialization.
/// Allocation authority must name the calling domain. Failure cancels reservations.
export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> abi::Handle throws (abi::Error)
export fn reserve(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> Reservation throws (abi::Error)
{
    if count == 0 {
        throw abi::Error::InvalidArg;
    }
    if count > limits::FRAMES as u64 {

    let owner = table.owner;
    let permission = try capability::authority(table, authority, abi::Rights(abi::ALLOCATE));
    if permission.object <> owner {
        throw abi::Error::Denied;
    }
    if not backing::domainLive(&store.backings, owner) {
    let backings = &mut store.backings;
    if not backing::domainLive(backings, owner) {
        throw abi::Error::BadHandle;
    }
    let handleSlot = try slots::reserve(&mut table.slots[..]);
    let pageSlot = try slots::reserve(&mut store.slots[..]) catch err {
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let backingSlot = try slots::reserve(&mut store.backings.slots[..]) catch err {
    let backingSlot = try slots::reserve(&mut backings.slots[..]) catch err {
        try! slots::cancel(&mut store.slots[..], pageSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let memory = try frames::reserve(&mut store.backings.pool, count as u32) catch err {
        try! slots::cancel(&mut store.backings.slots[..], backingSlot);
    let memory = try frames::reserve(&mut backings.pool, count as u32) catch err {
        try! slots::cancel(&mut backings.slots[..], backingSlot);
        try! slots::cancel(&mut store.slots[..], pageSlot);
        try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
    };
    let run = frames::commit(memory);
    let extent = try! frames::extent(&store.backings.pool, run);
    zero(extent.start, run.count);
    let allocation = backing::publish(&mut store.backings, backingSlot, run, owner);
    let object = slots::reference(&pageSlot);
    set store.records[object.index] = Page { backing: allocation, base: extent.start, count: run.count, origin: owner, handles: 1 };
    let page = try! slots::commit(&mut store.slots[..], pageSlot);
    return capability::publish(table, handleSlot, capability::Entry {
        kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
    });
    let extent = try! frames::extent(&backings.pool, run);
    match handleSlot {
        case slots::Reservation::Held(handle) => match pageSlot {
            case slots::Reservation::Held(page) => match backingSlot {
                case slots::Reservation::Held(backing) => return Reservation::Held(Allocation {
                    owner, handle, page, backing, run, base: extent.start,
                }),
            },
        },
    }
}

/// Clear private frames while shared metadata can be used by other harts.
export fn clear(reservation: &Reservation) {
    match reservation { case Reservation::Held(allocation) => zero(allocation.base, allocation.run.count), }
}

/// Check reserved generations before publishing or cancelling an allocation.
fn require(store: &Store, table: &capability::Table, allocation: &Allocation) {
    assert table.owner == allocation.owner;
    assert slots::matches(&table.slots[..], allocation.handle, slots::State::Reserved);
    assert slots::matches(&store.slots[..], allocation.page, slots::State::Reserved);
    assert slots::matches(&store.backings.slots[..], allocation.backing, slots::State::Reserved);
}

/// Publish cleared memory under metadata serialization while its owner stays live.
export unsafe fn publish(store: &mut Store, table: &mut capability::Table, reservation: Reservation) -> abi::Handle {
    match reservation {
        case Reservation::Held(allocation) => {
            require(store, table, &allocation);
            assert backing::domainLive(&store.backings, allocation.owner);
            let backing = backing::publish(&mut store.backings, slots::Reservation::Held(allocation.backing), allocation.run, allocation.owner);
            set store.records[allocation.page.index] = Page {
                backing, base: allocation.base, count: allocation.run.count, origin: allocation.owner, handles: 1,
            };
            let page = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(allocation.page));
            return capability::publish(table, slots::Reservation::Held(allocation.handle), capability::Entry {
                kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
            });
        },
    }
}

/// Return private allocation capacity under metadata serialization.
export fn cancel(store: &mut Store, table: &mut capability::Table, reservation: Reservation) {
    match reservation {
        case Reservation::Held(allocation) => {
            require(store, table, &allocation);
            try! frames::release(&mut store.backings.pool, allocation.run);
            try! slots::cancel(&mut store.backings.slots[..], slots::Reservation::Held(allocation.backing));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(allocation.page));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
        },
    }
}

/// Allocate zeroed pages while the caller has exclusive access to metadata.
export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(store, table, authority, count);
    clear(&pending);
    return publish(store, table, pending);
}

/// Split an exclusive page into two segments with the same backing and rights.
export fn split(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, leftCount: u64)
    -> abi::Handle throws (abi::Error)

kernel/kernel/registry.rad +43 -12

/// Return unpublished capacity after a failed load.
export fn cancel(store: &mut Store, reservation: slots::Reservation) {
    try! slots::cancel(&mut store.slots[..], reservation);
}

/// Validate dependencies without changing registry payloads.
unsafe fn validate(store: &Store, object: abi::Ref, package: &shared::Package) throws (abi::Error) {
/// Validated immutable package metadata ready for bounded publication.
export record Admission: Copy {
    /// Reserved package generation to receive the descriptor.
    object: abi::Ref,
    /// Persistent source and native descriptor.
    entry: catalog::Entry,
    /// Resolved resident dependencies in their declared order.
    dependencies: [abi::Ref; limits::PACKAGES],
    /// Number of initialized dependency references.
    count: u32,
}

/// Resolve admission against immutable packages while holding sole publisher ownership.
/// The caller keeps that ownership and the descriptor storage through commit or cancellation.
export unsafe fn admit(store: &Store, object: abi::Ref, source: *[u8], package: shared::Package) -> Admission throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Reserved) {
        throw abi::Error::BadHandle;
    }
    if try identify(store, &package.name[..], &source[..]) <> nil {
        throw abi::Error::Busy;
    }
    if package.name.len == 0 or package.slot <> object.index or package.dependencies.len > limits::PACKAGES {
        throw abi::Error::VerifyFailed;
    }
    let mut result = Admission { object, entry: catalog::Entry { source, package }, dependencies: undefined, count: package.dependencies.len };
    for dependency, i in package.dependencies {
        let target = find(store, &dependency[..]) else {
            throw abi::Error::VerifyFailed;
        };
        for prior in &package.dependencies[..i] {
            if mem::eq(prior, dependency) {
        for prior in &result.dependencies[..i] {
            if prior == target {
                throw abi::Error::VerifyFailed;
            }
        }
        set result.dependencies[i] = target;
    }
    return result;
}

/// Publish admitted metadata under serialization using only bounded reference copies.
export fn commit(store: &mut Store, reservation: slots::Reservation, admission: Admission) -> abi::Ref {
    let object = slots::reference(&reservation);
    assert object == admission.object and slots::matches(&store.slots[..], object, slots::State::Reserved);
    assert admission.count <= limits::PACKAGES;
    for i in 0..admission.count {
        let target = admission.dependencies[i];
        assert slots::matches(&store.slots[..], target, slots::State::Live);
        set store.dependencies[object.index][i] = target;
    }
    set store.counts[object.index] = admission.count;
    set store.entries[object.index] = admission.entry;
    return try! slots::commit(&mut store.slots[..], reservation);
}

/// Publish persistent compiler output after successful admission checks.
/// All descriptor pointers and source bytes must remain immutable and resident.
/// Failure cancels the reservation; duplicate content returns its existing slot.

    if let object = existing {
        cancel(store, reservation);
        return object;
    }
    let object = slots::reference(&reservation);
    try validate(store, object, &package) catch err {
    let admission = try admit(store, object, source, package) catch err {
        cancel(store, reservation);
        throw err;
    };
    for dependency, i in package.dependencies {
        let target = find(store, &dependency[..]) else panic "resident dependency";
        set store.dependencies[object.index][i] = target;
    }
    set store.counts[object.index] = package.dependencies.len;
    set store.entries[object.index] = catalog::Entry { source, package };
    return try! slots::commit(&mut store.slots[..], reservation);
    return commit(store, reservation, admission);
}

/// Read resident metadata through a generation-bearing package reference.
export fn get(store: &Store, object: abi::Ref) -> shared::Package throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) {

kernel/kernel/sync.rad +63 -0

1	1		//! RV64 synchronization and ordered device access.
2	2
	3	+	/// Next metadata ticket, modulo 2^32. Each hart can hold at most one ticket.
	4	+	static NEXT: u32 = 0;
	5	+	/// Metadata ticket permitted to enter the critical section.
	6	+	static SERVING: u32 = 0;
	7	+	/// Largest observed instruction count inside the metadata lock.
	8	+	static MAXIMUM: u64 = 0;
	9	+
	10	+	/// Interrupt state and start counter retained by one metadata transaction.
	11	+	export record Section: Copy {
	12	+	/// Original machine interrupt-enable bit.
	13	+	interrupts: u64,
	14	+	/// Retired instruction count after lock acquisition.
	15	+	start: u64,
	16	+	/// Ticket held by the acquiring hart until release.
	17	+	ticket: u32,
	18	+	}
	19	+
	20	+	/// Lock ownership that must be released on its acquiring hart.
	21	+	export union Guard: Once {
	22	+	/// One serialized metadata transaction with machine interrupts masked.
	23	+	Held(Section),
	24	+	}
	25	+
	26	+	/// Mask machine interrupts and return the original enable bit.
	27	+	export fn maskInterrupts() -> u64;
	28	+	/// Restore the supplied machine interrupt-enable bit.
	29	+	export fn restoreInterrupts(value: u64);
	30	+	/// Read this hart's retired instruction counter.
	31	+	fn retired() -> u64;
	32	+
	33	+	/// Acquire shared metadata in ticket order with local interrupts masked.
	34	+	/// Calls must not nest. At most one ticket per online hart can be outstanding.
	35	+	export fn enter() -> Guard {
	36	+	let interrupts = maskInterrupts();
	37	+	let ticket = nextTicket(&mut NEXT);
	38	+	while loadAcquire32(&SERVING) <> ticket {
	39	+	}
	40	+	let start = retired();
	41	+	return Guard::Held(Section { interrupts, start, ticket });
	42	+	}
	43	+
	44	+	/// Publish metadata and restore the acquiring hart's interrupt state.
	45	+	export fn leave(guard: Guard) {
	46	+	match guard {
	47	+	case Guard::Held(section) => {
	48	+	let work = retired() - section.start;
	49	+	if work > MAXIMUM {
	50	+	set MAXIMUM = work;
	51	+	}
	52	+	storeRelease32(&mut SERVING, ((section.ticket as u64 + 1) & 0xffffffff) as u32);
	53	+	restoreInterrupts(section.interrupts);
	54	+	},
	55	+	}
	56	+	}
	57	+
	58	+	/// Read the maximum measured metadata work under serialization.
	59	+	export fn maximum() -> u64 {
	60	+	let guard = enter();
	61	+	let value = MAXIMUM;
	62	+	leave(guard);
	63	+	return value;
	64	+	}
	65	+
3	66		/// Allocate one wrapping ticket from a naturally aligned u32 counter.
4	67		export fn nextTicket(counter: &mut u32) -> u32;
5	68		/// Read a naturally aligned shared word with acquire ordering.
6	69		export fn loadAcquire(value: &u64) -> u64;
7	70		/// Publish a naturally aligned shared word with release ordering.

kernel/kernel/sync.ras +24 -0

.export @kernel::sync::fetchAdd;
.export @kernel::sync::syncInstructions;
.export @kernel::sync::deviceFence;
.export @kernel::sync::read32;
.export @kernel::sync::write32;
.export @kernel::sync::maskInterrupts;
.export @kernel::sync::restoreInterrupts;
.export @kernel::sync::retired;

// Preserve the previous interrupt-enable bit while masking machine interrupts.
@kernel::sync::maskInterrupts
    csrr %t0 mstatus;
    andi %a0 %t0 8;
    andi %t0 %t0 -9;
    csrw mstatus %t0;
    ret;

// Restore interrupt delivery while preserving other machine status fields.
@kernel::sync::restoreInterrupts
    csrr %t0 mstatus;
    andi %t0 %t0 -9;
    or %t0 %t0 %a0;
    csrw mstatus %t0;
    ret;

// Count local retired instructions independently of other harts' progress.
@kernel::sync::retired
    csrr %a0 instret;
    ret;

// Allocate a wrapping ticket and return its zero-extended u32 value.
@kernel::sync::nextTicket
    li %t0 1;
    amoadd.w.aqrl %a0 %t0 (%a0);

kernel/kernel/tests/calls.rad +1 -1

/// An unknown user operation advances PC and returns only a negative a0 error.
@test unsafe fn trapReply() throws (testing::TestError) {
    let owner = initialize();
    let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
    set frame.registers[17] = 0xffffffffffffffff;
    calls::handle(owner, &mut frame, 0);
    calls::handle(owner, &mut frame, 0, calls::invoke);
    try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc);
    for i in 0..32 {
        if i == 10 or i == 17 {
            continue;
        }

kernel/kernel/tests/dispatch.rad +17 -0

    try testing::expect(ended.context == nil and ended.deadline == 0xffffffffffffffff);
    let other = try! dispatch::select(&BUDGETS, &DOMAINS, 1, 10);
    try testing::expect(other.context == nil and other.deadline == 0xffffffffffffffff);
}

/// A context becomes eligible only after its previous hart releases ownership.
@test unsafe fn ownership() throws (testing::TestError) {
    initialize();
    set DOMAINS.contexts[0].hart = 1;
    let future = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
    assert future.context == nil and future.deadline == 10;
    let occupied = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
    assert occupied.context == nil and occupied.deadline == 20;
    set DOMAINS.contexts[0].hart = nil;
    let free = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
    assert free.context == TABLE.owner and free.deadline == 20;
    set DOMAINS.contexts[0].hart = 0;
    let local = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
    assert local == free;
}

/// Pending domains and stale context generations cannot become runnable through a budget.
@test unsafe fn lifetimes() throws (testing::TestError) {
    initialize();
    set DOMAINS.records[0].state = domains::Lifecycle::Pending;
    let pending = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

        stack: range::Range { start: 0x2000, end: 0x4000 },
    };
    let mut anchor = trap::Hart {
        stackTop: 0x4000, stackBottom: 0x2000, kernelGp: 0x20000, handler,
        savedT0: 0, savedT1: 0, savedSp: 0,
        dispatchTop: 0x4000, dispatchFrame: undefined,
    };
    let first = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 10);
    try testing::expect(frame.pc == 0x40 and anchor.stackTop == 0xc000);
    try testing::expect(DOMAINS.contexts[0].hart == 0);
    set frame.pc = 0x88;

kernel/kernel/tests/domains.rad +76 -1

    try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
        try testing::expect(error == abi::Error::OutOfMemory); set exhausted = true;
    };
    try testing::expect(exhausted and DOMAINS.contextSlots[child.initial.index + 1].state == slots::State::Free);
    set PAGES.backings.pool.count = count;
    let context = try! domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    let reserved = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    let index = child.initial.index + 1;
    assert DOMAINS.contextSlots[index].state == slots::State::Reserved;
    let mut overlap = false;
    try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
        assert error == abi::Error::Busy; set overlap = true;
    };
    assert overlap;
    let reservedFrames = DOMAINS.contexts[index].kernelFrames;
    domains::contextCancel(&mut DOMAINS, &mut PAGES, reserved);
    assert DOMAINS.contextSlots[index].state == slots::State::Free;
    for i in reservedFrames.first..reservedFrames.first + reservedFrames.count {
        assert PAGES.backings.pool.free[i];
    }
    let pendingContext = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
    let stackBase = DOMAINS.contexts[index].kernelStack.start;
    let base = (&RAM[0]) as u64;
    let firstWord = ((stackBase - base) / 8) as u32;
    let lastWord = firstWord + domains::KERNEL_STACK_PAGES * 512 - 1;
    set RAM[firstWord] = 123; set RAM[lastWord] = 456;
    domains::contextClear(&pendingContext);
    assert RAM[firstWord] == 0 and RAM[lastWord] == 0;
    let context = domains::contextPublish(&mut DOMAINS, pendingContext);
    let created = try! domains::context(&DOMAINS, authority.object, context);
    let initial = DOMAINS.contexts[child.initial.index];
    try testing::expect(context <> child.initial and created.owner == authority.object);
    try testing::expect(created.frame.registers[3] == initial.frame.registers[3]);
    try testing::expect(created.kernelStack.start >= initial.kernelStack.end or created.kernelStack.end <= initial.kernelStack.start);

        assert (error == abi::Error::BadHandle); set rejected += 1;
    };
    assert (rejected == 3);
}

/// Domain reservations retain private capacity across another complete creation.
@test unsafe fn stagedCreation() throws (testing::TestError) {
    initialize(abi::CREATE);
    let pending = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    assert DOMAINS.slots[1].state == slots::State::Reserved;
    assert DOMAINS.contextSlots[0].state == slots::State::Reserved;
    assert TABLE.slots[2].state == slots::State::Reserved;
    assert BACKINGS.domains[1] == 0 and DOMAINS.events.queues[1].generation == 0;
    let other = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    try! domains::prepare(&PACKAGES, &pending);
    let first = try! domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, pending);
    let a = try! capability::get(&TABLE, first);
    let b = try! capability::get(&TABLE, other);
    let left = try! domains::get(&DOMAINS, a.object);
    let right = try! domains::get(&DOMAINS, b.object);
    assert a.object.index == 1 and b.object.index == 2;
    assert left.graph.table[0] <> right.graph.table[0];
    assert left.state == domains::Lifecycle::Pending and right.state == domains::Lifecycle::Pending;
    initialize(abi::CREATE);
    let cancelled = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    domains::cancel(&mut DOMAINS, &mut BACKINGS, &mut TABLE, cancelled);
    assert DOMAINS.slots[1].state == slots::State::Free;
    assert DOMAINS.contextSlots[0].state == slots::State::Free;
    assert TABLE.slots[2].state == slots::State::Free;
    for i in 0..BACKINGS.pool.count {
        assert BACKINGS.pool.free[i];
    }
    set PACKAGES.entries[0].package.relocations = &[shared::Relocation {
        offset: 0, count: 1, target: shared::DataRef { slot: 5, offset: 0 },
    }];
    let mut invalid = false;
    try domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE) catch error {
        assert error == abi::Error::VerifyFailed; set invalid = true;
    };
    assert invalid and DOMAINS.slots[1].state == slots::State::Free;
    for i in 0..BACKINGS.pool.count {
        assert BACKINGS.pool.free[i];
    }
    initialize(abi::CREATE);
    let abandoned = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    try! domains::prepare(&PACKAGES, &abandoned);
    try! backing::endDomain(&mut BACKINGS, TABLE.owner);
    let mut dead = false;
    try domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, abandoned) catch error {
        assert error == abi::Error::BadHandle; set dead = true;
    };
    assert dead and DOMAINS.slots[1].state == slots::State::Free;
    assert TABLE.slots[2].state == slots::State::Free;
    for i in 0..BACKINGS.pool.count {
        assert BACKINGS.pool.free[i];
    }
}

/// Creation installs only Events in the child and preserves selected authorizing rights.
@test unsafe fn pending() throws (testing::TestError) {
    initialize(abi::CREATE | abi::ALLOCATE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let authority = try! capability::get(&TABLE, handle);

kernel/kernel/tests/instances.rad +23 -0

fn word(address: u64) -> *mut u64 {
    let pointer = &RAM[0];
    return &mut RAM[((address - pointer as u64) / 8) as u32];
}

/// Private initialization retains its graph across other registry and frame transactions.
@test unsafe fn stagedGraph() throws (testing::TestError) {
    initialize();
    let root = register("root", &[], &[]);
    let pending = try! instances::reserve(&PACKAGES, &mut POOL, root);
    let later = register("later", &[], &[]);
    let other = try! instances::create(&PACKAGES, &mut POOL, later);
    set *word(other.table[later.index]) = 99;
    try! instances::initialize(&PACKAGES, &pending);
    let first = instances::commit(pending);
    assert first.table[root.index] <> 0 and first.table[later.index] == 0;
    assert *word(first.table[root.index]) == 7;
    assert *word(other.table[later.index]) == 99;
    assert first.frames.first + first.frames.count <= other.frames.first;
    try! frames::release(&mut POOL, first.frames);
    try! frames::release(&mut POOL, other.frames);
    let cancelled = try! instances::reserve(&PACKAGES, &mut POOL, root);
    instances::cancel(&mut POOL, cancelled);
    for i in 0..POOL.count {
        assert POOL.free[i];
    }
}

/// A diamond has one dependency instance per domain and independent relocated pointers.
@test unsafe fn diamond() throws (testing::TestError) {
    initialize();
    let base = register("base", &[], &[]);
    unsafe static relocation: [shared::Relocation; 1] = undefined;

kernel/kernel/tests/loader.rad +62 -0

        }
    }
    return count;
}

/// Reserved input stays private across allocation, source mutation, and contention.
@test unsafe fn stagedInput() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();
    let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
    assert LOADER.busy and TABLE.slots[2].state == slots::State::Reserved;
    let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
    let decoded = try! loader::decode(&pending);
    assert decoded.program.fns.len == 1 and decoded.name.len == 6;
    let replacement = input(99, false);
    assert decoded.program.fns[0].blocks[0].instrs[0] == il::Instr::Ret { val: il::Val::Imm(43) };
    let mut busy = false;
    try loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy;
    loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
    assert not LOADER.busy and TABLE.slots[2].state == slots::State::Free;
    assert free() + 1 == before;
    assert (try! pages::get(&PAGES, (try! capability::get(&TABLE, other)).object)).count == 1;
}

/// Compiled output stays private until publication and cancellation restores capacity.
@test unsafe fn stagedOutput() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();
    let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
    let decoded = try! loader::decode(&pending);
    let output = try! loader::reserveOutput(&mut PAGES.backings.pool, &mut PACKAGES);
    assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
    let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
    let used = try! loader::compile(&PACKAGES, &pending, &decoded, &output);
    assert used.code > 0 and used.metadata > 0;
    assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
    loader::cancelOutput(&mut PAGES.backings.pool, &mut PACKAGES, output);
    loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
    assert free() + 1 == before and not LOADER.busy;
    assert PACKAGES.slots[0].state == slots::State::Free;
    let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
    assert (try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0))).index == 0;
}

/// A caller whose backing lifetime ended cannot receive a completed Image handle.
@test unsafe fn completionLifetime() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
    let object = try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0));
    let before = free();
    let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
    try! backing::endDomain(&mut PAGES.backings, TABLE.owner);
    let mut dead = false;
    try loader::finish(&mut LOADER, &mut PAGES, &mut TABLE, pending, object) catch error {
        assert error == abi::Error::BadHandle; set dead = true;
    };
    assert dead and not LOADER.busy and free() == before;
    assert TABLE.slots[3].state == slots::State::Free;
    assert slots::matches(&PACKAGES.slots[..], object, slots::State::Live);
}

/// Resident source bytes survive mutation of the source page and workspace release.
@test unsafe fn residentSnapshot() throws (testing::TestError) {
    initialize();
    let request = input(43, false);
    let before = free();

kernel/kernel/tests/pages.rad +43 -0

/// Drop the test's handle and its object reference in the same serialized operation.
unsafe fn drop(handle: abi::Handle) {
    try! pages::drop(&mut STORE, &mut TABLE, handle);
}

/// Reserved frames stay private while another allocation changes shared metadata.
@test unsafe fn stagedAllocation() throws (testing::TestError) {
    initialize();
    let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 2);
    assert STORE.slots[0].state == slots::State::Reserved;
    assert TABLE.slots[1].state == slots::State::Reserved;
    assert not STORE.backings.pool.free[0] and not STORE.backings.pool.free[1];
    assert RAM[OFFSET] == 0xa5;
    let other = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 6);
    set RAM[OFFSET + 8192] = 0x7b;
    pages::clear(&pending);
    for i in OFFSET..OFFSET + 8192 {
        assert RAM[i] == 0;
    }
    assert RAM[OFFSET + 8192] == 0x7b;
    let handle = pages::publish(&mut STORE, &mut TABLE, pending);
    let entry = try! capability::get(&TABLE, handle);
    let page = try! pages::get(&STORE, entry.object);
    assert page.count == 2 and page.origin == TABLE.owner;
    drop(handle); drop(other);
    try! backing::endDomain(&mut STORE.backings, TABLE.owner);
    for i in 0..8 {
        assert STORE.backings.pool.free[i];
    }
}

/// Cancelling unpublished storage returns all capacity without exposing its bytes.
@test unsafe fn cancelAllocation() throws (testing::TestError) {
    initialize();
    let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 8);
    pages::cancel(&mut STORE, &mut TABLE, pending);
    assert STORE.slots[0].state == slots::State::Free;
    assert STORE.backings.slots[0].state == slots::State::Free;
    assert TABLE.slots[1].state == slots::State::Free;
    for i in 0..8 {
        assert STORE.backings.pool.free[i];
    }
    assert RAM[OFFSET] == 0xa5;
    let full = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
    let entry = try! capability::get(&TABLE, full);
    assert (try! pages::get(&STORE, entry.object)).count == 8;
}

/// Transfers keep both domains exposed without duplicating page handles.
@test unsafe fn transferExposure() throws (testing::TestError) {
    initialize();
    let receiver = abi::Ref { index: 1, generation: 1 };
    capability::initialize(&mut TARGET, receiver);

kernel/kernel/tests/registry.rad +24 -0

    let pending = try registry::reserve(&mut STORE);
    let slot = slots::reference(&pending);
    return try registry::publish(&mut STORE, pending, bytes, package(name, dependencies, slot.index));
}

/// Admission resolves dependencies before its reserved package becomes visible.
@test unsafe fn stagedAdmission() throws (testing::TestError) {
    registry::initialize(&mut STORE);
    let base = try! register("base", &[], "base bytes");
    let pending = try! registry::reserve(&mut STORE);
    let object = slots::reference(&pending);
    let admitted = try! registry::admit(&STORE, object, "child bytes", package("child", &["base"], object.index));
    assert STORE.slots[object.index].state == slots::State::Reserved;
    assert registry::find(&STORE, &"child"[..]) == nil;
    assert admitted.dependencies[0] == base;
    let published = registry::commit(&mut STORE, pending, admitted);
    assert published == object and STORE.counts[object.index] == 1;
    assert STORE.dependencies[object.index][0] == base;
    let rejected = try! registry::reserve(&mut STORE);
    let slot = slots::reference(&rejected);
    let mut duplicate = false;
    try registry::admit(&STORE, slot, "bad bytes", package("bad", &["base", "base"], slot.index)) catch error {
        assert error == abi::Error::VerifyFailed; set duplicate = true;
    };
    assert duplicate and STORE.slots[slot.index].state == slots::State::Reserved;
    registry::cancel(&mut STORE, rejected);
    assert STORE.slots[slot.index].state == slots::State::Free;
}

/// Identical bytes reuse one package; a name cannot identify different content.
@test unsafe fn identity() throws (testing::TestError) {
    registry::initialize(&mut STORE);
    let first = try! register("a", &[], "binary a");
    let second = try! register("a", &[], "binary a");

kernel/kernel/tests/trap.rad +1 -1

use kernel::trap;

/// The assembly frame includes x0-x31 and all four trap CSRs.
@test fn layout() throws (testing::TestError) {
    try testing::expect(@sizeOf(trap::Frame) == 288);
    try testing::expect(@sizeOf(trap::Hart) == 56);
    assert @sizeOf(trap::Hart) == 352;
    try testing::expect(trap::FRAME_SIZE % 16 == 0);
}

/// Interrupts retain their source identity; exceptions retain their fault class.
@test fn classification() throws (testing::TestError) {

kernel/kernel/trap.rad +5 -1

    savedT0: u64,
    /// Temporary preservation of x6 during entry.
    savedT1: u64,
    /// Interrupted stack pointer, before stack selection.
    savedSp: u64,
    /// Firmware stack top used while releasing and selecting context ownership.
    dispatchTop: u64,
    /// Hart-owned frame used after the interrupted context's stack is released.
    dispatchFrame: Frame,
}

/// Trap classes consumed by kernel dispatch.
export union Cause: Copy {
    /// Machine software interrupt.


/// Install trap entry for the current hart with machine interrupts disabled.
export fn install(hart: *mut Hart);
/// Restore a complete integer context through mret. The current mscratch must
/// name an installed Hart, and the frame must have machine interrupts disabled.
export fn resume(frame: &Frame) -> !;
export fn resume(frame: *Frame) -> !;

/// Disable machine interrupts and stop the current hart.
export fn halt() -> !;

test/dispatch/run +9 -5

#!/bin/sh
# Preempt and resume U-mode and M-mode work through production dispatch.
set -eu
emulator=${RAD_EMULATOR:-emulator}
fixture=${1:-dispatch}
harts=${2:-1}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp test/dispatch/spin.rad "$work/spin.rad"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"
cp test/dispatch/kernel/dispatchcheck.rad "$work/kernel/"
cp "test/$fixture/kernel/dispatchcheck.rad" "$work/kernel/"
length=$(wc -c < "$work/spin.ril")
printf '//! Binary user loop.\n/// Complete trusted input package.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/dispatchinput.rad"
od -An -v -tu1 "$work/spin.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/dispatch/machine.ras kernel/kernel/*.ras > "$work/dispatch.ras"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/dispatch.rv64"
printf 'dispatch: user calls, preemption, idle gap, and retained machine continuation passed\n'
for count in $harts; do
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
    printf '%s: %s-hart execution passed\n' "$fixture" "$count"
done

test/smp/kernel/dispatchcheck.rad added +175 -0

1	+	//! One domain executing separate user contexts on every online hart.
2	+	use std::mem;
3	+	use std::arch::rv64::shared;
4	+	use kernel::abi;
5	+	use kernel::slots;
6	+	use kernel::backing;
7	+	use kernel::pages;
8	+	use kernel::capability;
9	+	use kernel::registry;
10	+	use kernel::loader;
11	+	use kernel::domains;
12	+	use kernel::budgets;
13	+	use kernel::dispatch;
14	+	use kernel::boot;
15	+	use kernel::sync;
16	+	use kernel::dispatchinput;
17	+
18	+	/// Bootstrap resource authority.
19	+	unsafe static TABLE: capability::Table = undefined;
20	+	/// Release/acquire barrier after fixture publication.
21	+	static READY: u64 = 0;
22	+	/// Shared user domain generation.
23	+	unsafe static USER: abi::Ref = undefined;
24	+	/// Per-hart user execution contexts.
25	+	unsafe static CONTEXTS: [abi::Ref; 8] = undefined;
26	+	/// Mapped progress counters in the user's private state.
27	+	static COUNTERS: u64 = 0;
28	+	/// Validated physical memory mapping.
29	+	fn memory(address: u64) -> *mut u8;
30	+	/// Current kernel package-state table.
31	+	fn kernelGp() -> u64;
32	+	/// M-mode entry of the completion checker.
33	+	fn completion() -> u64;
34	+	/// Write one diagnostic byte to the fixture UART.
35	+	fn put(byte: u8);
36	+	/// Finish the machine test successfully.
37	+	fn finish();
38	+
39	+	/// Publish fixture state on the last initialized hart and wait on every hart.
40	+	export unsafe fn start(last: bool) {
41	+	if last {
42	+	setup();
43	+	sync::storeRelease(&mut READY, 1);
44	+	}
45	+	while sync::loadAcquire(&READY) == 0 {
46	+	}
47	+	}
48	+
49	+	/// Create one context per online hart before any dispatch can access the fixture.
50	+	unsafe fn setup() {
51	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
52	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
53	+	capability::initialize(&mut TABLE, owner);
54	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
55	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
56	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
57	+	});
58	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
59	+	let sourceEntry = try! capability::get(&TABLE, source);
60	+	let page = try! pages::get(&pages::STORE, sourceEntry.object);
61	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
62	+	let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
63	+	let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
64	+	loader::Request { authority, source, offset: 0, length: length as u64 });
65	+	let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
66	+	let cap = try! capability::get(&TABLE, handle);
67	+	set USER = cap.object;
68	+	let mut user = try! domains::get(&domains::STORE, USER);
69	+	let self = try! capability::install(&mut user.memory.table, capability::Entry {
70	+	kind: abi::Kind::Domain, object: USER, rights: abi::Rights(abi::ALLOCATE),
71	+	});
72	+	let target = try! registry::exported(&registry::STORE, user.image, &"spin::worker"[..]);
73	+	let case shared::Target::Function(entry) = target else panic "worker entry";
74	+	let counterTarget = try! registry::exported(&registry::STORE, user.image, &"spin::count"[..]);
75	+	let case shared::Target::Data(data) = counterTarget else panic "counter state";
76	+	set COUNTERS = user.graph.table[data.slot] + data.offset as u64;
77	+	for hart in 0..8 {
78	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 {
79	+	continue;
80	+	}
81	+	let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
82	+	let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
83	+	let permission = try! capability::get(&TABLE, stack);
84	+	let page = try! pages::get(&pages::STORE, permission.object);
85	+	let argument = @sliceOf(memory(page.base), 8);
86	+	for i in 0..8 {
87	+	set argument[i] = ((hart as u64) >> (i as u64 * 8)) as u8;
88	+	}
89	+	if hart == 0 {
90	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, page.base + 4096, page.base, 8);
91	+	set CONTEXTS[hart] = user.initial;
92	+	} else {
93	+	set CONTEXTS[hart] = try! domains::contextCreate(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
94	+	&TABLE, handle, abi::ContextStart { entry, stack: page.base + 4096, args: page.base, size: 8 });
95	+	}
96	+	}
97	+	let checkerHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
98	+	let checkerCap = try! capability::get(&TABLE, checkerHandle);
99	+	let checker = try! domains::get(&domains::STORE, checkerCap.object);
100	+	set domains::STORE.records[checkerCap.object.index].state = domains::Lifecycle::Active;
101	+	set domains::STORE.contexts[checker.initial.index].frame.pc = completion();
102	+	set domains::STORE.contexts[checker.initial.index].frame.status = 0x1880;
103	+	set domains::STORE.contexts[checker.initial.index].frame.registers[2] = domains::STORE.contexts[checker.initial.index].kernelStack.end;
104	+	set domains::STORE.contexts[checker.initial.index].frame.registers[3] = kernelGp();
105	+	let clock = dispatch::now(try! dispatch::timer(&boot::PLATFORM, 0));
106	+	let start = clock + 1000000;
107	+	let mut harts: u64 = 0;
108	+	for hart in 0..8 {
109	+	if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
110	+	set harts += 1;
111	+	}
112	+	}
113	+	let end = start + 10000000 * harts * harts;
114	+	for hart in 0..8 {
115	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 {
116	+	continue;
117	+	}
118	+	let budget = try! budgets::seed(&mut budgets::STORE, &mut TABLE, hart, start, 0xffffffffffffffff);
119	+	let final = try! budgets::split(&mut budgets::STORE, &mut TABLE, budget, end, clock);
120	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
121	+	budgets::Binding { budget, domain: handle, context: CONTEXTS[hart] }, clock);
122	+	if hart == 0 {
123	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
124	+	budgets::Binding { budget: final, domain: checkerHandle, context: checker.initial }, clock);
125	+	}
126	+	}
127	+	}
128	+
129	+	/// Check shared progress and context isolation after the user windows finish.
130	+	export unsafe fn verify() {
131	+	let guard = sync::enter();
132	+	let mut counts: [u64; 8] = [0; 8];
133	+	for hart in 0..8 {
134	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 {
135	+	continue;
136	+	}
137	+	let value = sync::loadAcquire(memory(COUNTERS + hart as u64 * 8) as *u64);
138	+	set counts[hart] = value;
139	+	let context = try! domains::context(&domains::STORE, USER, CONTEXTS[hart]);
140	+	assert context.owner == USER and (context.frame.status & 0x1800) == 0;
141	+	assert context.hart == nil or context.hart == hart;
142	+	}
143	+	sync::leave(guard);
144	+	for byte in "smp completed transactions:" {
145	+	put(byte);
146	+	}
147	+	for hart in 0..8 {
148	+	if (boot::PLATFORM.harts & (1 << hart)) == 0 {
149	+	continue;
150	+	}
151	+	put(32); number(counts[hart]);
152	+	}
153	+	put(10);
154	+	for hart in 0..8 {
155	+	if (boot::PLATFORM.harts & (1 << hart)) <> 0 {
156	+	assert counts[hart] == 16;
157	+	}
158	+	}
159	+	let maximum = sync::maximum();
160	+	assert maximum > 0 and maximum < 1000000;
161	+	for byte in "smp metadata instructions: 0x" {
162	+	put(byte);
163	+	}
164	+	number(maximum);
165	+	put(10);
166	+	finish();
167	+	}
168	+
169	+	/// Print one full-width hexadecimal diagnostic value.
170	+	fn number(value: u64) {
171	+	let digits = "0123456789abcdef";
172	+	for i in 0..16 {
173	+	put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]);
174	+	}
175	+	}

test/smp/machine.ras added +31 -0

1	+	//! Start all initialized harts after the fixture publishes its shared domains.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::start;
5	+	call @kernel::boot::run;
6	+	ebreak;
7	+	.export @kernel::dispatchcheck::memory;
8	+	.export @kernel::dispatchcheck::kernelGp;
9	+	.export @kernel::dispatchcheck::completion;
10	+	.export @kernel::dispatchcheck::finish;
11	+	.export @kernel::dispatchcheck::put;
12	+	@kernel::dispatchcheck::memory
13	+	ret;
14	+	@kernel::dispatchcheck::kernelGp
15	+	mv %a0 %gp;
16	+	ret;
17	+	@kernel::dispatchcheck::completion
18	+	la %a0 @complete;
19	+	ret;
20	+	@complete
21	+	call @kernel::dispatchcheck::verify;
22	+	ebreak;
23	+	@kernel::dispatchcheck::put
24	+	li %t0 0x10000000;
25	+	sb %a0 0(%t0);
26	+	ret;
27	+	@kernel::dispatchcheck::finish
28	+	li %t0 0x10001000;
29	+	li %t1 0x5555;
30	+	sw %t1 0(%t0);
31	+	ebreak;

test/smp/spin.rad added +34 -0

1	+	//! Concurrent user contexts with shared private state and separate call buffers.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Completed allocation transactions, indexed by the executing hart.
6	+	export static count: [u64; 8] = [0; 8];
7	+
8	+	/// Check hart identity and shared-table allocation through repeated direct calls.
9	+	fn run(hart: u64) {
10	+	let original = sys::currentContext();
11	+	assert original.hart == hart;
12	+	for i in 0..16 {
13	+	let page = try! sys::pageAllocate(abi::Handle(0), 1);
14	+	let info = sys::queryPage(page);
15	+	assert info.count == 1 and info.base <> 0;
16	+	try! sys::capabilityDrop(page);
17	+	let current = sys::currentContext();
18	+	assert current.context == original.context and current.hart == hart;
19	+	set count[hart as u32] += 1;
20	+	}
21	+	while true {
22	+	}
23	+	}
24	+
25	+	/// Start an additional context with its physical hart index in validated memory.
26	+	export fn worker(args: *u64, size: u64) {
27	+	assert size == 8;
28	+	run(*args);
29	+	}
30	+
31	+	/// The initial context owns hart zero's window.
32	+	@default fn main(env: *opaque) {
33	+	run(0);
34	+	}

test/sync/machine.ras +21 -0

//! Exercise code publication and contended metadata updates on each hart.
.constant EXPECTED HARTS * 100;
.text;
@entry
    li %t0 0x1888;
    csrw mstatus %t0;
    csrr %s7 mstatus;
    call @kernel::sync::maskInterrupts;
    li %t0 8;
    bne %a0 %t0 @fail;
    csrr %t0 mstatus;
    andi %t1 %s7 -9;
    bne %t0 %t1 @fail;
    call @kernel::sync::restoreInterrupts;
    csrr %t0 mstatus;
    bne %t0 %s7 @fail;
    li %a0 0;
    call @kernel::sync::restoreInterrupts;
    csrr %t0 mstatus;
    andi %t1 %s7 -9;
    bne %t0 %t1 @fail;
    call @kernel::sync::retired;
    mv %s5 %a0;
    call @kernel::sync::retired;
    bgeu %s5 %a0 @fail;
    csrr %s6 mhartid;
    li %s0 0x40010000;
    slli %s0 %s0 1;
    bnez %s6 @waitCode;
    li %t0 1;

269	270		budgets::Binding { budget: handle, domain: abi::Handle(arguments[1]), context: abi::reference(arguments[2]) }, now),
270	271		else => throw abi::Error::InvalidArg,
271	272		}
272	273		}
273	274
274		-	/// Complete a user ecall and preserve all registers except its scalar result.
275		-	export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64) {
	275	+	/// Check a retained domain's lifetime while metadata is serialized.
	276	+	unsafe fn live(owner: abi::Ref) -> bool {
	277	+	return slots::matches(&domains::STORE.slots[..], owner, slots::State::Live)
	278	+	and domains::STORE.records[owner.index].state <> domains::Lifecycle::Dead;
	279	+	}
	280	+
	281	+	/// Load a package with metadata locked only around resource transactions.
	282	+	unsafe fn runtimeLoad(domain: &mut domains::Domain, request: loader::Request, guard: sync::Guard) -> u64 throws (abi::Error) {
	283	+	let owner = domain.memory.table.owner;
	284	+	let pending = try loader::reserve(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, request)
	285	+	catch error {
	286	+	sync::leave(guard);
	287	+	throw error;
	288	+	};
	289	+	sync::leave(guard);
	290	+	let input = try loader::decode(&pending) catch error {
	291	+	let guard = sync::enter();
	292	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	293	+	sync::leave(guard); throw error;
	294	+	};
	295	+	let existing = try loader::identify(&registry::STORE, &pending, &input) catch error {
	296	+	let guard = sync::enter();
	297	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	298	+	sync::leave(guard); throw error;
	299	+	};
	300	+	let allocationGuard = sync::enter();
	301	+	if not live(owner) {
	302	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	303	+	sync::leave(allocationGuard); throw abi::Error::BadHandle;
	304	+	}
	305	+	if let object = existing {
	306	+	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	307	+	catch error {
	308	+	sync::leave(allocationGuard);
	309	+	throw error;
	310	+	};
	311	+	sync::leave(allocationGuard); return *result;
	312	+	}
	313	+	let output = try loader::reserveOutput(&mut pages::STORE.backings.pool, &mut registry::STORE) catch error {
	314	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	315	+	sync::leave(allocationGuard); throw error;
	316	+	};
	317	+	sync::leave(allocationGuard);
	318	+	let compiled = try loader::compile(&registry::STORE, &pending, &input, &output) catch error {
	319	+	let guard = sync::enter();
	320	+	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
	321	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	322	+	sync::leave(guard); throw error;
	323	+	};
	324	+	let publicationGuard = sync::enter();
	325	+	if not live(owner) {
	326	+	loader::cancelOutput(&mut pages::STORE.backings.pool, &mut registry::STORE, output);
	327	+	loader::cancel(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending);
	328	+	sync::leave(publicationGuard); throw abi::Error::BadHandle;
	329	+	}
	330	+	let object = loader::publishOutput(&mut loader::STATE, &mut pages::STORE.backings.pool, &mut registry::STORE, output, compiled);
	331	+	let result = try loader::finish(&mut loader::STATE, &mut pages::STORE, &mut domain.memory.table, pending, object)
	332	+	catch error {
	333	+	sync::leave(publicationGuard);
	334	+	throw error;
	335	+	};
	336	+	sync::leave(publicationGuard);
	337	+	return *result;
	338	+	}
	339	+
	340	+	/// Execute a call with short metadata transactions and private bulk initialization.
	341	+	/// The running context keeps its domain storage live until dispatch releases it.
	342	+	export unsafe fn synchronized(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
	343	+	assert arguments.len == 4;
	344	+	let guard = sync::enter();
	345	+	let mut domain = try domains::get(&domains::STORE, owner) catch error {
	346	+	sync::leave(guard);
	347	+	throw error;
	348	+	};
	349	+	if domain.state <> domains::Lifecycle::Active {
	350	+	sync::leave(guard);
	351	+	throw abi::Error::BadHandle;
	352	+	}
	353	+	let handle = abi::Handle(arguments[0]);
	354	+	match operation {
	355	+	case 20 => {
	356	+	let pending = try domains::reserve(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE,
	357	+	&mut domain.memory.table, handle, abi::Handle(arguments[1])) catch error {
	358	+	sync::leave(guard);
	359	+	throw error;
	360	+	};
	361	+	sync::leave(guard);
	362	+	try domains::prepare(&registry::STORE, &pending) catch error {
	363	+	let guard = sync::enter();
	364	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	365	+	sync::leave(guard); throw error;
	366	+	};
	367	+	let guard = sync::enter();
	368	+	if not live(owner) {
	369	+	domains::cancel(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending);
	370	+	sync::leave(guard); throw abi::Error::BadHandle;
	371	+	}
	372	+	let result = try domains::publish(&mut domains::STORE, &mut pages::STORE.backings, &mut domain.memory.table, pending)
	373	+	catch error {
	374	+	sync::leave(guard);
	375	+	throw error;
	376	+	};
	377	+	sync::leave(guard); return *result;
	378	+	},
	379	+	case 30 => {
	380	+	let pending = try pages::reserve(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1])
	381	+	catch error {
	382	+	sync::leave(guard);
	383	+	throw error;
	384	+	};
	385	+	sync::leave(guard);
	386	+	pages::clear(&pending);
	387	+	let guard = sync::enter();
	388	+	if not live(owner) {
	389	+	pages::cancel(&mut pages::STORE, &mut domain.memory.table, pending);
	390	+	sync::leave(guard); throw abi::Error::BadHandle;
	391	+	}
	392	+	let result = pages::publish(&mut pages::STORE, &mut domain.memory.table, pending);
	393	+	sync::leave(guard); return *result;
	394	+	},
	395	+	case 51 => return try runtimeLoad(&mut domain, loader::Request {
	396	+	authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3],
	397	+	}, guard),
	398	+	case 60 => {
	399	+	if arguments[2] <> @sizeOf(abi::ContextStart) as u64 or arguments[3] <> 0
	400	+	or not buffer(&domain, arguments[1], arguments[2], abi::Rights(abi::READ)) {
	401	+	sync::leave(guard); throw abi::Error::InvalidArg;
	402	+	}
	403	+	let words = @sliceOf(memory(arguments[1]), 4);
	404	+	let start = abi::ContextStart { entry: words[0], stack: words[1], args: words[2], size: words[3] };
	405	+	let pending = try domains::contextReserve(&mut domains::STORE, &mut pages::STORE, &registry::STORE,
	406	+	&domain.memory.table, handle, start) catch error {
	407	+	sync::leave(guard);
	408	+	throw error;
	409	+	};
	410	+	let mut target: abi::Ref = undefined;
	411	+	match &pending {
	412	+	case domains::ContextReservation::Held(allocation) => {
	413	+	set target = allocation.owner;
	414	+	},
	415	+	}
	416	+	sync::leave(guard);
	417	+	domains::contextClear(&pending);
	418	+	let guard = sync::enter();
	419	+	if not live(owner) or not live(target) {
	420	+	domains::contextCancel(&mut domains::STORE, &mut pages::STORE, pending);
	421	+	sync::leave(guard); throw abi::Error::BadHandle;
	422	+	}
	423	+	let result = domains::contextPublish(&mut domains::STORE, pending);
	424	+	sync::leave(guard); return abi::id(result);
	425	+	},
	426	+	else => {
	427	+	let result = try invoke(owner, operation, arguments, now) catch error {
	428	+	sync::leave(guard);
	429	+	throw error;
	430	+	};
	431	+	sync::leave(guard); return result;
	432	+	},
	433	+	}
	434	+	}
	435	+
	436	+	/// Complete a user ecall through the supplied execution boundary.
	437	+	export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64,
	438	+	execute: unsafe fn(abi::Ref, u64, &[u64], u64) -> u64 throws (abi::Error))
	439	+	{
276	440		assert trap::fromUser(frame) and frame.cause == 8;
277	441		assert trap::advanceCall(frame);
278	442		let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
279		-	let result = try invoke(owner, frame.registers[17], &args[..], now) catch error {
	443	+	let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
280	444		set frame.registers[10] = (0 as u64) - error as u64;
281	445		return;
282	446		};
283	447		set frame.registers[10] = result;
284	448		}

206	216		if window.start < result.deadline {
207	217		set result.deadline = window.start;
208	218		}
209	219		continue;
210	220		}
	221	+	if let owner = contexts.contexts[context.index].hart {
	222	+	if owner <> hart {
	223	+	if window.end < result.deadline {
	224	+	set result.deadline = window.end;
	225	+	}
	226	+	continue;
	227	+	}
	228	+	}
211	229		assert result.budget == nil;
212	230		set result.budget = abi::Ref { index: i, generation: windows.slots[i].generation };
213	231		set result.context = context;
214	232		if window.end < result.deadline {
215	233		set result.deadline = window.end;

234	234		}
235	235		}
236	236		throw abi::Error::InvalidArg;
237	237		}
238	238
239		-	/// Reject physical overlap with any live context's reserved user stack.
	239	+	/// Reject physical overlap with published or reserved user stacks.
240	240		fn stackAvailable(store: &Store, candidate: range::Range) throws (abi::Error) {
241	241		for i in 0..limits::CONTEXTS {
242		-	if store.contextSlots[i].state <> slots::State::Live {
	242	+	if store.contextSlots[i].state <> slots::State::Live and store.contextSlots[i].state <> slots::State::Reserved {
243	243		continue;
244	244		}
245	245		let used = store.contexts[i].userStack;
246	246		if used.start < used.end and candidate.start < used.end and used.start < candidate.end {
247	247		throw abi::Error::Busy;

298	298		}
299	299		}
300	300		return false;
301	301		}
302	302
303		-	/// Create an additional context with private kernel storage and shared domain state.
304		-	/// The caller serializes domain, capability, and physical allocation metadata.
305		-	export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
306		-	table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
	303	+	/// Private kernel storage retained by an unpublished context.
	304	+	export record ContextAllocation: Copy {
	305	+	/// Reserved context slot with an initialized user-stack claim.
	306	+	object: abi::Ref,
	307	+	/// Domain kept live until publication or cancellation.
	308	+	owner: abi::Ref,
	309	+	/// Reserved kernel-stack frames.
	310	+	frames: frames::Run,
	311	+	/// Validated physical base for private stack clearing.
	312	+	base: u64,
	313	+	}
	314	+
	315	+	/// Context storage that must be published or cancelled under serialization.
	316	+	export union ContextReservation: Once {
	317	+	/// One reserved context and its private kernel storage.
	318	+	Held(ContextAllocation),
	319	+	}
	320	+
	321	+	/// Reserve an additional context and both stack claims under metadata serialization.
	322	+	export unsafe fn contextReserve(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
	323	+	table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> ContextReservation throws (abi::Error)
307	324		{
308	325		let permission = try capability::authority(table, authority, abi::Rights(abi::EXECUTE));
309	326		let domain = try get(store, permission.object);
310	327		if domain.state == Lifecycle::Dead {
311	328		throw abi::Error::BadHandle;

320	337		let pending = try frames::reserve(&mut memory.backings.pool, KERNEL_STACK_PAGES) catch error {
321	338		try! slots::cancel(&mut store.contextSlots[..], slot); throw error;
322	339		};
323	340		let kernelFrames = frames::commit(pending);
324	341		let kernelStack = try! frames::extent(&memory.backings.pool, kernelFrames);
325		-	zero(kernelStack.start, kernelFrames.count);
326	342		let mut frame = trap::Frame { registers: [0; 32], pc: start.entry, status: 0x80, cause: 0, value: 0 };
327	343		set frame.registers[1] = returnAddress();
328	344		set frame.registers[2] = start.stack;
329	345		set frame.registers[3] = domain.graph.table.ptr as u64;
330	346		set frame.registers[10] = start.args;
331	347		set frame.registers[11] = start.size;
332	348		let object = slots::reference(&slot);
333	349		set store.contexts[object.index] = Context {
334	350		owner: permission.object, state: ContextState::Ready, hart: nil, kernelFrames, kernelStack, userStack, frame,
335	351		};
336		-	return try! slots::commit(&mut store.contextSlots[..], slot);
	352	+	match slot {
	353	+	case slots::Reservation::Held(object) => return ContextReservation::Held(ContextAllocation {
	354	+	object, owner: permission.object, frames: kernelFrames, base: kernelStack.start,
	355	+	}),
	356	+	}
	357	+	}
	358	+
	359	+	/// Clear an unpublished context's private kernel stack.
	360	+	export fn contextClear(reservation: &ContextReservation) {
	361	+	match reservation { case ContextReservation::Held(allocation) => zero(allocation.base, allocation.frames.count), }
	362	+	}
	363	+
	364	+	/// Check the reserved context and its retained physical allocation.
	365	+	fn contextRequired(store: &Store, allocation: &ContextAllocation) {
	366	+	assert slots::matches(&store.contextSlots[..], allocation.object, slots::State::Reserved);
	367	+	let context = store.contexts[allocation.object.index];
	368	+	assert context.owner == allocation.owner and context.kernelFrames == allocation.frames;
	369	+	assert context.hart == nil;
	370	+	}
	371	+
	372	+	/// Publish an initialized context while its domain stays live under serialization.
	373	+	export fn contextPublish(store: &mut Store, reservation: ContextReservation) -> abi::Ref {
	374	+	match reservation {
	375	+	case ContextReservation::Held(allocation) => {
	376	+	contextRequired(store, &allocation);
	377	+	assert slots::matches(&store.slots[..], allocation.owner, slots::State::Live);
	378	+	return try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
	379	+	},
	380	+	}
	381	+	}
	382	+
	383	+	/// Return an unpublished context's stack and slot under metadata serialization.
	384	+	export fn contextCancel(store: &mut Store, memory: &mut pages::Store, reservation: ContextReservation) {
	385	+	match reservation {
	386	+	case ContextReservation::Held(allocation) => {
	387	+	contextRequired(store, &allocation);
	388	+	try! frames::release(&mut memory.backings.pool, allocation.frames);
	389	+	try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(allocation.object));
	390	+	},
	391	+	}
	392	+	}
	393	+
	394	+	/// Create a context while the caller has exclusive access to shared metadata.
	395	+	export unsafe fn contextCreate(store: &mut Store, memory: &mut pages::Store, packages: &registry::Store,
	396	+	table: &capability::Table, authority: abi::Handle, start: abi::ContextStart) -> abi::Ref throws (abi::Error)
	397	+	{
	398	+	let pending = try contextReserve(store, memory, packages, table, authority, start);
	399	+	contextClear(&pending);
	400	+	return contextPublish(store, pending);
337	401		}
338	402
339	403		/// Memory allocated before the domain and its context become observable.
340		-	record Prepared: Copy {
	404	+	export record Prepared: Copy {
341	405		/// Frames for capabilities and the event ring.
342	406		allocation: frames::Run,
343	407		/// Mapped metadata at the allocation's start.
344	408		memory: *unsafe mut Memory,
345		-	/// Independent package-state graph.
346		-	graph: instances::Instance,
	409	+	/// Reserved private package-state graph.
	410	+	graph: instances::Allocation,
347	411		/// Retained stack allocation for the initial context.
348	412		kernelFrames: frames::Run,
349	413		/// Initial context's mapped stack bounds.
350	414		kernelStack: range::Range,
351	415		}
352	416
353	417		/// Allocate private storage and return all earlier allocations on failure.
354		-	unsafe fn prepare(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
	418	+	unsafe fn allocate(pool: &mut frames::Pool, packages: &registry::Store, image: abi::Ref) -> Prepared throws (abi::Error) {
355	419		let pending = try frames::reserve(pool, (@sizeOf(Memory) + 4095) / 4096);
356	420		let allocation = frames::commit(pending);
357		-	let graph = try instances::create(packages, pool, image) catch err {
	421	+	let graph = try instances::reserve(packages, pool, image) catch err {
358	422		try! frames::release(pool, allocation); throw err;
359	423		};
360	424		let pendingStack = try frames::reserve(pool, KERNEL_STACK_PAGES) catch err {
361		-	try! frames::release(pool, graph.frames);
	425	+	instances::cancel(pool, graph);
362	426		try! frames::release(pool, allocation); throw err;
363	427		};
364	428		let kernelFrames = frames::commit(pendingStack);
365	429		let kernelStack = try! frames::extent(pool, kernelFrames);
366	430		let extent = try! frames::extent(pool, allocation);
367		-	zero(extent.start, allocation.count);
368		-	zero(kernelStack.start, kernelFrames.count);
369		-	return Prepared { allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack };
	431	+	match graph {
	432	+	case instances::Reservation::Held(graph) => return Prepared {
	433	+	allocation, memory: memory(extent.start), graph, kernelFrames, kernelStack,
	434	+	},
	435	+	}
370	436		}
371	437
372	438		/// Return storage that has never been exposed to a published domain.
373	439		fn discard(pool: &mut frames::Pool, prepared: Prepared) {
374	440		try! frames::release(pool, prepared.kernelFrames);
375	441		try! frames::release(pool, prepared.graph.frames);
376	442		try! frames::release(pool, prepared.allocation);
377	443		}
378	444
379		-	/// Create an unscheduled domain with Events and one initial integer context.
380		-	/// The caller serializes publication and keeps the authorizing domain alive.
381		-	export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
382		-	table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
	445	+	/// Unpublished domain metadata retained during private initialization.
	446	+	export record Creation: Copy {
	447	+	/// Calling domain kept live until publication or cancellation.
	448	+	owner: abi::Ref,
	449	+	/// Reserved capability slot in the calling domain.
	450	+	handle: abi::Ref,
	451	+	/// Reserved domain slot.
	452	+	domain: abi::Ref,
	453	+	/// Reserved initial-context slot.
	454	+	context: abi::Ref,
	455	+	/// Management and inherited authority selected at admission.
	456	+	rights: abi::Rights,
	457	+	/// Resident root package.
	458	+	image: abi::Ref,
	459	+	/// Validated initial entry address.
	460	+	entry: u64,
	461	+	/// Private allocations retained until the domain becomes observable.
	462	+	storage: Prepared,
	463	+	}
	464	+
	465	+	/// Domain creation resources that must be published or cancelled.
	466	+	export union Reservation: Once {
	467	+	/// One private domain and its reserved metadata capacity.
	468	+	Held(Creation),
	469	+	}
	470	+
	471	+	/// Reserve domain capacity under metadata serialization while the caller stays live.
	472	+	export unsafe fn reserve(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
	473	+	table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> Reservation throws (abi::Error)
383	474		{
384	475		let permit = try capability::authority(table, authority, abi::Rights(abi::CREATE));
385	476		if not backing::domainLive(backings, table.owner) or not backing::domainLive(backings, permit.object) {
386	477		throw abi::Error::BadHandle;
387	478		}

105	105		assert stackTop == PLATFORM.stacks[hart as u32].end and (stackTop & 15) == 0;
106	106		let stack = PLATFORM.stacks[hart as u32];
107	107		set HARTS[hart as u32] = trap::Hart {
108	108		stackTop: stack.end, stackBottom: stack.start, kernelGp: stateTable as u64, handler: unexpected,
109	109		savedT0: 0, savedT1: 0, savedSp: 0,
	110	+	dispatchTop: stack.end, dispatchFrame: undefined,
110	111		};
111	112		trap::install(&mut HARTS[hart as u32]);
112	113		try! dispatch::initialize(&PLATFORM, hart as u32, stateTable as u64);
113	114		let mut count: u64 = 0;
114	115		for id in 0..limits::HARTS {

117	118		}
118	119		}
119	120		return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
120	121		}
121	122
122		-	/// Enter dispatch after this hart completes platform initialization.
123		-	export unsafe fn run() {
	123	+	/// Enter dispatch after every online hart publishes its initialized state.
	124	+	export unsafe fn run() -> ! {
	125	+	let mut count: u64 = 0;
	126	+	for id in 0..limits::HARTS {
	127	+	if (PLATFORM.harts & (1 << id)) <> 0 {
	128	+	set count += 1;
	129	+	}
	130	+	}
	131	+	while sync::loadAcquire(&ARRIVED) < count {
	132	+	}
	133	+	sync::syncInstructions();
124	134		dispatch::start(&mut HARTS[dispatch::hart()]);
125	135		}

12	12		use super::events;
13	13		use super::transactions;
14	14		use super::loader;
15	15		use super::dispatch;
16	16		use super::timers;
	17	+	use super::sync;
17	18
18	19		/// Validate resident object generations for capabilities without page ownership.
19	20		unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
20	21		match entry.kind {
21	22		case abi::Kind::Domain, abi::Kind::Events => {

8	8		use super::platform;
9	9		use super::trap;
10	10		use super::range;
11	11		use super::calls;
12	12		use super::timers;
	13	+	use super::sync;
13	14
14	15		/// Dispatch ownership and idle state retained by one physical hart.
15	16		export record State: Copy {
16	17		/// Physical hart whose windows this state selects.
17	18		hart: u32,

44	45		/// Runtime slots initialized before each hart enables timer interrupts.
45	46		unsafe static HARTS: [Runtime; limits::HARTS] = undefined;
46	47
47	48		/// Read the executing physical hart identifier.
48	49		export fn hart() -> u32;
	50	+	/// Read the current machine stack pointer.
	51	+	fn stackPointer() -> u64;
49	52		/// Address of the M-mode idle loop.
50	53		fn idleAddress() -> u64;
51	54		/// Enable machine timer delivery while global interrupts remain disabled.
52	55		fn enable();
53	56

62	65		state: State { hart: id, current: nil, budget: nil, idle, stack }, timer: device,
63	66		};
64	67		arm(device, 0xffffffffffffffff);
65	68		}
66	69
67		-	/// Reevaluate authority after a timer interrupt with shared metadata serialized.
68		-	export unsafe fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
	70	+	/// Tail-enter dispatch on the firmware stack and resume the selected frame.
	71	+	export fn interrupt(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> !;
	72	+
	73	+	/// Select authority from hart-owned storage with shared metadata serialized.
	74	+	export unsafe fn choose(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
69	75		let id = hart();
70	76		assert id < limits::HARTS;
	77	+	let stack = HARTS[id].state.stack;
	78	+	let pointer = stackPointer();
	79	+	assert pointer >= stack.start and pointer < stack.end;
	80	+	let guard = sync::enter();
71	81		let clock = now(HARTS[id].timer);
72	82		let timeout = try! timers::service(&mut timers::STORE, &mut domains::STORE, id, clock);
73	83		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
74	84		let mut deadline = choice.deadline;
75	85		if timeout < deadline {
76	86		set deadline = timeout;
77	87		}
78	88		arm(HARTS[id].timer, deadline);
	89	+	sync::leave(guard);
79	90		}
80	91
81	92		/// Read the running context's identity and domain-relative CPU authority.
82	93		export unsafe fn current() -> abi::CurrentContextInfo {
83	94		let id = hart();

87	98		let window = try! budgets::get(&budgets::STORE, budget);
88	99		return abi::CurrentContextInfo { context: abi::id(context), hart: id as u64, budget: *window.handle };
89	100		}
90	101
91	102		/// Complete the current user's call and select authority at the return boundary.
92		-	export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
	103	+	export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) -> ! {
93	104		let id = hart();
94	105		assert id < limits::HARTS;
95		-	let context = HARTS[id].state.current else {
96		-	trap::halt();
97		-	return;
98		-	};
	106	+	let context = HARTS[id].state.current else trap::halt();
	107	+	let guard = sync::enter();
99	108		assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
100	109		let owner = domains::STORE.contexts[context.index].owner;
101	110		if frame.registers[17] == abi::Operation::ContextReturn as u64 {
102	111		assert trap::advanceCall(frame);
103	112		try! domains::returned(&mut domains::STORE, owner, context);
	113	+	sync::leave(guard);
104	114		} else {
105		-	calls::handle(owner, frame, now(HARTS[id].timer));
	115	+	sync::leave(guard);
	116	+	calls::handle(owner, frame, now(HARTS[id].timer), calls::synchronized);
106	117		}
107	118		interrupt(frame, anchor);
108	119		}
109	120
110	121		/// Enter the first authorized context or the hart's retained idle frame.
111	122		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
112	123		let id = hart();
113	124		assert id < limits::HARTS;
114	125		let mut frame = HARTS[id].state.idle;
115		-	interrupt(&mut frame, anchor);
116	126		enable();
117		-	trap::resume(&frame);
	127	+	interrupt(&mut frame, anchor);
118	128		}
119	129
120	130		/// Read one validated, naturally aligned 64-bit device register.
121	131		fn read(address: u64) -> u64;
122	132		/// Write one validated, naturally aligned 64-bit device register.

1	1		//! Naturally aligned RV64 CLINT access with device ordering.
2	2		.text;
3	3		.export @kernel::dispatch::read;
4	4		.export @kernel::dispatch::write;
5	5		.export @kernel::dispatch::hart;
	6	+	.export @kernel::dispatch::stackPointer;
6	7		.export @kernel::dispatch::idleAddress;
7	8		.export @kernel::dispatch::enable;
	9	+	.export @kernel::dispatch::interrupt;
	10	+
	11	+	// Retain the frame outside the context stack before publishing its release.
	12	+	@kernel::dispatch::interrupt
	13	+	mv %t0 %a0;
	14	+	addi %t1 %a1 64;
	15	+	li %t2 36;
	16	+	@retainDispatchFrame
	17	+	ld %t3 0(%t0);
	18	+	sd %t3 0(%t1);
	19	+	addi %t0 %t0 8;
	20	+	addi %t1 %t1 8;
	21	+	addi %t2 %t2 -1;
	22	+	bnez %t2 @retainDispatchFrame;
	23	+	ld %sp 56(%a1);
	24	+	addi %a0 %a1 64;
	25	+	call @kernel::dispatch::choose;
	26	+	csrr %a0 mscratch;
	27	+	addi %a0 %a0 64;
	28	+	j @kernel::trap::resume;
8	29
9	30		// The executing hart selects its private dispatch runtime slot.
10	31		@kernel::dispatch::hart
11	32		csrr %a0 mhartid;
12	33		ret;
13	34
	35	+	// Dispatch releases context stacks only from the hart's firmware stack.
	36	+	@kernel::dispatch::stackPointer
	37	+	mv %a0 %sp;
	38	+	ret;
	39	+
14	40		// Saved idle frames enter this loop with machine interrupts enabled by mret.
15	41		@kernel::dispatch::idleAddress
16	42		la %a0 @dispatchIdle;
17	43		ret;
18	44		@dispatchIdle

396	487		};
397	488		let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch err {
398	489		try! slots::cancel(&mut store.slots[..], domainSlot);
399	490		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
400	491		};
401		-	let mut prepared = try prepare(&mut backings.pool, packages, root) catch err {
402		-	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
403		-	try! slots::cancel(&mut store.slots[..], domainSlot);
404		-	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
405		-	};
406		-	let object = slots::reference(&domainSlot);
407		-	try backing::registerDomain(backings, object) catch err {
408		-	discard(&mut backings.pool, prepared);
409		-	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
410		-	try! slots::cancel(&mut store.slots[..], domainSlot);
411		-	try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
412		-	};
413		-	capability::initialize(&mut prepared.memory.table, object);
414		-	try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
415		-	try! backing::endDomain(backings, object);
416		-	discard(&mut backings.pool, prepared);
	492	+	let initialSlot = slots::reference(&contextSlot);
	493	+	set store.contexts[initialSlot.index].userStack = range::Range { start: 0, end: 0 };
	494	+	let prepared = try allocate(&mut backings.pool, packages, root) catch err {
417	495		try! slots::cancel(&mut store.contextSlots[..], contextSlot);
418	496		try! slots::cancel(&mut store.slots[..], domainSlot);
419	497		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
420	498		};
421		-	let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
422		-	kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ \| abi::WRITE),
423		-	});
424		-	let initial = slots::reference(&contextSlot);
425		-	// MPIE enables interrupts after the first return into U-mode.
426		-	let mut frame = trap::Frame { registers: [0; 32], pc: entry, status: 0x80, cause: 0, value: 0 };
427		-	set frame.registers[1] = returnAddress();
428		-	set frame.registers[3] = prepared.graph.table.ptr as u64;
429		-	set store.contexts[initial.index] = Context {
430		-	owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
431		-	userStack: range::Range { start: 0, end: 0 }, frame,
432		-	};
433		-	set store.records[object.index] = Domain {
434		-	state: Lifecycle::Pending, creator: table.owner, parent: table.owner, image: root, initial,
435		-	allocation: prepared.allocation, memory: prepared.memory, graph: prepared.graph, events: queue,
	499	+	match handleSlot {
	500	+	case slots::Reservation::Held(handle) => match domainSlot {
	501	+	case slots::Reservation::Held(domain) => match contextSlot {
	502	+	case slots::Reservation::Held(context) => return Reservation::Held(Creation {
	503	+	owner: table.owner, handle, domain, context,
	504	+	rights: abi::Rights(MANAGEMENT_RIGHTS \| (*permit.rights & (abi::CREATE \| abi::ALLOCATE))),
	505	+	image: root, entry, storage: prepared,
	506	+	}),
	507	+	},
	508	+	},
	509	+	}
	510	+	}
	511	+
	512	+	/// Initialize private storage from resident packages while metadata can change.
	513	+	export unsafe fn prepare(packages: &registry::Store, reservation: &Reservation) throws (abi::Error) {
	514	+	match reservation {
	515	+	case Reservation::Held(creation) => {
	516	+	let mut storage = creation.storage;
	517	+	zero(storage.memory as u64, storage.allocation.count);
	518	+	zero(storage.kernelStack.start, storage.kernelFrames.count);
	519	+	try instances::fill(packages, &storage.graph.graph, storage.graph.base);
	520	+	capability::initialize(&mut storage.memory.table, creation.domain);
	521	+	},
	522	+	}
	523	+	}
	524	+
	525	+	/// Check the unpublished capacity retained by one creation transaction.
	526	+	fn require(store: &Store, table: &capability::Table, creation: &Creation) {
	527	+	assert table.owner == creation.owner;
	528	+	assert slots::matches(&table.slots[..], creation.handle, slots::State::Reserved);
	529	+	assert slots::matches(&store.slots[..], creation.domain, slots::State::Reserved);
	530	+	assert slots::matches(&store.contextSlots[..], creation.context, slots::State::Reserved);
	531	+	}
	532	+
	533	+	/// Return private domain storage and reserved slots under metadata serialization.
	534	+	export fn cancel(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation) {
	535	+	match reservation {
	536	+	case Reservation::Held(creation) => {
	537	+	require(store, table, &creation);
	538	+	discard(&mut backings.pool, creation.storage);
	539	+	try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
	540	+	try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
	541	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
	542	+	},
	543	+	}
	544	+	}
	545	+
	546	+	/// Publish an initialized domain and its Events under metadata serialization.
	547	+	export unsafe fn publish(store: &mut Store, backings: &mut backing::Store, table: &mut capability::Table, reservation: Reservation)
	548	+	-> abi::Handle throws (abi::Error)
	549	+	{
	550	+	match reservation {
	551	+	case Reservation::Held(creation) => {
	552	+	require(store, table, &creation);
	553	+	if not backing::domainLive(backings, creation.owner) {
	554	+	cancel(store, backings, table, Reservation::Held(creation));
	555	+	throw abi::Error::BadHandle;
	556	+	}
	557	+	let object = creation.domain;
	558	+	let initial = creation.context;
	559	+	let mut prepared = creation.storage;
	560	+	try backing::registerDomain(backings, object) catch err {
	561	+	cancel(store, backings, table, Reservation::Held(creation)); throw err;
	562	+	};
	563	+	try events::open(&mut store.events, object, &mut prepared.memory.ring) catch err {
	564	+	try! backing::endDomain(backings, object);
	565	+	cancel(store, backings, table, Reservation::Held(creation)); throw err;
	566	+	};
	567	+	let queue = try! capability::install(&mut prepared.memory.table, capability::Entry {
	568	+	kind: abi::Kind::Events, object, rights: abi::Rights(abi::READ \| abi::WRITE),
	569	+	});
	570	+	let graph = instances::commit(instances::Reservation::Held(prepared.graph));
	571	+	// MPIE enables interrupts after the first return into U-mode.
	572	+	let mut frame = trap::Frame { registers: [0; 32], pc: creation.entry, status: 0x80, cause: 0, value: 0 };
	573	+	set frame.registers[1] = returnAddress();
	574	+	set frame.registers[3] = graph.table.ptr as u64;
	575	+	set store.contexts[initial.index] = Context {
	576	+	owner: object, state: ContextState::Ready, hart: nil, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
	577	+	userStack: range::Range { start: 0, end: 0 }, frame,
	578	+	};
	579	+	set store.records[object.index] = Domain {
	580	+	state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
	581	+	allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
	582	+	};
	583	+	let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
	584	+	let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
	585	+	return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
	586	+	kind: abi::Kind::Domain, object: domain, rights: creation.rights,
	587	+	});
	588	+	},
	589	+	}
	590	+	}
	591	+
	592	+	/// Create a private domain while the caller has exclusive access to metadata.
	593	+	export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
	594	+	table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
	595	+	{
	596	+	let pending = try reserve(store, backings, packages, table, authority, image);
	597	+	try prepare(packages, &pending) catch err {
	598	+	cancel(store, backings, table, pending); throw err;
436	599		};
437		-	let context = try! slots::commit(&mut store.contextSlots[..], contextSlot);
438		-	let domain = try! slots::commit(&mut store.slots[..], domainSlot);
439		-	return capability::publish(table, handleSlot, capability::Entry {
440		-	kind: abi::Kind::Domain, object: domain,
441		-	rights: abi::Rights(MANAGEMENT_RIGHTS \| (*permit.rights & (abi::CREATE \| abi::ALLOCATE))),
442		-	});
	600	+	return try publish(store, backings, table, pending);
443	601		}

13	13		/// Package bases indexed by resident slot; absent packages have zero bases.
14	14		table: *unsafe [u64],
15	15		}
16	16
17	17		/// Bounded traversal and conservative storage requirement for a package graph.
18		-	record Graph: Copy {
	18	+	export record Graph: Copy {
19	19		/// Reachable package references, each present once.
20	20		packages: [abi::Ref; limits::PACKAGES],
21	21		/// Number of initialized package references.
22	22		count: u32,
23	23		/// Bytes for the base table, private state, and worst-case alignment padding.
24	24		bytes: u64,
25	25		}
26	26
	27	+	/// Private storage and the resident graph selected at reservation time.
	28	+	export record Allocation: Copy {
	29	+	/// Reachable immutable packages and their storage bounds.
	30	+	graph: Graph,
	31	+	/// Physical frames reserved for this instance.
	32	+	frames: frames::Run,
	33	+	/// Validated physical base for private initialization.
	34	+	base: u64,
	35	+	}
	36	+
	37	+	/// Graph storage that must be committed or returned to its frame pool.
	38	+	export union Reservation: Once {
	39	+	/// An unpublished private instance.
	40	+	Held(Allocation),
	41	+	}
	42	+
27	43		/// Map an allocated physical extent to typed storage at the call site.
28	44		fn memory(address: u64) -> *unsafe mut opaque;
29	45
30	46		/// Collect the reachable immutable packages with a bounded queue.
31	47		unsafe fn graph(store: &registry::Store, root: abi::Ref) -> Graph throws (abi::Error) {

59	75		}
60	76		return result;
61	77		}
62	78
63	79		/// Assign every base before initializing templates and their private pointers.
64		-	unsafe fn initialize(store: &registry::Store, graph: &Graph, base: u64) -> *unsafe [u64] throws (abi::Error) {
	80	+	/// The caller owns a writable extent of at least graph.bytes bytes at base.
	81	+	export unsafe fn fill(store: &registry::Store, graph: &Graph, base: u64) throws (abi::Error) {
65	82		let table = @sliceOf(memory(base) as *unsafe mut u64, limits::PACKAGES);
66	83		for i in 0..table.len {
67	84		set table[i] = 0;
68	85		}
69	86		let mut next = base + limits::PACKAGES as u64 * 8;

79	96		let bytes = @sliceOf(memory(table[object.index]) as *unsafe mut u8, package.memory);
80	97		try shared::instantiate(&package, &table[..], bytes) catch {
81	98		throw abi::Error::VerifyFailed;
82	99		};
83	100		}
84		-	return &table[..];
85	101		}
86	102
87		-	/// Allocate and initialize one private graph without copying resident code.
88		-	/// The caller serializes frame reservations and keeps the registry immutable.
89		-	export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
	103	+	/// Reserve one graph under metadata serialization while its packages stay resident.
	104	+	export unsafe fn reserve(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Reservation throws (abi::Error) {
90	105		let layout = try graph(store, root);
91	106		let count = ((layout.bytes + limits::FRAME_SIZE - 1) / limits::FRAME_SIZE) as u32;
92	107		let pending = try frames::reserve(pool, count);
93	108		let run = frames::commit(pending);
94	109		let extent = try! frames::extent(pool, run);
95		-	let table = try initialize(store, &layout, extent.start) catch err {
96		-	try! frames::release(pool, run); throw err;
	110	+	return Reservation::Held(Allocation { graph: layout, frames: run, base: extent.start });
	111	+	}
	112	+
	113	+	/// Initialize private state from the reservation's immutable resident packages.
	114	+	export unsafe fn initialize(store: &registry::Store, reservation: &Reservation) throws (abi::Error) {
	115	+	match reservation {
	116	+	case Reservation::Held(allocation) => try fill(store, &allocation.graph, allocation.base),
	117	+	}
	118	+	}
	119	+
	120	+	/// Consume initialized private storage for publication in a domain.
	121	+	export unsafe fn commit(reservation: Reservation) -> Instance {
	122	+	match reservation {
	123	+	case Reservation::Held(allocation) => return Instance {
	124	+	frames: allocation.frames,
	125	+	table: @sliceOf(memory(allocation.base) as *unsafe u64, limits::PACKAGES),
	126	+	},
	127	+	}
	128	+	}
	129	+
	130	+	/// Return unpublished graph storage under frame-pool serialization.
	131	+	export fn cancel(pool: &mut frames::Pool, reservation: Reservation) {
	132	+	match reservation {
	133	+	case Reservation::Held(allocation) => {
	134	+	try! frames::release(pool, allocation.frames);
	135	+	},
	136	+	}
	137	+	}
	138	+
	139	+	/// Allocate and initialize a private graph while the caller has exclusive metadata access.
	140	+	export unsafe fn create(store: &registry::Store, pool: &mut frames::Pool, root: abi::Ref) -> Instance throws (abi::Error) {
	141	+	let pending = try reserve(store, pool, root);
	142	+	try initialize(store, &pending) catch err {
	143	+	cancel(pool, pending); throw err;
97	144		};
98		-	return Instance { frames: run, table };
	145	+	return commit(pending);
99	146		}

85	85		export union Lease: Once {
86	86		/// Private temporary frames owned by the active load.
87	87		Held(frames::Run),
88	88		}
89	89
	90	+	/// Private input capacity retained throughout one load.
	91	+	export record Input: Copy {
	92	+	/// Domain kept live while the source and reserved handle are in use.
	93	+	owner: abi::Ref,
	94	+	/// Reserved Image handle slot in the caller's table.
	95	+	handle: abi::Ref,
	96	+	/// Private workspace frames.
	97	+	frames: frames::Run,
	98	+	/// Physical workspace base retained by the lease.
	99	+	work: u64,
	100	+	/// Validated physical source address.
	101	+	address: u64,
	102	+	/// Number of source bytes to snapshot.
	103	+	length: u32,
	104	+	}
	105	+
	106	+	/// Input capacity that must be cancelled or completed under metadata serialization.
	107	+	export union Reservation: Once {
	108	+	/// One admitted load and its private compiler storage.
	109	+	Held(Input),
	110	+	}
	111	+
90	112		/// Global runtime-loader ownership.
91	113		export unsafe static STATE: State = State { busy: false, owner: abi::Ref { index: 0, generation: 0 },
92	114		live: [false; limits::PACKAGES], resident: undefined };
93	115
94	116		/// Map a validated physical workspace extent to its compiler storage.

179	201		},
180	202		else => return abi::Error::VerifyFailed,
181	203		}
182	204		}
183	205
184		-	/// Used page counts for unpublished native output.
185		-	record OutputSize: Copy {
	206	+	/// Used native extents and validated metadata ready for publication.
	207	+	export record Compiled: Copy {
186	208		/// Pages occupied by generated code.
187	209		code: u32,
188	210		/// Pages occupied by the packed native catalog.
189	211		metadata: u32,
	212	+	/// Immutable descriptor and resolved dependencies for the reserved package.
	213	+	admission: registry::Admission,
190	214		}
191	215
192	216		/// Compile and pack one candidate into caller-owned physical output extents.
193	217		unsafe fn generate(work: *mut Workspace, input: &binary::Package, packages: &registry::Store,
194		-	slot: u32, code: u64, metadata: u64, length: u32) -> OutputSize throws (abi::Error)
	218	+	object: abi::Ref, code: u64, metadata: u64, length: u32) -> Compiled throws (abi::Error)
195	219		{
196	220		let count = try registry::imports(packages, &input.dependencies[..], &mut work.imports[..]);
197	221		let mut arena = alloc::new(&mut work.code[..]);
198	222		let mut scratch = alloc::new(&mut work.scratch[..]);
199		-	let package = try shared::compile(&*input, slot, code, &work.imports[..count],
	223	+	let package = try shared::compile(&*input, object.index, code, &work.imports[..count],
200	224		shared::Storage {
201	225		data: &mut work.data[..], symbols: &mut work.symbols[..], exports: &mut work.exports[..],
202	226		template: &mut work.template[..], relocations: &mut work.relocations[..],
203	227		}, &mut arena, &mut scratch) catch error {
204	228		throw backend(error);

216	240		};
217	241		let mut codePages = (codeBytes + 4095) / 4096;
218	242		if codePages == 0 {
219	243		set codePages = 1;
220	244		}
221		-	return OutputSize { code: codePages, metadata: (used + 4095) / 4096 };
	245	+	let native = descriptor(metadata);
	246	+	let admission = try registry::admit(packages, object, native.source, native.package);
	247	+	return Compiled { code: codePages, metadata: (used + 4095) / 4096, admission };
222	248		}
223	249
224	250		/// Return unused output frames while preserving at least one frame for each extent.
225	251		fn trim(pool: &mut frames::Pool, run: frames::Run, count: u32) -> frames::Run {
226	252		assert count > 0 and count <= run.count;

228	254		try! frames::release(pool, frames::Run { first: run.first + count, count: run.count - count });
229	255		}
230	256		return frames::Run { first: run.first, count };
231	257		}
232	258
233		-	/// Snapshot, decode, compile, and publish a package while its caller remains live.
234		-	unsafe fn produce(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
235		-	work: *mut Workspace, address: u64, length: u32) -> abi::Ref throws (abi::Error)
236		-	{
237		-	try! mem::copy(&mut work.input[..length], @sliceOf(memory(address), length));
238		-	let mut decoder = alloc::new(&mut work.decoded[..]);
239		-	let input = try program::decode(&work.input[..length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
240		-	if err == binary::Error::Storage {
241		-	throw abi::Error::Exhausted;
242		-	}
243		-	throw abi::Error::VerifyFailed;
244		-	};
245		-	let existing = try registry::identify(packages, &input.name[..], &work.input[..length]);
246		-	if let object = existing {
247		-	return object;
	259	+	/// Snapshot and decode trusted input into the held private workspace.
	260	+	/// The decoded package stays valid until this reservation is completed or cancelled.
	261	+	export unsafe fn decode(reservation: &Reservation) -> binary::Package throws (abi::Error) {
	262	+	match reservation {
	263	+	case Reservation::Held(input) => {
	264	+	let mut work = workspace(input.work);
	265	+	try! mem::copy(&mut work.input[..input.length], @sliceOf(memory(input.address), input.length));
	266	+	let mut decoder = alloc::new(&mut work.decoded[..]);
	267	+	return try program::decode(&work.input[..input.length], &mut decoder, binary::Limits { registers: 8192, blocks: 4096 }) catch err {
	268	+	if err == binary::Error::Storage {
	269	+	throw abi::Error::Exhausted;
	270	+	}
	271	+	throw abi::Error::VerifyFailed;
	272	+	};
	273	+	},
248	274		}
	275	+	}
	276	+
	277	+	/// Reserved native output and its immutable physical extents.
	278	+	export record Output: Copy {
	279	+	/// Reserved package slot selected for generated state references.
	280	+	object: abi::Ref,
	281	+	/// Private executable frames.
	282	+	code: frames::Run,
	283	+	/// Private source and descriptor frames.
	284	+	metadata: frames::Run,
	285	+	/// Validated physical executable base.
	286	+	codeBase: u64,
	287	+	/// Validated physical descriptor base.
	288	+	metadataBase: u64,
	289	+	}
	290	+
	291	+	/// Native output that must be published or cancelled under serialization.
	292	+	export union OutputReservation: Once {
	293	+	/// One unpublished package and its private output frames.
	294	+	Held(Output),
	295	+	}
	296	+
	297	+	/// Reserve a package slot and native output under metadata serialization.
	298	+	export fn reserveOutput(pool: &mut frames::Pool, packages: &mut registry::Store) -> OutputReservation throws (abi::Error) {
249	299		let slot = try registry::reserve(packages);
250	300		let object = slots::reference(&slot);
251	301		let codeReservation = try frames::reserve(pool, CODE_BYTES / 4096) catch err {
252	302		registry::cancel(packages, slot);
253	303		throw err;

257	307		try! frames::release(pool, code); registry::cancel(packages, slot); throw err;
258	308		};
259	309		let metadata = frames::commit(metadataReservation);
260	310		let codeRange = try! frames::extent(pool, code);
261	311		let metadataRange = try! frames::extent(pool, metadata);
262		-	let sizes = try generate(work, &input, packages, object.index, codeRange.start, metadataRange.start, length) catch err {
263		-	try! frames::release(pool, metadata); try! frames::release(pool, code);
264		-	registry::cancel(packages, slot); throw err;
265		-	};
266		-	let retainedCode = trim(pool, code, sizes.code);
267		-	let retainedMetadata = trim(pool, metadata, sizes.metadata);
268		-	let native = descriptor(metadataRange.start);
269		-	sync::syncInstructions();
270		-	let result = try registry::publish(packages, slot, native.source, native.package) catch err {
271		-	try! frames::release(pool, retainedMetadata); try! frames::release(pool, retainedCode); throw err;
272		-	};
273		-	assert result == object;
274		-	set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
275		-	set state.live[result.index] = true;
276		-	return result;
	312	+	match slot {
	313	+	case slots::Reservation::Held(object) => return OutputReservation::Held(Output {
	314	+	object, code, metadata, codeBase: codeRange.start, metadataBase: metadataRange.start,
	315	+	}),
	316	+	}
277	317		}
278	318
279		-	/// Load trusted RIL and install an Image capability after complete native publication.
280		-	/// The caller keeps its domain alive and serializes shared metadata transactions.
281		-	export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
282		-	table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
	319	+	/// Compile a decoded package into reserved private output using resident dependencies.
	320	+	export unsafe fn compile(packages: &registry::Store, reservation: &Reservation, input: &binary::Package, output: &OutputReservation)
	321	+	-> Compiled throws (abi::Error)
	322	+	{
	323	+	match reservation {
	324	+	case Reservation::Held(source) => match output {
	325	+	case OutputReservation::Held(target) => {
	326	+	let work = workspace(source.work);
	327	+	return try generate(work, input, packages, target.object, target.codeBase, target.metadataBase, source.length);
	328	+	},
	329	+	},
	330	+	}
	331	+	}
	332	+
	333	+	/// Return unpublished output frames and its package slot under serialization.
	334	+	export fn cancelOutput(pool: &mut frames::Pool, packages: &mut registry::Store, reservation: OutputReservation) {
	335	+	match reservation {
	336	+	case OutputReservation::Held(output) => {
	337	+	assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
	338	+	try! frames::release(pool, output.metadata);
	339	+	try! frames::release(pool, output.code);
	340	+	registry::cancel(packages, slots::Reservation::Held(output.object));
	341	+	},
	342	+	}
	343	+	}
	344	+
	345	+	/// Publish completed native output and retain its used frames under serialization.
	346	+	export fn publishOutput(state: &mut State, pool: &mut frames::Pool, packages: &mut registry::Store,
	347	+	reservation: OutputReservation, sizes: Compiled) -> abi::Ref
	348	+	{
	349	+	match reservation {
	350	+	case OutputReservation::Held(output) => {
	351	+	assert slots::matches(&packages.slots[..], output.object, slots::State::Reserved);
	352	+	let retainedCode = trim(pool, output.code, sizes.code);
	353	+	let retainedMetadata = trim(pool, output.metadata, sizes.metadata);
	354	+	sync::syncInstructions();
	355	+	let result = registry::commit(packages, slots::Reservation::Held(output.object), sizes.admission);
	356	+	assert result == output.object;
	357	+	set state.resident[result.index] = Resident { code: retainedCode, metadata: retainedMetadata };
	358	+	set state.live[result.index] = true;
	359	+	return result;
	360	+	},
	361	+	}
	362	+	}
	363	+
	364	+	/// Compare the snapshot with immutable resident content while the workspace is held.
	365	+	export unsafe fn identify(packages: &registry::Store, reservation: &Reservation, input: &binary::Package) -> ?abi::Ref throws (abi::Error) {
	366	+	match reservation {
	367	+	case Reservation::Held(source) => return try registry::identify(packages, &input.name[..], &workspace(source.work).input[..source.length]),
	368	+	}
	369	+	}
	370	+
	371	+	/// Reserve an Image handle and exclusive workspace under metadata serialization.
	372	+	export unsafe fn reserve(state: &mut State, store: &mut pages::Store,
	373	+	table: &mut capability::Table, request: Request) -> Reservation throws (abi::Error)
283	374		{
284	375		let address = try source(store, table, request);
285	376		let handle = try slots::reserve(&mut table.slots[..]);
286	377		let lease = try acquire(state, &mut store.backings.pool, table.owner) catch err {
287	378		try! slots::cancel(&mut table.slots[..], handle); throw err;
288	379		};
289		-	let work = storage(&store.backings.pool, &lease);
290		-	let object = try produce(state, &mut store.backings.pool, packages, work, address, request.length as u32) catch err {
291		-	release(state, &mut store.backings.pool, lease);
292		-	try! slots::cancel(&mut table.slots[..], handle); throw err;
	380	+	let work = storage(&store.backings.pool, &lease) as u64;
	381	+	match handle {
	382	+	case slots::Reservation::Held(handle) => match lease {
	383	+	case Lease::Held(frames) => return Reservation::Held(Input {
	384	+	owner: table.owner, handle, frames, work, address, length: request.length as u32,
	385	+	}),
	386	+	},
	387	+	}
	388	+	}
	389	+
	390	+	/// Check workspace ownership and the caller's unpublished handle slot.
	391	+	fn require(state: &State, table: &capability::Table, input: &Input) {
	392	+	assert state.busy and state.owner == input.owner and table.owner == input.owner;
	393	+	assert slots::matches(&table.slots[..], input.handle, slots::State::Reserved);
	394	+	}
	395	+
	396	+	/// Return the workspace and reserved handle under metadata serialization.
	397	+	export fn cancel(state: &mut State, store: &mut pages::Store, table: &mut capability::Table, reservation: Reservation) {
	398	+	match reservation {
	399	+	case Reservation::Held(input) => {
	400	+	require(state, table, &input);
	401	+	release(state, &mut store.backings.pool, Lease::Held(input.frames));
	402	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(input.handle));
	403	+	},
	404	+	}
	405	+	}
	406	+
	407	+	/// Install a resident package handle and return the workspace under serialization.
	408	+	export fn finish(state: &mut State, store: &mut pages::Store, table: &mut capability::Table,
	409	+	reservation: Reservation, object: abi::Ref) -> abi::Handle throws (abi::Error)
	410	+	{
	411	+	match reservation {
	412	+	case Reservation::Held(input) => {
	413	+	require(state, table, &input);
	414	+	if not backing::domainLive(&store.backings, input.owner) {
	415	+	cancel(state, store, table, Reservation::Held(input)); throw abi::Error::BadHandle;
	416	+	}
	417	+	let result = capability::publish(table, slots::Reservation::Held(input.handle), capability::Entry {
	418	+	kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
	419	+	});
	420	+	release(state, &mut store.backings.pool, Lease::Held(input.frames));
	421	+	return result;
	422	+	},
	423	+	}
	424	+	}
	425	+
	426	+	/// Load trusted RIL while the caller has exclusive access to shared metadata.
	427	+	export unsafe fn load(state: &mut State, store: &mut pages::Store, packages: &mut registry::Store,
	428	+	table: &mut capability::Table, request: Request) -> abi::Handle throws (abi::Error)
	429	+	{
	430	+	let pending = try reserve(state, store, table, request);
	431	+	let input = try decode(&pending) catch err {
	432	+	cancel(state, store, table, pending);
	433	+	throw err;
	434	+	};
	435	+	let existing = try identify(packages, &pending, &input) catch err {
	436	+	cancel(state, store, table, pending);
	437	+	throw err;
	438	+	};
	439	+	if let object = existing {
	440	+	return try finish(state, store, table, pending, object);
	441	+	}
	442	+	let output = try reserveOutput(&mut store.backings.pool, packages) catch err {
	443	+	cancel(state, store, table, pending);
	444	+	throw err;
	445	+	};
	446	+	let sizes = try compile(packages, &pending, &input, &output) catch err {
	447	+	cancelOutput(&mut store.backings.pool, packages, output);
	448	+	cancel(state, store, table, pending); throw err;
293	449		};
294		-	let result = capability::publish(table, handle, capability::Entry {
295		-	kind: abi::Kind::Image, object, rights: abi::Rights(registry::IMAGE_RIGHTS),
296		-	});
297		-	release(state, &mut store.backings.pool, lease);
298		-	return result;
	450	+	let object = publishOutput(state, &mut store.backings.pool, packages, output, sizes);
	451	+	return try finish(state, store, table, pending, object);
299	452		}

92	92		}
93	93		}
94	94		return true;
95	95		}
96	96
97		-	/// Allocate and zero frames before publishing a page and its capability.
	97	+	/// Private frames and unpublished slots held while memory is cleared.
	98	+	export record Allocation: Copy {
	99	+	/// Calling domain whose table contains the reserved handle.
	100	+	owner: abi::Ref,
	101	+	/// Unpublished capability slot.
	102	+	handle: abi::Ref,
	103	+	/// Unpublished page-object slot.
	104	+	page: abi::Ref,
	105	+	/// Unpublished backing-allocation slot.
	106	+	backing: abi::Ref,
	107	+	/// Frames excluded from allocation until publication or cancellation.
	108	+	run: frames::Run,
	109	+	/// Validated physical base for private initialization.
	110	+	base: u64,
	111	+	}
	112	+
	113	+	/// Allocation capacity that must be published or cancelled under serialization.
	114	+	export union Reservation: Once {
	115	+	/// Private memory and its three reserved metadata slots.
	116	+	Held(Allocation),
	117	+	}
	118	+
	119	+	/// Reserve page capacity under metadata serialization for private initialization.
98	120		/// Allocation authority must name the calling domain. Failure cancels reservations.
99		-	export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
100		-	-> abi::Handle throws (abi::Error)
	121	+	export fn reserve(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
	122	+	-> Reservation throws (abi::Error)
101	123		{
102	124		if count == 0 {
103	125		throw abi::Error::InvalidArg;
104	126		}
105	127		if count > limits::FRAMES as u64 {

108	130		let owner = table.owner;
109	131		let permission = try capability::authority(table, authority, abi::Rights(abi::ALLOCATE));
110	132		if permission.object <> owner {
111	133		throw abi::Error::Denied;
112	134		}
113		-	if not backing::domainLive(&store.backings, owner) {
	135	+	let backings = &mut store.backings;
	136	+	if not backing::domainLive(backings, owner) {
114	137		throw abi::Error::BadHandle;
115	138		}
116	139		let handleSlot = try slots::reserve(&mut table.slots[..]);
117	140		let pageSlot = try slots::reserve(&mut store.slots[..]) catch err {
118	141		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
119	142		};
120		-	let backingSlot = try slots::reserve(&mut store.backings.slots[..]) catch err {
	143	+	let backingSlot = try slots::reserve(&mut backings.slots[..]) catch err {
121	144		try! slots::cancel(&mut store.slots[..], pageSlot);
122	145		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
123	146		};
124		-	let memory = try frames::reserve(&mut store.backings.pool, count as u32) catch err {
125		-	try! slots::cancel(&mut store.backings.slots[..], backingSlot);
	147	+	let memory = try frames::reserve(&mut backings.pool, count as u32) catch err {
	148	+	try! slots::cancel(&mut backings.slots[..], backingSlot);
126	149		try! slots::cancel(&mut store.slots[..], pageSlot);
127	150		try! slots::cancel(&mut table.slots[..], handleSlot); throw err;
128	151		};
129	152		let run = frames::commit(memory);
130		-	let extent = try! frames::extent(&store.backings.pool, run);
131		-	zero(extent.start, run.count);
132		-	let allocation = backing::publish(&mut store.backings, backingSlot, run, owner);
133		-	let object = slots::reference(&pageSlot);
134		-	set store.records[object.index] = Page { backing: allocation, base: extent.start, count: run.count, origin: owner, handles: 1 };
135		-	let page = try! slots::commit(&mut store.slots[..], pageSlot);
136		-	return capability::publish(table, handleSlot, capability::Entry {
137		-	kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
138		-	});
	153	+	let extent = try! frames::extent(&backings.pool, run);
	154	+	match handleSlot {
	155	+	case slots::Reservation::Held(handle) => match pageSlot {
	156	+	case slots::Reservation::Held(page) => match backingSlot {
	157	+	case slots::Reservation::Held(backing) => return Reservation::Held(Allocation {
	158	+	owner, handle, page, backing, run, base: extent.start,
	159	+	}),
	160	+	},
	161	+	},
	162	+	}
	163	+	}
	164	+
	165	+	/// Clear private frames while shared metadata can be used by other harts.
	166	+	export fn clear(reservation: &Reservation) {
	167	+	match reservation { case Reservation::Held(allocation) => zero(allocation.base, allocation.run.count), }
	168	+	}
	169	+
	170	+	/// Check reserved generations before publishing or cancelling an allocation.
	171	+	fn require(store: &Store, table: &capability::Table, allocation: &Allocation) {
	172	+	assert table.owner == allocation.owner;
	173	+	assert slots::matches(&table.slots[..], allocation.handle, slots::State::Reserved);
	174	+	assert slots::matches(&store.slots[..], allocation.page, slots::State::Reserved);
	175	+	assert slots::matches(&store.backings.slots[..], allocation.backing, slots::State::Reserved);
	176	+	}
	177	+
	178	+	/// Publish cleared memory under metadata serialization while its owner stays live.
	179	+	export unsafe fn publish(store: &mut Store, table: &mut capability::Table, reservation: Reservation) -> abi::Handle {
	180	+	match reservation {
	181	+	case Reservation::Held(allocation) => {
	182	+	require(store, table, &allocation);
	183	+	assert backing::domainLive(&store.backings, allocation.owner);
	184	+	let backing = backing::publish(&mut store.backings, slots::Reservation::Held(allocation.backing), allocation.run, allocation.owner);
	185	+	set store.records[allocation.page.index] = Page {
	186	+	backing, base: allocation.base, count: allocation.run.count, origin: allocation.owner, handles: 1,
	187	+	};
	188	+	let page = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(allocation.page));
	189	+	return capability::publish(table, slots::Reservation::Held(allocation.handle), capability::Entry {
	190	+	kind: abi::Kind::Page, object: page, rights: abi::Rights(DEFAULT_RIGHTS),
	191	+	});
	192	+	},
	193	+	}
	194	+	}
	195	+
	196	+	/// Return private allocation capacity under metadata serialization.
	197	+	export fn cancel(store: &mut Store, table: &mut capability::Table, reservation: Reservation) {
	198	+	match reservation {
	199	+	case Reservation::Held(allocation) => {
	200	+	require(store, table, &allocation);
	201	+	try! frames::release(&mut store.backings.pool, allocation.run);
	202	+	try! slots::cancel(&mut store.backings.slots[..], slots::Reservation::Held(allocation.backing));
	203	+	try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(allocation.page));
	204	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(allocation.handle));
	205	+	},
	206	+	}
	207	+	}
	208	+
	209	+	/// Allocate zeroed pages while the caller has exclusive access to metadata.
	210	+	export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
	211	+	-> abi::Handle throws (abi::Error)
	212	+	{
	213	+	let pending = try reserve(store, table, authority, count);
	214	+	clear(&pending);
	215	+	return publish(store, table, pending);
139	216		}
140	217
141	218		/// Split an exclusive page into two segments with the same backing and rights.
142	219		export fn split(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, leftCount: u64)
143	220		-> abi::Handle throws (abi::Error)

61	61		/// Return unpublished capacity after a failed load.
62	62		export fn cancel(store: &mut Store, reservation: slots::Reservation) {
63	63		try! slots::cancel(&mut store.slots[..], reservation);
64	64		}
65	65
66		-	/// Validate dependencies without changing registry payloads.
67		-	unsafe fn validate(store: &Store, object: abi::Ref, package: &shared::Package) throws (abi::Error) {
	66	+	/// Validated immutable package metadata ready for bounded publication.
	67	+	export record Admission: Copy {
	68	+	/// Reserved package generation to receive the descriptor.
	69	+	object: abi::Ref,
	70	+	/// Persistent source and native descriptor.
	71	+	entry: catalog::Entry,
	72	+	/// Resolved resident dependencies in their declared order.
	73	+	dependencies: [abi::Ref; limits::PACKAGES],
	74	+	/// Number of initialized dependency references.
	75	+	count: u32,
	76	+	}
	77	+
	78	+	/// Resolve admission against immutable packages while holding sole publisher ownership.
	79	+	/// The caller keeps that ownership and the descriptor storage through commit or cancellation.
	80	+	export unsafe fn admit(store: &Store, object: abi::Ref, source: *[u8], package: shared::Package) -> Admission throws (abi::Error) {
	81	+	if not slots::matches(&store.slots[..], object, slots::State::Reserved) {
	82	+	throw abi::Error::BadHandle;
	83	+	}
	84	+	if try identify(store, &package.name[..], &source[..]) <> nil {
	85	+	throw abi::Error::Busy;
	86	+	}
68	87		if package.name.len == 0 or package.slot <> object.index or package.dependencies.len > limits::PACKAGES {
69	88		throw abi::Error::VerifyFailed;
70	89		}
	90	+	let mut result = Admission { object, entry: catalog::Entry { source, package }, dependencies: undefined, count: package.dependencies.len };
71	91		for dependency, i in package.dependencies {
72	92		let target = find(store, &dependency[..]) else {
73	93		throw abi::Error::VerifyFailed;
74	94		};
75		-	for prior in &package.dependencies[..i] {
76		-	if mem::eq(prior, dependency) {
	95	+	for prior in &result.dependencies[..i] {
	96	+	if prior == target {
77	97		throw abi::Error::VerifyFailed;
78	98		}
79	99		}
	100	+	set result.dependencies[i] = target;
80	101		}
	102	+	return result;
	103	+	}
	104	+
	105	+	/// Publish admitted metadata under serialization using only bounded reference copies.
	106	+	export fn commit(store: &mut Store, reservation: slots::Reservation, admission: Admission) -> abi::Ref {
	107	+	let object = slots::reference(&reservation);
	108	+	assert object == admission.object and slots::matches(&store.slots[..], object, slots::State::Reserved);
	109	+	assert admission.count <= limits::PACKAGES;
	110	+	for i in 0..admission.count {
	111	+	let target = admission.dependencies[i];
	112	+	assert slots::matches(&store.slots[..], target, slots::State::Live);
	113	+	set store.dependencies[object.index][i] = target;
	114	+	}
	115	+	set store.counts[object.index] = admission.count;
	116	+	set store.entries[object.index] = admission.entry;
	117	+	return try! slots::commit(&mut store.slots[..], reservation);
81	118		}
82	119
83	120		/// Publish persistent compiler output after successful admission checks.
84	121		/// All descriptor pointers and source bytes must remain immutable and resident.
85	122		/// Failure cancels the reservation; duplicate content returns its existing slot.

92	129		if let object = existing {
93	130		cancel(store, reservation);
94	131		return object;
95	132		}
96	133		let object = slots::reference(&reservation);
97		-	try validate(store, object, &package) catch err {
	134	+	let admission = try admit(store, object, source, package) catch err {
98	135		cancel(store, reservation);
99	136		throw err;
100	137		};
101		-	for dependency, i in package.dependencies {
102		-	let target = find(store, &dependency[..]) else panic "resident dependency";
103		-	set store.dependencies[object.index][i] = target;
104		-	}
105		-	set store.counts[object.index] = package.dependencies.len;
106		-	set store.entries[object.index] = catalog::Entry { source, package };
107		-	return try! slots::commit(&mut store.slots[..], reservation);
	138	+	return commit(store, reservation, admission);
108	139		}
109	140
110	141		/// Read resident metadata through a generation-bearing package reference.
111	142		export fn get(store: &Store, object: abi::Ref) -> shared::Package throws (abi::Error) {
112	143		if not slots::matches(&store.slots[..], object, slots::State::Live) {

8	8		.export @kernel::sync::fetchAdd;
9	9		.export @kernel::sync::syncInstructions;
10	10		.export @kernel::sync::deviceFence;
11	11		.export @kernel::sync::read32;
12	12		.export @kernel::sync::write32;
	13	+	.export @kernel::sync::maskInterrupts;
	14	+	.export @kernel::sync::restoreInterrupts;
	15	+	.export @kernel::sync::retired;
	16	+
	17	+	// Preserve the previous interrupt-enable bit while masking machine interrupts.
	18	+	@kernel::sync::maskInterrupts
	19	+	csrr %t0 mstatus;
	20	+	andi %a0 %t0 8;
	21	+	andi %t0 %t0 -9;
	22	+	csrw mstatus %t0;
	23	+	ret;
	24	+
	25	+	// Restore interrupt delivery while preserving other machine status fields.
	26	+	@kernel::sync::restoreInterrupts
	27	+	csrr %t0 mstatus;
	28	+	andi %t0 %t0 -9;
	29	+	or %t0 %t0 %a0;
	30	+	csrw mstatus %t0;
	31	+	ret;
	32	+
	33	+	// Count local retired instructions independently of other harts' progress.
	34	+	@kernel::sync::retired
	35	+	csrr %a0 instret;
	36	+	ret;
13	37
14	38		// Allocate a wrapping ticket and return its zero-extended u32 value.
15	39		@kernel::sync::nextTicket
16	40		li %t0 1;
17	41		amoadd.w.aqrl %a0 %t0 (%a0);

166	166		/// An unknown user operation advances PC and returns only a negative a0 error.
167	167		@test unsafe fn trapReply() throws (testing::TestError) {
168	168		let owner = initialize();
169	169		let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
170	170		set frame.registers[17] = 0xffffffffffffffff;
171		-	calls::handle(owner, &mut frame, 0);
	171	+	calls::handle(owner, &mut frame, 0, calls::invoke);
172	172		try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc);
173	173		for i in 0..32 {
174	174		if i == 10 or i == 17 {
175	175		continue;
176	176		}

57	57		try testing::expect(ended.context == nil and ended.deadline == 0xffffffffffffffff);
58	58		let other = try! dispatch::select(&BUDGETS, &DOMAINS, 1, 10);
59	59		try testing::expect(other.context == nil and other.deadline == 0xffffffffffffffff);
60	60		}
61	61
	62	+	/// A context becomes eligible only after its previous hart releases ownership.
	63	+	@test unsafe fn ownership() throws (testing::TestError) {
	64	+	initialize();
	65	+	set DOMAINS.contexts[0].hart = 1;
	66	+	let future = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 9);
	67	+	assert future.context == nil and future.deadline == 10;
	68	+	let occupied = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
	69	+	assert occupied.context == nil and occupied.deadline == 20;
	70	+	set DOMAINS.contexts[0].hart = nil;
	71	+	let free = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
	72	+	assert free.context == TABLE.owner and free.deadline == 20;
	73	+	set DOMAINS.contexts[0].hart = 0;
	74	+	let local = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);
	75	+	assert local == free;
	76	+	}
	77	+
62	78		/// Pending domains and stale context generations cannot become runnable through a budget.
63	79		@test unsafe fn lifetimes() throws (testing::TestError) {
64	80		initialize();
65	81		set DOMAINS.records[0].state = domains::Lifecycle::Pending;
66	82		let pending = try! dispatch::select(&BUDGETS, &DOMAINS, 0, 10);

122	138		stack: range::Range { start: 0x2000, end: 0x4000 },
123	139		};
124	140		let mut anchor = trap::Hart {
125	141		stackTop: 0x4000, stackBottom: 0x2000, kernelGp: 0x20000, handler,
126	142		savedT0: 0, savedT1: 0, savedSp: 0,
	143	+	dispatchTop: 0x4000, dispatchFrame: undefined,
127	144		};
128	145		let first = try! dispatch::exchange(&mut state, &BUDGETS, &mut DOMAINS, &mut frame, &mut anchor, 10);
129	146		try testing::expect(frame.pc == 0x40 and anchor.stackTop == 0xc000);
130	147		try testing::expect(DOMAINS.contexts[0].hart == 0);
131	148		set frame.pc = 0x88;

129	129		try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
130	130		try testing::expect(error == abi::Error::OutOfMemory); set exhausted = true;
131	131		};
132	132		try testing::expect(exhausted and DOMAINS.contextSlots[child.initial.index + 1].state == slots::State::Free);
133	133		set PAGES.backings.pool.count = count;
134		-	let context = try! domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
	134	+	let reserved = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
	135	+	let index = child.initial.index + 1;
	136	+	assert DOMAINS.contextSlots[index].state == slots::State::Reserved;
	137	+	let mut overlap = false;
	138	+	try domains::contextCreate(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start) catch error {
	139	+	assert error == abi::Error::Busy; set overlap = true;
	140	+	};
	141	+	assert overlap;
	142	+	let reservedFrames = DOMAINS.contexts[index].kernelFrames;
	143	+	domains::contextCancel(&mut DOMAINS, &mut PAGES, reserved);
	144	+	assert DOMAINS.contextSlots[index].state == slots::State::Free;
	145	+	for i in reservedFrames.first..reservedFrames.first + reservedFrames.count {
	146	+	assert PAGES.backings.pool.free[i];
	147	+	}
	148	+	let pendingContext = try! domains::contextReserve(&mut DOMAINS, &mut PAGES, &PACKAGES, &TABLE, handle, start);
	149	+	let stackBase = DOMAINS.contexts[index].kernelStack.start;
	150	+	let base = (&RAM[0]) as u64;
	151	+	let firstWord = ((stackBase - base) / 8) as u32;
	152	+	let lastWord = firstWord + domains::KERNEL_STACK_PAGES * 512 - 1;
	153	+	set RAM[firstWord] = 123; set RAM[lastWord] = 456;
	154	+	domains::contextClear(&pendingContext);
	155	+	assert RAM[firstWord] == 0 and RAM[lastWord] == 0;
	156	+	let context = domains::contextPublish(&mut DOMAINS, pendingContext);
135	157		let created = try! domains::context(&DOMAINS, authority.object, context);
136	158		let initial = DOMAINS.contexts[child.initial.index];
137	159		try testing::expect(context <> child.initial and created.owner == authority.object);
138	160		try testing::expect(created.frame.registers[3] == initial.frame.registers[3]);
139	161		try testing::expect(created.kernelStack.start >= initial.kernelStack.end or created.kernelStack.end <= initial.kernelStack.start);

246	268		assert (error == abi::Error::BadHandle); set rejected += 1;
247	269		};
248	270		assert (rejected == 3);
249	271		}
250	272
	273	+	/// Domain reservations retain private capacity across another complete creation.
	274	+	@test unsafe fn stagedCreation() throws (testing::TestError) {
	275	+	initialize(abi::CREATE);
	276	+	let pending = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	277	+	assert DOMAINS.slots[1].state == slots::State::Reserved;
	278	+	assert DOMAINS.contextSlots[0].state == slots::State::Reserved;
	279	+	assert TABLE.slots[2].state == slots::State::Reserved;
	280	+	assert BACKINGS.domains[1] == 0 and DOMAINS.events.queues[1].generation == 0;
	281	+	let other = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	282	+	try! domains::prepare(&PACKAGES, &pending);
	283	+	let first = try! domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, pending);
	284	+	let a = try! capability::get(&TABLE, first);
	285	+	let b = try! capability::get(&TABLE, other);
	286	+	let left = try! domains::get(&DOMAINS, a.object);
	287	+	let right = try! domains::get(&DOMAINS, b.object);
	288	+	assert a.object.index == 1 and b.object.index == 2;
	289	+	assert left.graph.table[0] <> right.graph.table[0];
	290	+	assert left.state == domains::Lifecycle::Pending and right.state == domains::Lifecycle::Pending;
	291	+	initialize(abi::CREATE);
	292	+	let cancelled = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	293	+	domains::cancel(&mut DOMAINS, &mut BACKINGS, &mut TABLE, cancelled);
	294	+	assert DOMAINS.slots[1].state == slots::State::Free;
	295	+	assert DOMAINS.contextSlots[0].state == slots::State::Free;
	296	+	assert TABLE.slots[2].state == slots::State::Free;
	297	+	for i in 0..BACKINGS.pool.count {
	298	+	assert BACKINGS.pool.free[i];
	299	+	}
	300	+	set PACKAGES.entries[0].package.relocations = &[shared::Relocation {
	301	+	offset: 0, count: 1, target: shared::DataRef { slot: 5, offset: 0 },
	302	+	}];
	303	+	let mut invalid = false;
	304	+	try domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE) catch error {
	305	+	assert error == abi::Error::VerifyFailed; set invalid = true;
	306	+	};
	307	+	assert invalid and DOMAINS.slots[1].state == slots::State::Free;
	308	+	for i in 0..BACKINGS.pool.count {
	309	+	assert BACKINGS.pool.free[i];
	310	+	}
	311	+	initialize(abi::CREATE);
	312	+	let abandoned = try! domains::reserve(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	313	+	try! domains::prepare(&PACKAGES, &abandoned);
	314	+	try! backing::endDomain(&mut BACKINGS, TABLE.owner);
	315	+	let mut dead = false;
	316	+	try domains::publish(&mut DOMAINS, &mut BACKINGS, &mut TABLE, abandoned) catch error {
	317	+	assert error == abi::Error::BadHandle; set dead = true;
	318	+	};
	319	+	assert dead and DOMAINS.slots[1].state == slots::State::Free;
	320	+	assert TABLE.slots[2].state == slots::State::Free;
	321	+	for i in 0..BACKINGS.pool.count {
	322	+	assert BACKINGS.pool.free[i];
	323	+	}
	324	+	}
	325	+
251	326		/// Creation installs only Events in the child and preserves selected authorizing rights.
252	327		@test unsafe fn pending() throws (testing::TestError) {
253	328		initialize(abi::CREATE \| abi::ALLOCATE);
254	329		let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
255	330		let authority = try! capability::get(&TABLE, handle);

48	48		fn word(address: u64) -> *mut u64 {
49	49		let pointer = &RAM[0];
50	50		return &mut RAM[((address - pointer as u64) / 8) as u32];
51	51		}
52	52
	53	+	/// Private initialization retains its graph across other registry and frame transactions.
	54	+	@test unsafe fn stagedGraph() throws (testing::TestError) {
	55	+	initialize();
	56	+	let root = register("root", &[], &[]);
	57	+	let pending = try! instances::reserve(&PACKAGES, &mut POOL, root);
	58	+	let later = register("later", &[], &[]);
	59	+	let other = try! instances::create(&PACKAGES, &mut POOL, later);
	60	+	set *word(other.table[later.index]) = 99;
	61	+	try! instances::initialize(&PACKAGES, &pending);
	62	+	let first = instances::commit(pending);
	63	+	assert first.table[root.index] <> 0 and first.table[later.index] == 0;
	64	+	assert *word(first.table[root.index]) == 7;
	65	+	assert *word(other.table[later.index]) == 99;
	66	+	assert first.frames.first + first.frames.count <= other.frames.first;
	67	+	try! frames::release(&mut POOL, first.frames);
	68	+	try! frames::release(&mut POOL, other.frames);
	69	+	let cancelled = try! instances::reserve(&PACKAGES, &mut POOL, root);
	70	+	instances::cancel(&mut POOL, cancelled);
	71	+	for i in 0..POOL.count {
	72	+	assert POOL.free[i];
	73	+	}
	74	+	}
	75	+
53	76		/// A diamond has one dependency instance per domain and independent relocated pointers.
54	77		@test unsafe fn diamond() throws (testing::TestError) {
55	78		initialize();
56	79		let base = register("base", &[], &[]);
57	80		unsafe static relocation: [shared::Relocation; 1] = undefined;

89	89		}
90	90		}
91	91		return count;
92	92		}
93	93
	94	+	/// Reserved input stays private across allocation, source mutation, and contention.
	95	+	@test unsafe fn stagedInput() throws (testing::TestError) {
	96	+	initialize();
	97	+	let request = input(43, false);
	98	+	let before = free();
	99	+	let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
	100	+	assert LOADER.busy and TABLE.slots[2].state == slots::State::Reserved;
	101	+	let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
	102	+	let decoded = try! loader::decode(&pending);
	103	+	assert decoded.program.fns.len == 1 and decoded.name.len == 6;
	104	+	let replacement = input(99, false);
	105	+	assert decoded.program.fns[0].blocks[0].instrs[0] == il::Instr::Ret { val: il::Val::Imm(43) };
	106	+	let mut busy = false;
	107	+	try loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request) catch error {
	108	+	assert error == abi::Error::Busy; set busy = true;
	109	+	};
	110	+	assert busy;
	111	+	loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
	112	+	assert not LOADER.busy and TABLE.slots[2].state == slots::State::Free;
	113	+	assert free() + 1 == before;
	114	+	assert (try! pages::get(&PAGES, (try! capability::get(&TABLE, other)).object)).count == 1;
	115	+	}
	116	+
	117	+	/// Compiled output stays private until publication and cancellation restores capacity.
	118	+	@test unsafe fn stagedOutput() throws (testing::TestError) {
	119	+	initialize();
	120	+	let request = input(43, false);
	121	+	let before = free();
	122	+	let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
	123	+	let decoded = try! loader::decode(&pending);
	124	+	let output = try! loader::reserveOutput(&mut PAGES.backings.pool, &mut PACKAGES);
	125	+	assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
	126	+	let other = try! pages::allocate(&mut PAGES, &mut TABLE, abi::Handle(0), 1);
	127	+	let used = try! loader::compile(&PACKAGES, &pending, &decoded, &output);
	128	+	assert used.code > 0 and used.metadata > 0;
	129	+	assert PACKAGES.slots[0].state == slots::State::Reserved and not LOADER.live[0];
	130	+	loader::cancelOutput(&mut PAGES.backings.pool, &mut PACKAGES, output);
	131	+	loader::cancel(&mut LOADER, &mut PAGES, &mut TABLE, pending);
	132	+	assert free() + 1 == before and not LOADER.busy;
	133	+	assert PACKAGES.slots[0].state == slots::State::Free;
	134	+	let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
	135	+	assert (try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0))).index == 0;
	136	+	}
	137	+
	138	+	/// A caller whose backing lifetime ended cannot receive a completed Image handle.
	139	+	@test unsafe fn completionLifetime() throws (testing::TestError) {
	140	+	initialize();
	141	+	let request = input(43, false);
	142	+	let handle = try! loader::load(&mut LOADER, &mut PAGES, &mut PACKAGES, &mut TABLE, request);
	143	+	let object = try! registry::image(&PACKAGES, &TABLE, handle, abi::Rights(0));
	144	+	let before = free();
	145	+	let pending = try! loader::reserve(&mut LOADER, &mut PAGES, &mut TABLE, request);
	146	+	try! backing::endDomain(&mut PAGES.backings, TABLE.owner);
	147	+	let mut dead = false;
	148	+	try loader::finish(&mut LOADER, &mut PAGES, &mut TABLE, pending, object) catch error {
	149	+	assert error == abi::Error::BadHandle; set dead = true;
	150	+	};
	151	+	assert dead and not LOADER.busy and free() == before;
	152	+	assert TABLE.slots[3].state == slots::State::Free;
	153	+	assert slots::matches(&PACKAGES.slots[..], object, slots::State::Live);
	154	+	}
	155	+
94	156		/// Resident source bytes survive mutation of the source page and workspace release.
95	157		@test unsafe fn residentSnapshot() throws (testing::TestError) {
96	158		initialize();
97	159		let request = input(43, false);
98	160		let before = free();

46	46		/// Drop the test's handle and its object reference in the same serialized operation.
47	47		unsafe fn drop(handle: abi::Handle) {
48	48		try! pages::drop(&mut STORE, &mut TABLE, handle);
49	49		}
50	50
	51	+	/// Reserved frames stay private while another allocation changes shared metadata.
	52	+	@test unsafe fn stagedAllocation() throws (testing::TestError) {
	53	+	initialize();
	54	+	let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 2);
	55	+	assert STORE.slots[0].state == slots::State::Reserved;
	56	+	assert TABLE.slots[1].state == slots::State::Reserved;
	57	+	assert not STORE.backings.pool.free[0] and not STORE.backings.pool.free[1];
	58	+	assert RAM[OFFSET] == 0xa5;
	59	+	let other = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 6);
	60	+	set RAM[OFFSET + 8192] = 0x7b;
	61	+	pages::clear(&pending);
	62	+	for i in OFFSET..OFFSET + 8192 {
	63	+	assert RAM[i] == 0;
	64	+	}
	65	+	assert RAM[OFFSET + 8192] == 0x7b;
	66	+	let handle = pages::publish(&mut STORE, &mut TABLE, pending);
	67	+	let entry = try! capability::get(&TABLE, handle);
	68	+	let page = try! pages::get(&STORE, entry.object);
	69	+	assert page.count == 2 and page.origin == TABLE.owner;
	70	+	drop(handle); drop(other);
	71	+	try! backing::endDomain(&mut STORE.backings, TABLE.owner);
	72	+	for i in 0..8 {
	73	+	assert STORE.backings.pool.free[i];
	74	+	}
	75	+	}
	76	+
	77	+	/// Cancelling unpublished storage returns all capacity without exposing its bytes.
	78	+	@test unsafe fn cancelAllocation() throws (testing::TestError) {
	79	+	initialize();
	80	+	let pending = try! pages::reserve(&mut STORE, &mut TABLE, abi::Handle(0), 8);
	81	+	pages::cancel(&mut STORE, &mut TABLE, pending);
	82	+	assert STORE.slots[0].state == slots::State::Free;
	83	+	assert STORE.backings.slots[0].state == slots::State::Free;
	84	+	assert TABLE.slots[1].state == slots::State::Free;
	85	+	for i in 0..8 {
	86	+	assert STORE.backings.pool.free[i];
	87	+	}
	88	+	assert RAM[OFFSET] == 0xa5;
	89	+	let full = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 8);
	90	+	let entry = try! capability::get(&TABLE, full);
	91	+	assert (try! pages::get(&STORE, entry.object)).count == 8;
	92	+	}
	93	+
51	94		/// Transfers keep both domains exposed without duplicating page handles.
52	95		@test unsafe fn transferExposure() throws (testing::TestError) {
53	96		initialize();
54	97		let receiver = abi::Ref { index: 1, generation: 1 };
55	98		capability::initialize(&mut TARGET, receiver);