radiance · commit

compiler/radiance/images.rad +2 -2

//! Binary RIL image catalogs and native image append through the shared backend.

mod graph;
mod native;
use std::lang::il::images::graph;
use std::lang::il::images::native;

use std::mem;
use std::fmt;
use std::sys::unix;
use std::collections::dict;

kernel/Makefile +39 -14

EMU ?= $(or $(RAD_EMULATOR),emulator)
HOST_EMU ?= $(EMU)
COMPILER := ../bin/radiance.rv64.dev
COMPILE := $(HOST_EMU) -memory-size=385024 -data-size=348160 -stack-size=512 -run $(COMPILER)
MODULES := $(wildcard core/*.rad)
CORE_ASM := arch/atomic.ras arch/context.ras arch/clock.ras arch/mmio.ras arch/physical.ras arch/smp.ras
CORE := -pkg core -mod core.rad $(addprefix -mod ,$(MODULES) $(CORE_ASM))
STD_FILES := $(addprefix ../,$(shell cat ../std.lib))
STD := -pkg std $(addprefix -mod ,$(STD_FILES))
CORE_ASM := arch/atomic.ras arch/context.ras arch/clock.ras arch/mmio.ras arch/physical.ras arch/smp.ras arch/loading.ras
CORE := $(STD) -pkg core -mod core.rad $(addprefix -mod ,$(MODULES) $(CORE_ASM))
CHECK_MODULES := $(wildcard check/*.rad)
USER_INPUTS := user.rad $(wildcard user/*.rad user/*/*.rad) core/abi.rad
USER_BASE := -pkg abi -mod core/abi.rad -pkg user -mod user.rad -mod user/sys.rad -pkg probe -mod user/probe.rad
USER_BASE := -pkg abi -mod core/abi.rad -pkg user -mod user.rad -mod user/sys.rad -mod user/launch.rad -pkg probe -mod user/probe.rad
BASE_IMAGES := sample_root sample_control sample_scalars sample_memory sample_overflow sample_events sample_alias sample_instance
CONTROL_IMAGES := control flow exiting faulting spinning admin remote
BASE_RIL := $(addprefix build/,$(addsuffix .ril,$(BASE_IMAGES)))
CONTROL_RIL := $(addprefix build/,$(addsuffix .ril,$(CONTROL_IMAGES)))
IMAGE_MODULES_sample_control := user/sample_control/math.rad
IMAGE_MODULES_control := user/control/lifecycle.rad user/control/authority.rad
ROOT_CATALOG := -zero-bss -pkg images -mod build/root/images.rad -mod user/sys.ras -image build/loader.ril
LOADING_CATALOG := -zero-bss -pkg images -mod build/loading/images.rad -mod user/sys.ras -image build/load_check.ril
BASE_CATALOG := -zero-bss -pkg images -mod build/baseline/images.rad -mod user/sys.ras $(addprefix -image ,$(BASE_RIL))
CONTROL_CATALOG := -zero-bss -pkg images -mod build/control/images.rad -mod user/sys.ras $(addprefix -image ,$(CONTROL_RIL))
MACHINE := $(EMU) -machine -no-guard-stack -max-steps=1000000000 -count-instructions
MACHINE := $(EMU) -machine -memory-size=196608 -no-guard-stack -max-steps=1000000000 -count-instructions

.PHONY: all check clean compiler-check
.DELETE_ON_ERROR:
all: kernel.rv64



build/control/images.rad: $(CONTROL_RIL) $(COMPILER)
	mkdir -p $(@D)
	$(COMPILE) -catalog $@ $(addprefix -image ,$(CONTROL_RIL))

kernel.rv64: main.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(COMPILER)
	$(COMPILE) $(CONTROL_CATALOG) $(CORE) -pkg kernel -start arch/start.ras -mod main.rad -entry kernel -o $@
build/root/images.rad: build/loader.ril $(COMPILER)
	mkdir -p $(@D)
	$(COMPILE) -catalog $@ -image $<

build/loading/images.rad: build/load_check.ril $(COMPILER)
	mkdir -p $(@D)
	$(COMPILE) -catalog $@ -image $<

check.rv64: check.rad core.rad $(MODULES) $(CORE_ASM) $(CHECK_MODULES) build/baseline/images.rad user/sys.ras $(COMPILER)
# Keep the admission test input as raw RIL bytes.
build/loading/input.ras: build/runtime.ril
	mkdir -p $(@D)
	{ printf '.text;\n.export @"loading::input";\n@"loading::input"\nla %%a0 @input;\nret;\n'; \
	  printf '.export @"loading::inputSize";\n@"loading::inputSize"\nli %%a0 %s;\nret;\n.data;\n@input\n' "$$(wc -c < $<)"; \
	  od -An -v -tu1 $< | sed -e 's/^ *//' -e 's/  */, /g' -e 's/^/.byte /' -e 's/$$/;/'; \
	} > $@

kernel.rv64: main.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/root/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(ROOT_CATALOG) $(CORE) -pkg kernel -start arch/start.ras -mod main.rad -entry kernel -o $@

check.rv64: check.rad core.rad $(MODULES) $(CORE_ASM) $(CHECK_MODULES) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg check -mod check.rad $(addprefix -mod ,$(CHECK_MODULES)) -entry check -o $@

context.rv64: context.rad context/wait.rad check/context.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(COMPILER)
context.rv64: context.rad context/wait.rad check/context.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg context -start arch/entry.ras -mod context.rad -mod context/wait.rad -mod check/context.ras -entry context -o $@

interrupt.rv64: interrupt.rad check/interrupt.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(COMPILER)
interrupt.rv64: interrupt.rad check/interrupt.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg interrupt -start arch/entry.ras -mod interrupt.rad -mod check/interrupt.ras -entry interrupt -o $@

native.rv64: native.rad native/instances.rad arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(COMPILER)
native.rv64: native.rad native/instances.rad arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg native -start arch/entry.ras -mod native.rad -mod native/instances.rad -entry native -o $@

parallel.rv64: parallel.rad parallel/mailboxes.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(COMPILER)
parallel.rv64: parallel.rad parallel/mailboxes.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(CONTROL_CATALOG) $(CORE) -pkg parallel -start arch/start.ras -mod parallel.rad -mod parallel/mailboxes.rad -entry parallel -o $@

check: all check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64
loading.rv64: loading.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/loading/images.rad build/loading/input.ras user/sys.ras $(STD_FILES) $(COMPILER)
	$(COMPILE) $(LOADING_CATALOG) $(CORE) -pkg loading -start arch/start.ras -mod loading.rad -mod build/loading/input.ras -entry loading -o $@

check: all check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64 loading.rv64
	$(HOST_EMU) -run check.rv64
	$(MACHINE) -max-steps=1000000 -run context.rv64
	$(MACHINE) -harts=2 -irq=3 -irq-at=4000000 -uart-rx=52 -uart-rx-at=8000000 -run interrupt.rv64
	$(MACHINE) -run native.rv64
	$(MACHINE) -harts=1 -run parallel.rv64
	$(MACHINE) -harts=2 -run parallel.rv64
	$(MACHINE) -harts=8 -run parallel.rv64
	$(MACHINE) -harts=1 -run loading.rv64
	$(MACHINE) -harts=2 -run loading.rv64
	@output="$$( $(MACHINE) -harts=2 -run kernel.rv64 2>build/boot.log )"; status=$$?; \
	if test "$$status" -ne 2 || test "$$output" != CONTROL-OK; then cat build/boot.log; exit 1; fi
	if test "$$status" -ne 2 || test "$$output" != RIL-READY; then cat build/boot.log; exit 1; fi

clean:
	rm -rf build
	rm -f kernel.rv64 check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64
	rm -f kernel.rv64 check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64 loading.rv64

kernel/arch/context.ras +2 -0

	mret;
@idleLoop
	wfi;
	j      @idleLoop;
@enterUser
	// Synchronize this hart with native images published under the kernel lock.
	fence.i;
	mv     %t6    %a0;
	ld     %t0    256(%t6);
	csrw   mepc   %t0;
	li     %t0    0x80;
	csrw   mstatus %t0;

kernel/arch/entry.ras +2 -1

.text;
	la     %t0    @bootFault;
	csrw   mtvec  %t0;
	csrr   %a0    mhartid;
	bnez   %a0    @bootIdle;
	call   @"::default";
	la     %t0    @"::default";
	jalr   %ra    %t0    0;
	li     %t1    0x5555;
	j      @bootFinish;
@bootFault
	csrr   %t1    mcause;
	slli   %t1    %t1    16;

kernel/arch/loading.ras added +40 -0

1	+	// Exact user::sys native allowlist; index is below 14.
2	+	// Each table entry is a two-instruction la followed by ret.
3	+	.text;
4	+	.export @"core::loading::bindingAddress";
5	+	@"core::loading::bindingAddress"
6	+	slli %t1 %a0 1;
7	+	add %a0 %a0 %t1;
8	+	slli %a0 %a0 2;
9	+	la %t0 @bindings;
10	+	add %t0 %t0 %a0;
11	+	jalr %zero %t0 0;
12	+	@bindings
13	+	la %a0 @"user::sys::rawCall";
14	+	ret;
15	+	la %a0 @"user::sys::pointer";
16	+	ret;
17	+	la %a0 @"user::sys::loadAcquire";
18	+	ret;
19	+	la %a0 @"user::sys::storeRelease";
20	+	ret;
21	+	la %a0 @"user::sys::lockAcquire";
22	+	ret;
23	+	la %a0 @"user::sys::lockRelease";
24	+	ret;
25	+	la %a0 @"user::sys::read8";
26	+	ret;
27	+	la %a0 @"user::sys::read16";
28	+	ret;
29	+	la %a0 @"user::sys::read32";
30	+	ret;
31	+	la %a0 @"user::sys::read64";
32	+	ret;
33	+	la %a0 @"user::sys::write8";
34	+	ret;
35	+	la %a0 @"user::sys::write16";
36	+	ret;
37	+	la %a0 @"user::sys::write32";
38	+	ret;
39	+	la %a0 @"user::sys::write64";
40	+	ret;

kernel/arch/start.ras +2 -1

    bgeu %a0 %t0 @bootFault;
    la %sp @hartStacks;
    addi %t0 %a0 1;
    slli %t0 %t0 16;
    add %sp %sp %t0;
    call @"::default";
    la %t0 @"::default";
    jalr %ra %t0 0;
    csrr %t0 mhartid;
    bnez %t0 @bootIdle;
    li %t1 0x5555;
    j @bootFinish;
@bootFault

kernel/check.rad +1 -1

mod activation;
mod lifecycle;
mod calls;

/// Run the available kernel mechanism checks.
@default fn main() -> u32 {
@default unsafe fn main() -> u32 {
    frames::run();
    boot::run();
    handles::run();
    domains::run();
    capabilities::run();

kernel/check/activation.rad +15 -12

use core::memory;
use core::pages;
use core::platform;
use core::resources;
use core::state;
use images;
use core::images;

/// Test-owned physical memory for Page and private-stack allocations.
static RAM: [u8; 64 * frames::PAGE_SIZE] = undefined;
/// Domain slots include spare creation capacity unless explicitly exhausted.
static DOMAINS: [domains::Domain; 3] = undefined;

static PINS: [u16; 64] = undefined;
static ASSIGNED: [bool; 64] = undefined;
static GRANTS: [u64; 3] = undefined;
/// Shared state for the exercised public operations.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Zero only the supplied simulated physical range.
fn clear(base: u64, size: u32) {
    assert base <= RAM.len as u64 and size as u64 <= RAM.len as u64 - base;
    for i in base as u32..base as u32 + size { set RAM[i] = 0; }

    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 3);
    domains::init(&mut DOMAINS[..]);
    resources::init(&mut OBJECTS[..]);
    for i in 0..CONTEXTS.len { contexts::init(&mut CONTEXTS[i], i); }
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    let handle = domains::root(&mut DOMAINS[..], images::ROOT);
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let handle = domains::root(&mut DOMAINS[..], KERNEL.images.root);
    let root = try! domains::resolve(&DOMAINS[..], 0, handle, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, images::ROOT);
    let child = try! domains::create(&mut DOMAINS[..], root, KERNEL.images.root);
    return handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
}

/// Read a rejected call's public error without accepting accidental success.
fn createError(authority: abi::Handle, image: u64) -> abi::Error {


/// A rejected image/capacity request cannot consume authority or an incarnation.
fn creation() {
    let target = setup();
    let self = abi::Handle { bits: 0 };
    assert createError(self, images::COUNT as u64) == abi::Error::VerifyFailed;
    assert createError(self, KERNEL.images.count as u64) == abi::Error::VerifyFailed;
    assert createError(self, 0x100000000) == abi::Error::VerifyFailed;
    assert createError(target, images::ROOT as u64) == abi::Error::Denied;
    assert createError(target, KERNEL.images.root as u64) == abi::Error::Denied;
    let child = try! domains::resolve(&DOMAINS[..], 0, target, abi::EXECUTE);
    let foreign = handles::install(&mut DOMAINS[0].handles, 3, child, abi::CREATE);
    assert createError(foreign, images::ROOT as u64) == abi::Error::Denied;
    assert createError(foreign, KERNEL.images.root as u64) == abi::Error::Denied;
    set DOMAINS[0].events.reserved = events::CRITICAL;
    assert createError(self, images::ROOT as u64) == abi::Error::Exhausted;
    assert createError(self, KERNEL.images.root as u64) == abi::Error::Exhausted;
    assert DOMAINS[0].events.reserved == events::CRITICAL;
    set DOMAINS[0].events.reserved = 1;
    let root = try! domains::authority(&DOMAINS[..], 0, self, abi::CREATE);
    for i in 4..abi::MAX_HANDLES {
        let _handle = handles::install(&mut DOMAINS[0].handles, i, root, abi::EXECUTE);
    }
    assert createError(self, images::ROOT as u64) == abi::Error::Exhausted;
    assert createError(self, KERNEL.images.root as u64) == abi::Error::Exhausted;
    for i in 4..abi::MAX_HANDLES { let _entry = handles::remove(&mut DOMAINS[0].handles, i); }
    assert DOMAINS[2].epoch == 0 and DOMAINS[2].state == domains::Lifecycle::Dead;
    let _last = try! domains::create(&mut DOMAINS[..], root, images::ROOT);
    assert createError(self, images::ROOT as u64) == abi::Error::Exhausted;
    let _last = try! domains::create(&mut DOMAINS[..], root, KERNEL.images.root);
    assert createError(self, KERNEL.images.root as u64) == abi::Error::Exhausted;
    assert POOL.available == 64;
    assert domains::live(&DOMAINS[..], try! domains::resolve(&DOMAINS[..], 0, target, abi::EXECUTE));
}

/// Verify stack edges, adjacent args, allocation rollback, and context isolation.

    assert DOMAINS[1].env.context == first;
    assert CONTEXTS[a].env.stackBase == 0 and CONTEXTS[a].env.stackTop == 8192;
    assert CONTEXTS[a].env.argsPointer == 4096 and CONTEXTS[a].env.argsSize == 8192;
    assert CONTEXTS[a].env.eventsPointer == &DOMAINS[1].events.ring as u64;
    assert CONTEXTS[a].env.privateStackTop - CONTEXTS[a].env.privateStackBase == activation::PRIVATE_STACK_SIZE as u64;
    assert CONTEXTS[a].frame.pc == images::get(images::ROOT).entry;
    assert CONTEXTS[a].frame.pc == images::get(KERNEL.images, KERNEL.images.root).entry;
    assert startError(target, 8192, 0, 0, true) == abi::Error::NotPending;
    // Empty args need no Page, even for an otherwise invalid pointer.
    let second = try! activation::context(&mut KERNEL, 0, target, 4096, 0xffffffffffffffff, 0, clear);
    let b = try! contexts::resolve(&CONTEXTS[..], second);
    assert first <> second and a <> b;

kernel/check/budget_caps.rad +5 -1

use core::handles;
use core::memory;
use core::platform;
use core::resources;
use core::state;
use core::images;

/// Two independent protection domains for authority checks.
static DOMAINS: [domains::Domain; 2] = undefined;
/// Exactly enough resource slots for a reserve, a loan, and its replacement.
static OBJECTS: [resources::Slot; 3] = undefined;

static ASSIGNED: [bool; 1] = undefined;
/// Empty-pool persistent domain grants.
static GRANTS: [u64; 2] = undefined;
/// Shared kernel mechanism storage.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Reject a split without consuming any authority from the source.
fn reject(source: abi::Handle, expected: abi::Error) {
    let _handle = try budget_caps::split(&mut KERNEL, 0, source, 1) catch error {
        assert error == expected;

    set machine.memoryCount = 0;
    set machine.reservedCount = 0;
    try! frames::init(&mut POOL, &machine);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 2);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let rootHandle = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, 0);
    let target = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
    set DOMAINS[0].state = domains::Lifecycle::Active;

kernel/check/calls.rad +25 -6

use core::lifecycle;
use core::memory;
use core::platform;
use core::resources;
use core::state;
use core::images;
use core::loading;

static DOMAINS: [domains::Domain; 2] = undefined;
static OBJECTS: [resources::Slot; 2] = undefined;
static CONTEXTS: [contexts::Context; 1] = undefined;
static POOL: frames::Pool = undefined;
static PINS: [u16; 1] = undefined;
static ASSIGNED: [bool; 1] = undefined;
static GRANTS: [u64; 2] = undefined;
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Exercise a returning operation through exactly the public seven-word input.
fn reply(caller: u32, number: u64, args: [u64; 7], now: u64) -> [u64; 4] {
unsafe fn reply(caller: u32, number: u64, args: [u64; 7], now: u64) -> [u64; 4] {
    let result = try! calls::dispatch(&mut KERNEL, caller, number, &args[..], now);
    match result {
        case calls::Outcome::Reply { values } => return values,
        else => panic "reply: expected completed direct call",
    }
}

/// Observe a boundary rejection without substituting a lower-level helper.
fn reject(caller: u32, number: u64, args: [u64; 7], expected: abi::Error) {
unsafe fn reject(caller: u32, number: u64, args: [u64; 7], expected: abi::Error) {
    let _result = try calls::dispatch(&mut KERNEL, caller, number, &args[..], 100) catch error {
        assert error == expected;
        return;
    };
    panic "reject: invalid direct call succeeded";
}

/// Slot kinds, reserved bits, and MMIO widths/rights remain full-width inputs.
fn materialization(root: abi::Handle) {
unsafe fn materialization(root: abi::Handle) {
    let device = try! resources::create(&mut OBJECTS[..], resources::Value::Device(fdt::Range { base: 0x1000, size: 8 }));
    resources::retain(&mut OBJECTS[..], device);
    let handle = handles::install(&mut DOMAINS[0].handles, 3, device, abi::READ);
    let readonly = reply(0, 10, [handle.bits, 0, 0, 0, 0, 0, 0], 0)[0];
    let info = reply(0, 45, [readonly, 0, 0, 0, 0, 0, 0], 0);

    reject(0, 62, [3, 3, 0, 0, 0, 0, 0], abi::Error::BadHandle);
    set OBJECTS[device.index].epoch = device.epoch;
}

/// Running metadata is conservatively debited; Run keeps admission transactional.
fn execution(target: abi::Handle, child: abi::Object) {
unsafe fn execution(target: abi::Handle, child: abi::Object) {
    set DOMAINS[child.index].state = domains::Lifecycle::Active;
    try! contexts::prepare(&mut CONTEXTS[0], child, 0, 0, DOMAINS[child.index].env);
    let id = contexts::identity(&CONTEXTS[0]);
    set DOMAINS[child.index].env.context = id;
    let resource = try! resources::create(&mut OBJECTS[..], resources::Value::Budget(try! budgets::init(0, 1000)));

    reject(0, 73, args, abi::Error::Busy);
    set CONTEXTS[0].status = contexts::Status::Stopped;
    reject(0, 75, [target.bits, id + (1 as u64 << 32), 0, 0, 0, 0, 0], abi::Error::InvalidArg);
}

/// Admission rejects bad authority and byte ranges before touching physical RAM.
unsafe fn admission(root: abi::Handle, target: abi::Handle) {
    let count = KERNEL.images.count;
    let available = POOL.available;
    reject(1, 77, [0, 4096, 4, 0, 0, 0, 0], abi::Error::Denied);
    reject(0, 77, [target.bits, 4096, 4, 0, 0, 0, 0], abi::Error::Denied);
    reject(0, 77, [root.bits, 4096, 0, 0, 0, 0, 0], abi::Error::InvalidArg);
    reject(0, 77, [root.bits, 4096, loading::MAX_BYTES as u64 + 1, 0, 0, 0, 0], abi::Error::InvalidArg);
    reject(0, 77, [root.bits, 0xfffffffffffffffe, 4, 0, 0, 0, 0], abi::Error::InvalidArg);
    reject(0, 77, [root.bits, 4096, 4, 0, 0, 0, 0], abi::Error::Denied);
    assert KERNEL.images.count == count and POOL.available == available;
}

/// Independent target-local administrative authority survives source death.
export fn run() {
export unsafe fn run() {
    domains::init(&mut DOMAINS[..]);
    contexts::init(&mut CONTEXTS[0], 0);
    resources::init(&mut OBJECTS[..]);
    let mut machine: platform::Platform = undefined;
    set machine.memoryCount = 0;
    set machine.reservedCount = 0;
    try! frames::init(&mut POOL, &machine);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 2);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let rootHandle = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, 0);
    let target = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
    admission(rootHandle, target);
    materialization(rootHandle);
    execution(target, child);
    reject(0, 76, [rootHandle.bits, target.bits, 0x100000020, 0, 0, 0, 0], abi::Error::InvalidArg);
    reject(0, 76, [rootHandle.bits, target.bits, abi::DESTROY as u64, 0, 0, 0, 0], abi::Error::InvalidArg);
    let createGrant = (abi::CREATE | abi::GRANT) as u64;

kernel/check/capabilities.rad +5 -1

use core::resources;
use core::frames;
use core::memory;
use core::platform;
use core::state;
use core::images;

/// Domain storage for capability transactions.
static DOMAINS: [domains::Domain; 3] = undefined;
/// Physical resources used by the transactions.
static OBJECTS: [resources::Slot; 4] = undefined;

static ASSIGNED: [bool; 1] = undefined;
/// Per-domain grant backing storage.
static GRANTS: [u64; 3] = undefined;
/// Shared mechanism state for each transaction.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Return a grant error as a test value.
fn grantError(source: abi::Handle, target: abi::Handle) -> abi::Error {
    let _handle = try capabilities::grant(&mut KERNEL, 0, source, target, abi::RIGHTS as u64) catch error {
        return error;

    set machine.memoryCount = 0;
    set machine.reservedCount = 0;
    try! frames::init(&mut POOL, &machine);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 3);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let rootHandle = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, 1);
    let childHandle = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
    let device = try! resources::create(&mut OBJECTS[..], resources::Value::Device(fdt::Range { base: 0x10000000, size: 256 }));

kernel/check/devices.rad +5 -1

use core::handles;
use core::memory;
use core::platform;
use core::resources;
use core::state;
use core::images;

/// One caller and its device authority.
static DOMAINS: [domains::Domain; 1] = undefined;
/// Device windows, including deliberately invalid physical bounds.
static OBJECTS: [resources::Slot; 4] = undefined;

static ASSIGNED: [bool; 1] = undefined;
/// Domain grant backing.
static GRANTS: [u64; 1] = undefined;
/// Shared state for the checked materialization entry point.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Install a real device resource and read-only capability without touching MMIO.
fn device(base: u64, size: u64) -> abi::Handle {
    let object = try! resources::create(&mut OBJECTS[..], resources::Value::Device(fdt::Range { base, size }));
    resources::retain(&mut OBJECTS[..], object);

    set machine.memoryCount = 0;
    set machine.reservedCount = 0;
    try! frames::init(&mut POOL, &machine);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 1);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let root = domains::root(&mut DOMAINS[..], 0);
    let registers = device(0x1000, 16);
    assert try! devices::access(&KERNEL, 0, registers, 8, 8, abi::READ) == 0x1008;
    assert error(registers, 16, 1, abi::READ) == abi::Error::InvalidArg;
    assert error(registers, 12, 8, abi::READ) == abi::Error::InvalidArg;

kernel/check/lifecycle.rad +5 -1

use core::memory;
use core::pages;
use core::platform;
use core::resources;
use core::state;
use core::images;

/// Simulated physical backing, never dereferenced as a host physical address.
static RAM: [u8; 64 * frames::PAGE_SIZE] = undefined;
/// Enough slots for a creator chain, survivors, and an unrelated replacement.
static DOMAINS: [domains::Domain; 8] = undefined;

static PINS: [u16; 64] = undefined;
static ASSIGNED: [bool; 64] = undefined;
static GRANTS: [u64; 8] = undefined;
/// Public mechanism state shared by each isolated scenario.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Clear only the allocator's simulated physical range.
fn clear(base: u64, size: u32) {
    assert base <= RAM.len as u64 and size as u64 <= RAM.len as u64 - base;
    for i in base as u32..base as u32 + size { set RAM[i] = 0; }

    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], DOMAINS.len);
    domains::init(&mut DOMAINS[..]);
    resources::init(&mut OBJECTS[..]);
    for i in 0..CONTEXTS.len { contexts::init(&mut CONTEXTS[i], i); }
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let root = domains::root(&mut DOMAINS[..], 0);
    return try! domains::resolve(&DOMAINS[..], 0, root, abi::CREATE);
}

/// Install test-owned authority through the real handle generation mechanism.

kernel/check/notifications.rad +5 -1

use core::memory;
use core::notifications;
use core::platform;
use core::resources;
use core::state;
use core::images;

/// Independent sender and receiver domain storage.
static DOMAINS: [domains::Domain; 2] = undefined;
/// Reserve and execution-loan resource slots.
static OBJECTS: [resources::Slot; 3] = undefined;

static ASSIGNED: [bool; 1] = undefined;
/// Empty-pool grants.
static GRANTS: [u64; 2] = undefined;
/// Kernel mechanism storage.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Consume the receiver's next Wakeup with acquire/release progress.
fn consume(token: u32, sender: u32) {
    let queue = &mut DOMAINS[1].events;
    let head = atomic::load(&queue.ring.head);

    set machine.memoryCount = 0;
    set machine.reservedCount = 0;
    try! frames::init(&mut POOL, &machine);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 2);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let rootHandle = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, 0);
    let target = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
    set DOMAINS[0].state = domains::Lifecycle::Active;

kernel/check/pages.rad +5 -1

use core::memory;
use core::pages;
use core::platform;
use core::resources;
use core::state;
use core::images;

/// Test-owned physical RAM.
static RAM: [u8; 16 * frames::PAGE_SIZE] = undefined;
/// Domain storage for allocator and recipient authority.
static DOMAINS: [domains::Domain; 3] = undefined;

static ASSIGNED: [bool; 16] = undefined;
/// Per-domain persistent grant sets.
static GRANTS: [u64; 3] = undefined;
/// State shared by all tested control-plane operations.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Clear a physical allocation before it is exposed through a handle.
fn clear(base: u64, size: u32) {
    assert base <= RAM.len as u64 and size as u64 <= RAM.len as u64 - base;
    for i in base as u32..base as u32 + size { set RAM[i] = 0; }

    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 3);
    domains::init(&mut DOMAINS[..]);
    resources::init(&mut OBJECTS[..]);
    contexts::init(&mut CONTEXTS[0], 0);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let rootHandle = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, 1);
    let childHandle = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
    let left = try! pages::allocate(&mut KERNEL, 0, abi::Handle { bits: 0 }, 4, clear);

kernel/context/wait.rad +5 -1

use core::notifications;
use core::platform;
use core::resources;
use core::state;
use core::timers;
use core::images;

/// Trusted test program that requests a timeout and consumes its event after Wait.
fn entry() -> u64;
/// One complete domain and event queue.
static DOMAINS: [domains::Domain; 1] = undefined;

static GRANTS: [u64; 1] = undefined;
/// Private one-shot timer storage.
static TIMERS: timers::Store = undefined;
/// Shared kernel mechanisms.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Resume one authorized user interval until a real trap and charge it once.
unsafe fn step() -> u64 {
    let context = &mut CONTEXTS[0];
    let budget = budget_caps::value(&mut KERNEL, context.budget);

    set machine.memoryCount = 0;
    set machine.reservedCount = 0;
    try! frames::init(&mut POOL, &machine);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 1);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let handle = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, handle, abi::EXECUTE);
    set DOMAINS[0].state = domains::Lifecycle::Active;
    set DOMAINS[0].env.eventsPointer = &DOMAINS[0].events.ring as u64;
    try! contexts::prepare(&mut CONTEXTS[0], root, entry(), 0, DOMAINS[0].env);

kernel/core.rad +2 -0

export mod calls;
export mod execution;
export mod control;
export mod teardown;
export mod runtime;
export mod images;
export mod loading;

kernel/core/activation.rad +13 -10

use core::frames;
use core::handles;
use core::pages;
use core::resources;
use core::state;
use images;
use core::images;
use core::smp;

/// Fixed private native stack per context: 64 KiB for locals and saved registers.
/// The authorized Page stack has separate bounds in Env.
export constant PRIVATE_STACK_SIZE: u32 = 64 * 1024;


    run: frames::Run,
    /// Aligned mutable-state base within the owned run.
    base: u64,
}

/// Allocate and copy a checked catalog initializer before domain installation.
/// Allocate and copy a published image initializer before domain installation.
/// The caller installs the returned ownership or reclaims it on failure.
export fn initialize(pool: *mut frames::Pool, image: u32, clear: fn(u64, u32), copy: fn(u64, u64, u32)) -> ImageState throws (abi::Error) {
    let descriptor = images::get(image);
export fn initialize(pool: *mut frames::Pool, registry: *images::Registry, image: u32, clear: fn(u64, u32), copy: fn(u64, u64, u32)) -> ImageState throws (abi::Error) {
    let descriptor = images::get(registry, image);
    let mut run = frames::Run { first: 0, count: 0 };
    let mut base: u64 = 0;
    if descriptor.stateSize <> 0 {
        let alignment = descriptor.stateAlignment as u64;
        // These are trusted compiler facts, not caller-provided metadata.

        };
        set run = frames::install(allocation);
        let first = run.first as u64 * frames::PAGE_SIZE as u64;
        set base = (first + alignment - 1) & ~(alignment - 1);
        copy(base, descriptor.initial, descriptor.stateSize);
        // The immutable relocation helper may have been compiled on another hart.
        smp::syncInstructions();
        descriptor.relocate(base);
    }
    return ImageState { run, base };
}

/// Instantiate an admitted catalog graph under the caller's Create authority.
/// image is only a catalog index, never an entry address or runtime descriptor.
/// Instantiate an admitted image graph under the caller's Create authority.
/// image is only a stable image ID, never an entry address or runtime descriptor.
/// clear(base, size) zeros exactly the supplied physical allocation;
/// copy(destination, source, size) copies exactly the immutable initializer.
/// Both callbacks obey the serialization and lifetime rules of this module.
/// All capacity checks precede allocation; failure installs no authority or
/// domain incarnation and returns any provisionally allocated private frames.
export fn create(kernel: *mut state::State, caller: u32, authority: abi::Handle, image: u64, clear: fn(u64, u32), copy: fn(u64, u64, u32)) -> abi::Handle throws (abi::Error) {
    let creator = try domains::authority(kernel.domains, caller, authority, abi::CREATE);
    if creator.index <> caller { throw abi::Error::Denied; }
    if image >= images::COUNT as u64 { throw abi::Error::VerifyFailed; }
    if image >= kernel.images.count as u64 { throw abi::Error::VerifyFailed; }
    let slot = try handles::vacant(&kernel.domains[caller].handles);
    let _domainSlot = try domains::vacant(kernel.domains);
    // Refresh only consumed event credits; do not reserve a child until commit.
    try events::refresh(&mut kernel.domains[caller].events);
    if kernel.domains[caller].events.reserved == events::CRITICAL {
        throw abi::Error::Exhausted;
    }
    let initial = try initialize(kernel.memory.pool, image as u32, clear, copy);
    let initial = try initialize(kernel.memory.pool, kernel.images, image as u32, clear, copy);
    // The consumer can publish another head during allocation. Revalidate it
    // when reserving the terminal event; even that late failure returns RAM.
    let object = try domains::create(kernel.domains, creator, image as u32) catch error {
        if initial.run.count <> 0 { frames::reclaim(kernel.memory.pool, initial.run); }
        throw error;

            break;
        }
    }
    if slot == kernel.contexts.len { throw abi::Error::Exhausted; }
    let domain = &kernel.domains[owner.index];
    assert domain.image < images::COUNT;
    let descriptor = images::get(domain.image);
    assert domain.image < kernel.images.count;
    let descriptor = images::get(kernel.images, domain.image);
    let allocation = try frames::allocate(kernel.memory.pool, PRIVATE_STACK_SIZE / frames::PAGE_SIZE, clear) catch {
        throw abi::Error::OutOfMemory;
    };
    let private = frames::install(allocation);
    let privateBase = private.first as u64 * frames::PAGE_SIZE as u64;

kernel/core/boot.rad +10 -7

use core::physical;
use core::platform;
use core::resources;
use core::smp;
use core::state;
use images;
use core::images;

/// Space for one Page per tracked frame plus all bounded non-Page resources.
constant RESOURCE_CAPACITY: u32 = frames::MAX_FRAMES + 512 + platform::MAX_DEVICES + platform::MAX_IRQS;
/// Root's authorized Page stack; activation allocates a separate native stack.
constant ROOT_STACK_FRAMES: u32 = 16;

    let contextStore = storage(pool, contexts::CAPACITY, @sizeOf(contexts::Context), @alignOf(contexts::Context));
    let pinStore = storage(pool, frames::MAX_FRAMES, @sizeOf(u16), @alignOf(u16));
    let objectStore = storage(pool, frames::MAX_FRAMES, @sizeOf(bool), @alignOf(bool));
    let grantCount = frames::MAX_FRAMES / 64 * abi::MAX_DOMAINS;
    let grantStore = storage(pool, grantCount, @sizeOf(u64), @alignOf(u64));
    let imageStore = storage(pool, 1, @sizeOf(images::Registry), @alignOf(images::Registry));
    set kernel.images = imageStore as *mut images::Registry;
    images::init(kernel.images);
    set kernel.domains = @sliceOf(domainStore as *mut domains::Domain, abi::MAX_DOMAINS);
    set kernel.resources = @sliceOf(resourceStore as *mut resources::Slot, RESOURCE_CAPACITY);
    set kernel.contexts = @sliceOf(contextStore as *mut contexts::Context, contexts::CAPACITY);
    domains::init(kernel.domains);
    resources::init(kernel.resources);

export unsafe fn init(kernel: *mut state::State, pool: *mut frames::Pool, machine: *platform::Platform, controller: *mut interrupts::Controller) -> u64 {
    assert pool.limit > 0 and pool.limit <= frames::MAX_FRAMES;
    assert machine.deviceCount <= platform::MAX_DEVICES;
    assert machine.irqCount > 0 and machine.irqCount < platform::MAX_IRQS;
    assert machine.hartCount > 0 and machine.hartCount <= platform::MAX_HARTS;
    assert machine.frequency > 0 and images::ROOT < images::COUNT;
    assert machine.frequency > 0;
    let budgetBaseSlot = 2 + machine.deviceCount + machine.irqCount;
    let bootstrapBudgetSlot = budgetBaseSlot + machine.hartCount;
    assert bootstrapBudgetSlot + 3 <= abi::MAX_HANDLES;
    let mut bootHartIndex = machine.hartCount;
    for i in 0..machine.hartCount {

        if machine.harts[i].id == 0 { set bootHartIndex = i; }
    }
    assert bootHartIndex < machine.hartCount;
    tables(kernel, pool);
    interrupts::init(controller, machine, bootHartIndex);
    let root = domains::root(kernel.domains, images::ROOT);
    let root = domains::root(kernel.domains, kernel.images.root);
    let owner = try! domains::resolve(kernel.domains, 0, root, 0);
    let initial = try! activation::initialize(pool, images::ROOT, physical::clear, physical::copy);
    let initial = try! activation::initialize(pool, kernel.images, kernel.images.root, physical::clear, physical::copy);
    set kernel.domains[0].private = initial.run;
    set kernel.domains[0].env.stateBase = initial.base;

    // Fixed slots contain no implicit discovery or ambient resource authority.
    for i in 0..machine.deviceCount {

    let bootstrap = try! budgets::init(0, frequency * 60);
    let bootBudget = install(kernel, bootstrapBudgetSlot, resources::Value::Budget(bootstrap), BUDGET_RIGHTS);

    // These two Page objects follow all fixed authority slots: startup, then stack.
    // Large catalogs may span several frames but still occupy one startup Page.
    let wordCount = 8 as u64 + machine.hartCount as u64 * 2 + images::COUNT as u64;
    let wordCount = 8 as u64 + machine.hartCount as u64 * 2 + kernel.images.count as u64;
    let byteCount = wordCount * 8;
    let startupFrames = (byteCount + frames::PAGE_SIZE as u64 - 1) / frames::PAGE_SIZE as u64;
    let startup = try! pages::allocate(kernel, 0, root, startupFrames, physical::clear);
    let args = try! pages::access(kernel, 0, startup, abi::READ | abi::WRITE);
    let words = @sliceOf(physical::bytes(args.base, byteCount as u32).ptr as *mut u64, wordCount as u32);

    set words[2] = machine.irqCount as u64;
    set words[3] = machine.hartCount as u64;
    set words[4] = budgetBaseSlot as u64;
    set words[5] = bootstrapBudgetSlot as u64;
    set words[6] = frequency;
    set words[7] = images::COUNT as u64;
    set words[7] = kernel.images.count as u64;
    for i in 0..machine.hartCount {
        set words[8 + i * 2] = machine.harts[i].id as u64;
        set words[9 + i * 2] = (budgetBaseSlot + i) as u64;
    }
    for i in 0..images::COUNT {
    for i in 0..kernel.images.count {
        set words[8 + machine.hartCount * 2 + i] = i as u64;
    }
    let stack = try! pages::allocate(kernel, 0, root, ROOT_STACK_FRAMES as u64, physical::clear);
    let stackRange = try! pages::access(kernel, 0, stack, abi::WRITE);
    let id = try! activation::activate(kernel, 0, root, stackRange.base + stackRange.size,

kernel/core/calls.rad +4 -1

use core::domains;
use core::events;
use core::frames;
use core::handles;
use core::lifecycle;
use core::loading;
use core::notifications;
use core::pages;
use core::physical;
use core::platform;
use core::resources;

/// Returning effects finish here. Runtime completes control obligations before
/// replying, advances returning ecalls, and faults query/slot errors (44..48,
/// 62, 74, 75) and undefined calls rather than exposing their error return.
/// Calls 60/61/62 support trusted runtime wrappers: Page bounds, one transient
/// Device access, and a typed existing-slot lookup. RIL verification is deferred.
export fn dispatch(kernel: *mut state::State, caller: u32, number: u64, arguments: *[u64], now: u64) -> Outcome throws (abi::Error) {
/// The caller holds the kernel lock; available frames must name mapped free RAM.
export unsafe fn dispatch(kernel: *mut state::State, caller: u32, number: u64, arguments: *[u64], now: u64) -> Outcome throws (abi::Error) {
    assert caller < kernel.domains.len and arguments.len >= 7;
    let first = abi::Handle { bits: arguments[0] };
    let second = abi::Handle { bits: arguments[1] };
    let mut values: [u64; 4] = [0; 4];
    match number {

            set values[1] = kernel.contexts[index].hart as u64;
            set values[2] = kernel.contexts[index].status as u64;
            set values[3] = try contextRemaining(kernel, index, now);
        },
        case 76 => set values[0] = (try delegate(kernel, caller, first, second, arguments[2])).bits,
        case 77 => set values[0] = try loading::load(kernel, caller, first, arguments[1], arguments[2]),
        else => throw abi::Error::InvalidArg,
    }
    return Outcome::Reply { values };
}

kernel/core/control.rad +1 -1


/// Distinguish undefined calls from errors returned by defined control operations.
fn defined(number: u64) -> bool {
    return (number >= 10 and number <= 12) or (number >= 20 and number <= 23)
        or number == 30 or number == 31 or (number >= 40 and number <= 50)
        or (number >= 60 and number <= 62) or (number >= 70 and number <= 76);
        or (number >= 60 and number <= 62) or (number >= 70 and number <= 77);
}

/// Commit one call. Architectural return and budget charging must precede this.
unsafe fn apply(runtime: *mut execution::State, index: u32, now: u64) throws (abi::Error) {
    let kernel = runtime.kernel;

kernel/core/images.rad added +78 -0

1	+	//! Machine-lifetime native images, serialized by the kernel critical section.
2	+	//!
3	+	//! IDs are monotonically published and never reused. Registry allocations are
4	+	//! not Page objects or domain private state and survive every domain teardown.
5	+
6	+	use core::abi;
7	+	use core::frames;
8	+	use images;
9	+
10	+	/// Maximum admitted images, including immutable bootstrap descriptors.
11	+	export constant CAPACITY: u32 = 64;
12	+
13	+	/// Shared native code and initializer with separately allocated private state.
14	+	export record Descriptor: Copy {
15	+	/// Native entry accepting a context Env pointer.
16	+	entry: u64,
17	+	/// Immutable bytes copied into each new domain's private state.
18	+	initial: u64,
19	+	/// Bytes required for each domain's private image state.
20	+	stateSize: u32,
21	+	/// Power-of-two alignment of private image state.
22	+	stateAlignment: u32,
23	+	/// Repair private pointers after copying initializer bytes to base.
24	+	relocate: fn(u64),
25	+	}
26	+
27	+	/// Stable image storage borrowed by one kernel state for machine lifetime.
28	+	export record Registry: Copy {
29	+	/// Published descriptors; only indices below count may be read.
30	+	descriptors: [Descriptor; CAPACITY],
31	+	/// Exclusive native allocations; bootstrap images use empty runs.
32	+	native: [frames::Run; CAPACITY],
33	+	/// Next image ID and number of fully published descriptors.
34	+	count: u32,
35	+	/// Bootstrap root image ID, retained independently of later admissions.
36	+	root: u32,
37	+	}
38	+
39	+	/// Initialize fresh registry storage from the linked bootstrap package once.
40	+	/// No live registry may be reset because its native allocations are permanent.
41	+	export fn init(registry: *mut Registry) {
42	+	assert images::COUNT > 0 and images::COUNT <= CAPACITY;
43	+	assert images::ROOT < images::COUNT;
44	+	set registry.count = 0;
45	+	set registry.root = images::ROOT;
46	+	for i in 0..images::COUNT {
47	+	let source = images::get(i);
48	+	set registry.descriptors[i] = Descriptor {
49	+	entry: source.entry, initial: source.initial,
50	+	stateSize: source.stateSize, stateAlignment: source.stateAlignment,
51	+	relocate: source.relocate,
52	+	};
53	+	set registry.native[i] = frames::Run { first: 0, count: 0 };
54	+	}
55	+	set registry.count = images::COUNT;
56	+	}
57	+
58	+	/// Preflight the next stable ID without publishing or reserving a descriptor.
59	+	export fn vacant(registry: *Registry) -> u32 throws (abi::Error) {
60	+	if registry.count == CAPACITY { throw abi::Error::Exhausted; }
61	+	return registry.count;
62	+	}
63	+
64	+	/// Read a published immutable descriptor while holding the kernel lock.
65	+	export fn get(registry: *Registry, image: u32) -> Descriptor {
66	+	assert image < registry.count;
67	+	return registry.descriptors[image];
68	+	}
69	+
70	+	/// Publish a finished native allocation under the previously checked next ID.
71	+	/// Caller holds the global lock across preflight, compilation and publication.
72	+	/// Every architectural user entry instruction-fences its hart before execution.
73	+	export fn publish(registry: *mut Registry, image: u32, descriptor: Descriptor, run: frames::Run) {
74	+	assert image == registry.count and image < CAPACITY and run.count > 0;
75	+	set registry.descriptors[image] = descriptor;
76	+	set registry.native[image] = run;
77	+	set registry.count = image + 1;
78	+	}

kernel/core/loading.rad added +120 -0

1	+	//! Trusted binary RIL admission under the global kernel critical section.
2	+	//!
3	+	//! Decoding and native compilation do not constitute a safety or provenance
4	+	//! proof. The input snapshot, string pool and bounded compiler arenas occupy
5	+	//! temporary physical frames; no compiler workspace lives in kernel BSS.
6	+
7	+	use core::abi;
8	+	use core::domains;
9	+	use core::frames;
10	+	use core::images;
11	+	use core::pages;
12	+	use core::physical;
13	+	use core::state;
14	+	use std::lang::alloc;
15	+	use std::lang::il;
16	+	use std::lang::strings;
17	+
18	+	/// Largest accepted binary RIL byte range.
19	+	export constant MAX_BYTES: u32 = 16 * 1024 * 1024;
20	+	/// Temporary contiguous storage reclaimed after every admission attempt.
21	+	export constant WORKSPACE_BYTES: u32 = 96 * 1024 * 1024;
22	+	/// Persistent compiler metadata and emitted code before final placement.
23	+	constant ARENA_BYTES: u32 = 32 * 1024 * 1024;
24	+	/// Number of permitted native user::sys primitives, in address-table order.
25	+	constant BINDING_COUNT: u32 = 14;
26	+
27	+	/// Address of one exact user::sys RAS primitive; index is below BINDING_COUNT.
28	+	/// The native table contains no privileged kernel exports.
29	+	fn bindingAddress(index: u32) -> u64;
30	+
31	+	/// Fill the complete runtime native allowlist without exposing kernel symbols.
32	+	fn bindings(destination: *mut [il::images::Binding]) {
33	+	let names: [*[u8]; BINDING_COUNT] = [
34	+	"user::sys::rawCall", "user::sys::pointer",
35	+	"user::sys::loadAcquire", "user::sys::storeRelease",
36	+	"user::sys::lockAcquire", "user::sys::lockRelease",
37	+	"user::sys::read8", "user::sys::read16",
38	+	"user::sys::read32", "user::sys::read64",
39	+	"user::sys::write8", "user::sys::write16",
40	+	"user::sys::write32", "user::sys::write64",
41	+	];
42	+	for i in 0..BINDING_COUNT {
43	+	set destination[i] = il::images::Binding { name: names[i], address: bindingAddress(i) };
44	+	}
45	+	}
46	+
47	+	/// Compile a private immutable snapshot and retain only a completed allocation.
48	+	/// The caller owns workspace and reclaims it on both success and failure.
49	+	unsafe fn compile(kernel: *mut state::State, image: u32, source: u64, size: u32, workspace: frames::Run) -> u64 throws (abi::Error) {
50	+	let base = workspace.first as u64 * frames::PAGE_SIZE as u64;
51	+	physical::copy(base, source, size);
52	+	let snapshot = physical::bytes(base, size);
53	+	let poolBase = base + MAX_BYTES as u64;
54	+	let pool = physical::bytes(poolBase, @sizeOf(strings::Pool)).ptr as mut opaque as mut strings::Pool;
55	+	// Frame allocation cleared every pool slot and count without a large value copy.
56	+	let poolBytes = (@sizeOf(strings::Pool) + frames::PAGE_SIZE - 1) & ~(frames::PAGE_SIZE - 1);
57	+	let arenaBase = poolBase + poolBytes as u64;
58	+	let mut arena = alloc::new(physical::bytes(arenaBase, ARENA_BYTES));
59	+	let scratchBase = arenaBase + ARENA_BYTES as u64;
60	+	let scratchBytes = WORKSPACE_BYTES - MAX_BYTES - poolBytes - ARENA_BYTES;
61	+	let mut scratch = alloc::new(physical::bytes(scratchBase, scratchBytes));
62	+	let mut allowed: [il::images::Binding; BINDING_COUNT] = undefined;
63	+	bindings(&mut allowed[..]);
64	+	let mut prepared = try il::images::prepare(snapshot, &allowed[..], &mut arena, &mut scratch, pool) catch {
65	+	throw abi::Error::VerifyFailed;
66	+	};
67	+	let alignment = prepared.alignment as u64;
68	+	if prepared.size == 0 or alignment == 0 or alignment & (alignment - 1) <> 0 {
69	+	throw abi::Error::VerifyFailed;
70	+	}
71	+	let mut padding: u64 = 0;
72	+	if alignment > frames::PAGE_SIZE as u64 { set padding = alignment - frames::PAGE_SIZE as u64; }
73	+	let bytes = prepared.size as u64 + padding;
74	+	let count = (bytes + frames::PAGE_SIZE as u64 - 1) / frames::PAGE_SIZE as u64;
75	+	if count > frames::MAX_FRAMES as u64 { throw abi::Error::OutOfMemory; }
76	+	let allocation = try frames::allocate(kernel.memory.pool, count as u32, physical::clear) catch {
77	+	throw abi::Error::OutOfMemory;
78	+	};
79	+	let native = frames::install(allocation);
80	+	let first = native.first as u64 * frames::PAGE_SIZE as u64;
81	+	let destination = (first + alignment - 1) & ~(alignment - 1);
82	+	let compiled = try il::images::finish(&mut prepared, physical::bytes(destination, prepared.size)) catch {
83	+	frames::reclaim(kernel.memory.pool, native);
84	+	throw abi::Error::VerifyFailed;
85	+	};
86	+	images::publish(kernel.images, image, images::Descriptor {
87	+	entry: compiled.entry, initial: compiled.initial,
88	+	stateSize: compiled.stateSize, stateAlignment: compiled.stateAlignment,
89	+	relocate: compiled.relocate,
90	+	}, native);
91	+	return image as u64;
92	+	}
93	+
94	+	/// Admit trusted binary RIL without consuming or changing any input Page rights.
95	+	/// Requires live caller-self Create\|Allocate authority and complete Read coverage.
96	+	/// All range, capacity and authority failures precede allocation. Compiler errors
97	+	/// publish nothing; workspace and provisional native frames are always reclaimed.
98	+	/// The global lock remains held through snapshot, compilation and publication.
99	+	/// The frame pool must contain only mapped RAM and exclude all live kernel data.
100	+	export unsafe fn load(kernel: *mut state::State, caller: u32, authority: abi::Handle, source: u64, size: u64) -> u64 throws (abi::Error) {
101	+	let owner = try domains::authority(kernel.domains, caller, authority, abi::CREATE \| abi::ALLOCATE) catch {
102	+	throw abi::Error::Denied;
103	+	};
104	+	if owner.index <> caller { throw abi::Error::Denied; }
105	+	if size == 0 or size > MAX_BYTES as u64 or source > 0xffffffffffffffff - size {
106	+	throw abi::Error::InvalidArg;
107	+	}
108	+	if not pages::covers(kernel, caller, source, size, abi::READ) { throw abi::Error::Denied; }
109	+	let image = try images::vacant(kernel.images);
110	+	let allocation = try frames::allocate(kernel.memory.pool, WORKSPACE_BYTES / frames::PAGE_SIZE, physical::clear) catch {
111	+	throw abi::Error::OutOfMemory;
112	+	};
113	+	let workspace = frames::install(allocation);
114	+	let admitted = try compile(kernel, image, source, size as u32, workspace) catch error {
115	+	frames::reclaim(kernel.memory.pool, workspace);
116	+	throw error;
117	+	};
118	+	frames::reclaim(kernel.memory.pool, workspace);
119	+	return admitted;
120	+	}

kernel/core/smp.rad +1 -1

/// clearing after an unlocked empty observation can lose a concurrent kick.
export unsafe fn acknowledge(clint: u64, bank: u32);

/// Synchronize this hart's instruction fetch after observing published code.
/// Each executing hart must call this before entering newly published code.
export unsafe fn syncInstructions();
export fn syncInstructions();

kernel/core/state.rad +3 -0


use core::domains;
use core::contexts;
use core::memory;
use core::resources;
use core::images;

/// Kernel-owned state accessed while the machine critical section is held.
export record State: Copy {
    /// Protection-domain slots and their capability tables.
    domains: *mut [domains::Domain],

    resources: *mut [resources::Slot],
    /// Execution identities and protected architectural frames.
    contexts: *mut [contexts::Context],
    /// Physical allocations and persistent domain-lifetime pins.
    memory: memory::Memory,
    /// Borrowed machine-lifetime image descriptors and native frame ownership.
    images: *mut images::Registry,
}

kernel/interrupt.rad +5 -1

use core::memory;
use core::mmio;
use core::platform;
use core::resources;
use core::state;
use core::images;

/// One ordered byte access to an already checked device address.
unsafe fn read8(address: u64) -> u8;
/// One ordered byte write to an already checked device address.
unsafe fn write8(address: u64, value: u8);

static GRANTS: [u64; 2] = undefined;
/// Private PLIC ownership and coalescing state.
static PLIC: interrupts::Controller = undefined;
/// Serialized control-plane state.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Install exclusive authority for one real PLIC source.
fn interrupt(source: u32, root: abi::Object) -> abi::Handle {
    let object = try! resources::create(&mut OBJECTS[..], resources::Value::Interrupt(resources::Interrupt { number: source, target: root }));
    resources::retain(&mut OBJECTS[..], object);

    set empty.memoryCount = 0;
    set empty.reservedCount = 0;
    try! frames::init(&mut POOL, &empty);
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 2);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let own = domains::root(&mut DOMAINS[..], 0);
    let root = try! domains::resolve(&DOMAINS[..], 0, own, abi::CREATE);
    let child = try! domains::create(&mut DOMAINS[..], root, 0);
    let target = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
    let device = try! resources::create(&mut OBJECTS[..], resources::Value::Device(machine.devices[0].region));

kernel/loading.rad added +76 -0

1	+	//! Runtime RIL admission and execution through the actual user syscall path.
2	+	use core::abi;
3	+	use core::boot;
4	+	use core::contexts;
5	+	use core::domains;
6	+	use core::handles;
7	+	use core::pages;
8	+	use core::physical;
9	+	use core::runtime;
10	+	use core::smp;
11	+	use core::execution;
12	+
13	+	/// Embedded binary RIL fixture, with no linked native implementation.
14	+	fn input() -> u64;
15	+	/// Exact binary fixture byte extent.
16	+	fn inputSize() -> u64;
17	+
18	+	/// Supply a root-owned input Page after kernel boot has completed.
19	+	unsafe fn arguments(execution: *mut execution::State) {
20	+	let kernel = execution.kernel;
21	+	let root = try! handles::get(&kernel.domains[0].handles, 0, abi::Kind::Domain);
22	+	let context = try! contexts::resolve(kernel.contexts, kernel.domains[0].env.context);
23	+	let env = &mut kernel.contexts[context].env;
24	+	let boot = @sliceOf(physical::bytes(env.argsPointer, env.argsSize as u32).ptr as *u64, env.argsSize as u32 / 8);
25	+	let localReserve = boot[9];
26	+	let frequency = boot[6];
27	+	let mut remoteReserve: u64 = 0;
28	+	if boot[3] > 1 { set remoteReserve = boot[11]; }
29	+	let mut output: u64 = 0;
30	+	for i in 0..execution.machine.deviceCount {
31	+	if execution.machine.devices[i].region.base == 0x10000000 {
32	+	set output = (try! handles::get(&kernel.domains[0].handles, i + 2, abi::Kind::Device)).bits;
33	+	}
34	+	}
35	+	assert output <> 0;
36	+	let size = inputSize();
37	+	assert size > 0 and size <= 16 * 1024 * 1024;
38	+	let source = try! pages::allocate(kernel, 0, root, (size + 4095) / 4096, physical::clear);
39	+	let sourceRange = try! pages::access(kernel, 0, source, abi::WRITE);
40	+	physical::copy(sourceRange.base, input(), size as u32);
41	+	let args = try! pages::allocate(kernel, 0, root, 1, physical::clear);
42	+	let range = try! pages::access(kernel, 0, args, abi::WRITE);
43	+	let words = @sliceOf(physical::bytes(range.base, 48).ptr as *mut u64, 6);
44	+	set words[0] = localReserve;
45	+	set words[1] = frequency;
46	+	set words[2] = source.bits;
47	+	set words[3] = size;
48	+	set words[4] = remoteReserve;
49	+	set words[5] = output;
50	+	set env.argsPointer = range.base;
51	+	set env.argsSize = 48;
52	+	}
53	+
54	+	@default unsafe fn main(hardware: u64, description: *u8) -> u32 {
55	+	let execution = boot::start(hardware, description);
56	+	let hart = hardware as u32;
57	+	if hart == 0 {
58	+	let guard = smp::acquire(&mut execution.lock);
59	+	arguments(execution);
60	+	smp::release(guard);
61	+	}
62	+	runtime::initializeHart(execution, hart);
63	+	loop {
64	+	runtime::step(execution, hart);
65	+	if hart == 0 {
66	+	let guard = smp::acquire(&mut execution.lock);
67	+	let root = &execution.kernel.domains[0];
68	+	let finished = root.state == domains::Lifecycle::Dead;
69	+	if finished {
70	+	assert root.terminalKind == 4 and root.terminalCode == 0;
71	+	}
72	+	smp::release(guard);
73	+	if finished { return 0; }
74	+	}
75	+	}
76	+	}

kernel/native.rad +7 -4

use core::pages;
use core::physical;
use core::platform;
use core::resources;
use core::state;
use images;
use core::images;

mod instances;

/// Isolated consumer used by the native graph regressions.
static DOMAINS: [domains::Domain; 3] = undefined;

static CLAIMS: [bool; frames::MAX_FRAMES] = undefined;
/// Domain-lifetime frame bitmap.
static GRANTS: [u64; 3 * frames::MAX_FRAMES / 64] = undefined;
/// Actual capability and Page mechanisms used by the fixture calls.
static KERNEL: state::State = undefined;
/// Image identities owned by this isolated kernel state.
static IMAGES: images::Registry = undefined;

/// Run one graph against actual Page authority and check its terminal call.
unsafe fn run(image: u32, expected: u32, own: abi::Handle, queued: bool) {
    let descriptor = images::get(image);
    let descriptor = images::get(KERNEL.images, image);
    let stack = try! pages::allocate(&mut KERNEL, 0, own, 1, physical::clear);
    let range = try! pages::access(&KERNEL, 0, stack, abi::READ | abi::WRITE);
    let args = physical::bytes(range.base, 8).ptr as *mut u64;
    set *args = stack.bits if image == 6 else own.bits;
    let private = frames::install(try! frames::allocate(&mut POOL, 16, physical::clear));

    domains::init(&mut DOMAINS[..]);
    resources::init(&mut OBJECTS[..]);
    for i in 0..CONTEXTS.len { contexts::init(&mut CONTEXTS[i], i); }
    let mut ram: memory::Memory = undefined;
    memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut CLAIMS[..], &mut GRANTS[..], 3);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
    let own = domains::root(&mut DOMAINS[..], images::ROOT);
    images::init(&mut IMAGES);
    set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
    let own = domains::root(&mut DOMAINS[..], KERNEL.images.root);
    let expected: [u32; 7] = [42, 21, 73, 77, 0, 42, 81];
    for i in 0..expected.len { run(i, expected[i], own, i == 5); }
    run(5, 42, own, false);
    instances::run(&mut KERNEL, own, machine.clint.base);
    return 0;

kernel/user.rad +1 -0

1	1		//! User bindings for the kernel's capability-checked direct-call ABI.
2	2
3	3		export unsafe mod sys;
	4	+	export mod launch;

kernel/user/launch.rad added +56 -0

1	+	//! Sequential child execution with explicit stack and finite CPU authority.
2	+	use abi;
3	+	use user::sys;
4	+
5	+	/// Consume terminal reports for one child; the caller owns the event consumer.
6	+	fn terminal(env: *sys::Env, child: u16) -> ?sys::Event {
7	+	while let event = sys::eventsPop(env) {
8	+	if (event.kind == 3 or event.kind == 4) and event.value == child as u64 {
9	+	return event;
10	+	}
11	+	}
12	+	return nil;
13	+	}
14	+
15	+	/// Activate one child and resume voluntary yields until its terminal report.
16	+	fn execute(env: *sys::Env, child: abi::Handle, budget: abi::Handle, stack: abi::Handle) -> sys::Event throws (abi::Error) {
17	+	let id = sys::queryDomain(child).id;
18	+	let bytes = try sys::pageSliceMut(stack, 0, 8192);
19	+	for i in 0..bytes.len { set bytes[i] = 0; }
20	+	let _stack = try sys::capabilityGrant(stack, child, (abi::READ \| abi::WRITE) as u64);
21	+	let top = sys::queryPage(stack).base + 8192;
22	+	try sys::domainActivate(child, top, &bytes[..0]);
23	+	let context = sys::queryContext(child, 0).id;
24	+	try sys::budgetBind(budget, child, context);
25	+	loop {
26	+	try sys::contextRun(child, context);
27	+	if let event = terminal(env, id) { return event; }
28	+	let info = sys::queryContext(child, context);
29	+	if info.remaining == 0 { throw abi::Error::Exhausted; }
30	+	while sys::queryContext(child, context).status == 3 {
31	+	try sys::wait();
32	+	if let event = terminal(env, id) { return event; }
33	+	}
34	+	}
35	+	}
36	+
37	+	/// Run a fresh image instance with an empty argument range and an 8 KiB Page stack.
38	+	/// The caller retains stack for reuse. Each invocation consumes at most ticks
39	+	/// from reserve. The image remains admitted after the child terminates.
40	+	export fn run(env: *sys::Env, authority: abi::Handle, reserve: abi::Handle,
41	+	stack: abi::Handle, image: u64, ticks: u64) -> sys::Event throws (abi::Error) {
42	+	let budget = try sys::budgetSplit(reserve, ticks);
43	+	let child = try sys::domainCreate(authority, image) catch error {
44	+	try! sys::capabilityDrop(budget);
45	+	throw error;
46	+	};
47	+	let id = sys::queryDomain(child).id;
48	+	let event = try execute(env, child, budget, stack) catch error {
49	+	try! sys::domainDestroy(child, 0);
50	+	while terminal(env, id) == nil { try! sys::wait(); }
51	+	try! sys::capabilityDrop(budget);
52	+	throw error;
53	+	};
54	+	try sys::capabilityDrop(budget);
55	+	return event;
56	+	}

kernel/user/load_check.rad added +91 -0

1	+	//! Machine checks for runtime admission, rejected input, and image lifetime.
2	+	use abi;
3	+	use user::sys;
4	+	use user::launch;
5	+
6	+	/// Admission must reject malformed or unsupported input without publishing it.
7	+	fn rejected(root: abi::Handle, bytes: *[u8]) {
8	+	let _image = try sys::imageLoad(root, bytes) catch error {
9	+	assert error == abi::Error::VerifyFailed;
10	+	return;
11	+	};
12	+	sys::abort();
13	+	}
14	+
15	+	/// Locate an encoded native declaration without altering the binary layout.
16	+	fn nativeName(bytes: *[u8]) -> u32 {
17	+	let name = "user::sys::rawCall";
18	+	assert bytes.len >= name.len;
19	+	for i in 0..bytes.len - name.len + 1 {
20	+	let mut matches = true;
21	+	for j in 0..name.len {
22	+	if bytes[i + j] <> name[j] { set matches = false; break; }
23	+	}
24	+	if matches { return i + 11; }
25	+	}
26	+	sys::abort();
27	+	return 0;
28	+	}
29	+
30	+	/// Run a fresh instance on another hart and await its terminal report.
31	+	fn remote(env: *sys::Env, root: abi::Handle, reserve: abi::Handle,
32	+	stack: abi::Handle, image: u64, ticks: u64) {
33	+	let child = try! sys::domainCreate(root, image);
34	+	let id = sys::queryDomain(child).id;
35	+	let _page = try! sys::capabilityGrant(stack, child, (abi::READ \| abi::WRITE) as u64);
36	+	let args = try! sys::pageSlice(stack, 0, 0);
37	+	try! sys::domainActivate(child, sys::queryPage(stack).base + 8192, args);
38	+	let context = sys::queryContext(child, 0).id;
39	+	let budget = try! sys::budgetSplit(reserve, ticks);
40	+	try! sys::budgetBind(budget, child, context);
41	+	try! sys::contextRun(child, context);
42	+	loop {
43	+	if let event = sys::eventsPop(env) {
44	+	assert event.kind == 4 and event.code == 42 and event.value == id as u64;
45	+	break;
46	+	}
47	+	try! sys::wait();
48	+	}
49	+	try! sys::capabilityDrop(budget);
50	+	}
51	+
52	+	@default fn main(env: *sys::Env) -> u32 {
53	+	let args = sys::envArgs(env);
54	+	assert args.len == 48;
55	+	let words = @sliceOf(args.ptr as *u64, 6);
56	+	let root = sys::handleSlot(0, abi::Kind::Domain);
57	+	let reserve = sys::handleSlot(words[0], abi::Kind::Budget);
58	+	let ticks = words[1] * 5;
59	+	let input = abi::Handle { bits: words[2] };
60	+	let bytes = try! sys::pageSliceMut(input, 0, words[3] as u32);
61	+	assert bytes.len > 4;
62	+	let stack = try! sys::pageAllocate(root, 2);
63	+	let name = nativeName(bytes);
64	+	let original = bytes[name];
65	+	set bytes[name] = 120;
66	+	rejected(root, bytes);
67	+	set bytes[name] = original;
68	+	rejected(root, &bytes[..bytes.len - 1]);
69	+	let first = try! sys::imageLoad(root, bytes);
70	+	let firstRun = try! launch::run(env, root, reserve, stack, first, ticks);
71	+	assert firstRun.kind == 4 and firstRun.code == 42;
72	+	let second = try! sys::imageLoad(root, bytes);
73	+	assert first <> second;
74	+	// Later compilation and source writes must not change an admitted image.
75	+	for i in 0..bytes.len { set bytes[i] = 0; }
76	+	try! sys::capabilityDrop(input);
77	+	let nextRun = try! launch::run(env, root, reserve, stack, second, ticks);
78	+	assert nextRun.kind == 4 and nextRun.code == 42;
79	+	let independent = try! launch::run(env, root, reserve, stack, first, ticks);
80	+	assert independent.kind == 4 and independent.code == 42;
81	+	if words[4] <> 0 {
82	+	remote(env, root, sys::handleSlot(words[4], abi::Kind::Budget), stack, first, ticks);
83	+	}
84	+	let device = abi::Handle { bits: words[5] };
85	+	let message = "LOADING-OK\n";
86	+	for byte in message {
87	+	while (try! sys::deviceRead8(device, 5)) & 32 == 0 {}
88	+	try! sys::deviceWrite8(device, 0, byte);
89	+	}
90	+	return 0;
91	+	}

kernel/user/loader.rad added +95 -0

1	+	//! Trusted RIL admission over the reference machine UART.
2	+	//! Wait for RIL-READY, send a little-endian u32 length, wait for RIL-DATA,
3	+	//! then send exactly that many bytes. A zero length exits the root domain.
4	+	//! Programs run sequentially with empty arguments and finite CPU budgets.
5	+	use abi;
6	+	use user::sys;
7	+	use user::launch;
8	+
9	+	/// Maximum binary RIL request size.
10	+	constant MAX_IMAGE: u32 = 16 * 1024 * 1024;
11	+
12	+	/// Select the reference UART from the installed boot Device capabilities.
13	+	fn uart(count: u64) -> abi::Handle {
14	+	for i in 0..count {
15	+	let device = sys::handleSlot(i + 2, abi::Kind::Device);
16	+	let info = sys::queryDevice(device);
17	+	if info.base == 0x10000000 and info.size >= 8 { return device; }
18	+	}
19	+	sys::abort();
20	+	return sys::domainSelf();
21	+	}
22	+
23	+	/// Write complete text through individually authorized UART accesses.
24	+	fn print(device: abi::Handle, text: *[u8]) {
25	+	for byte in text {
26	+	while (try! sys::deviceRead8(device, 5)) & 32 == 0 {}
27	+	try! sys::deviceWrite8(device, 0, byte);
28	+	}
29	+	}
30	+
31	+	/// Write a stable eight-digit hexadecimal status or image identifier.
32	+	fn number(device: abi::Handle, value: u32) {
33	+	let digits = "0123456789abcdef";
34	+	let mut bytes: [u8; 8] = undefined;
35	+	for i in 0..8 { set bytes[i] = digits[((value >> ((7 - i) * 4)) & 15)]; }
36	+	print(device, &bytes[..]);
37	+	}
38	+
39	+	/// Receive one byte, blocking only after draining the UART and its IRQ event.
40	+	fn receive(env: *sys::Env, device: abi::Handle) -> u8 throws (abi::Error) {
41	+	loop {
42	+	let status = try sys::deviceRead8(device, 5);
43	+	if status & 2 <> 0 { throw abi::Error::InvalidArg; }
44	+	if status & 1 <> 0 { return try sys::deviceRead8(device, 0); }
45	+	while let _event = sys::eventsPop(env) {}
46	+	// Recheck after consuming an IRQ so a newly arrived byte cannot be lost.
47	+	let ready = try sys::deviceRead8(device, 5);
48	+	if ready & 2 <> 0 { throw abi::Error::InvalidArg; }
49	+	if ready & 1 == 0 { try sys::wait(); }
50	+	}
51	+	}
52	+
53	+	/// Read a request length without alignment or host-endian assumptions.
54	+	fn length(env: *sys::Env, device: abi::Handle) -> u32 throws (abi::Error) {
55	+	let mut size: u32 = 0;
56	+	for i in 0..4 { set size \|= (try receive(env, device)) as u32 << (i * 8); }
57	+	return size;
58	+	}
59	+
60	+	@default fn main(env: *sys::Env) -> u32 {
61	+	let args = sys::envArgs(env);
62	+	assert args.len >= 80 and args.ptr as u64 & 7 == 0;
63	+	let words = @sliceOf(args.ptr as *u64, args.len / 8);
64	+	assert words[0] == 1 and words[3] > 0 and words[7] == 1;
65	+	let root = sys::handleSlot(0, abi::Kind::Domain);
66	+	let reserve = sys::handleSlot(words[9], abi::Kind::Budget);
67	+	let ticks = words[6] * 5;
68	+	let device = uart(words[1]);
69	+	let input = try! sys::pageAllocate(root, MAX_IMAGE as u64 / 4096);
70	+	let bytes = try! sys::pageSliceMut(input, 0, MAX_IMAGE);
71	+	let stack = try! sys::pageAllocate(root, 2);
72	+	try! sys::deviceWrite8(device, 2, 7);
73	+	try! sys::deviceWrite8(device, 1, 1);
74	+	loop {
75	+	print(device, "RIL-READY\n");
76	+	let size = try length(env, device) catch { print(device, "RIL-TRANSPORT-ERROR\n"); return 1; };
77	+	if size == 0 { return 0; }
78	+	if size > MAX_IMAGE { print(device, "RIL-SIZE-ERROR\n"); return 1; }
79	+	print(device, "RIL-DATA\n");
80	+	for i in 0..size {
81	+	set bytes[i] = try receive(env, device) catch { print(device, "RIL-TRANSPORT-ERROR\n"); return 1; };
82	+	}
83	+	let image = try sys::imageLoad(root, &bytes[..size]) catch error {
84	+	print(device, "RIL-ERROR "); number(device, error as u32); print(device, "\n");
85	+	continue;
86	+	};
87	+	print(device, "RIL-IMAGE "); number(device, image as u32); print(device, "\n");
88	+	let event = try launch::run(env, root, reserve, stack, image, ticks) catch error {
89	+	print(device, "RIL-RUN-ERROR "); number(device, error as u32); print(device, "\n");
90	+	continue;
91	+	};
92	+	print(device, "RIL-TERMINAL "); number(device, event.kind as u32);
93	+	print(device, " "); number(device, event.code); print(device, "\n");
94	+	}
95	+	}

kernel/user/runtime.rad added +26 -0

1	+	//! Runtime image state, repeated function pointers, and native syscall bindings.
2	+	use user::sys;
3	+
4	+	/// Per-domain counter copied from the immutable image initializer.
5	+	static COUNT: u32 = 41;
6	+
7	+	/// Update the counter and read it through its instance-relative address.
8	+	fn increment(value: *u32) -> u32 {
9	+	set COUNT += 1;
10	+	return *value;
11	+	}
12	+
13	+	/// Repeated native function pointers in private writable state.
14	+	static LOCAL: [fn(*u32) -> u32; 2] = [increment; 2];
15	+	/// Repeated native function pointers in immutable shared data.
16	+	constant SHARED: [fn(*u32) -> u32; 2] = [increment; 2];
17	+
18	+	@default fn main(env: *sys::Env) -> u32 {
19	+	let result = LOCAL[0](&COUNT);
20	+	assert SHARED[1](&COUNT) == 43;
21	+	try! sys::timeout(1, 0x52494c);
22	+	try! sys::wait();
23	+	let event = sys::eventsPop(env) else { sys::abort(); return 1; };
24	+	assert event.kind == 2 and event.code == 0x52494c;
25	+	return result;
26	+	}

kernel/user/sys.rad +13 -3

//! Capability-checked user calls and views of kernel-authorized memory.
//! The image catalog links the native primitives in user/sys.ras.
//! Native primitives in user/sys.ras are the runtime image binding boundary.

use abi;

/// Startup ABI: exactly the ten u64 fields of core::domains::Env, in order.
/// Each entry receives its own immutable Env pointer in a0. Retain that pointer

/// Drop only this local capability, not another domain's copy or its object.
export fn capabilityDrop(handle: abi::Handle) throws (abi::Error) {
    try check(invoke(12, handle.bits, 0, 0, 0).error);
}

/// Create a pending domain using explicit Create authority and a catalog index.
/// image is a u64 image-catalog index, not a handle or a source-code address.
/// Create a pending domain using explicit Create authority and an admitted ID.
/// image is a stable u64 image ID, not a handle or a source-code address.
export fn domainCreate(authority: abi::Handle, image: u64) -> abi::Handle throws (abi::Error) {
    let reply = invoke(20, authority.bits, image, 0, 0);
    try check(reply.error);
    return abi::Handle { bits: reply.values[0] };
}

/// Admit trusted binary RIL using caller-self Create and Allocate authority.
/// Every byte must have live readable Page coverage. The kernel snapshots and
/// compiles the input; the returned stable image ID outlives all domain instances.
/// This admission does not provide a safety or provenance verification proof.
export fn imageLoad(authority: abi::Handle, ril: *[u8]) -> u64 throws (abi::Error) {
    let reply = invoke(77, authority.bits, ril.ptr as u64, ril.len as u64, 0);
    try check(reply.error);
    return reply.values[0];
}

/// Activate at the admitted entry; args is the exact readable byte range.
/// The kernel checks Page authority and stack ownership. Pass args.ptr, NEVER
/// the address of its fat-slice descriptor. Subslice first to pass fewer bytes.
export fn domainActivate(domain: abi::Handle, stackTop: u64, args: *[u8]) throws (abi::Error) {
    try check(invoke(21, domain.bits, stackTop, args.ptr as u64, args.len as u64).error);

lib/std/lang/il.rad +1 -0

// TODO: Labels should have their own type.
// TODO: Blocks should have an instruction in `Instr`.

export mod printer;
export mod binary;
export mod images;

use std::mem;
use std::lang::alloc;

/// Source location for debug info.

lib/std/lang/il/images.rad added +202 -0

1	+	//! Freestanding compilation and placement of trusted binary RIL images.
2	+	//!
3	+	//! Structural and backend-capacity checks do not establish memory safety,
4	+	//! pointer provenance, or authority confinement of generated native code.
5	+
6	+	export mod graph;
7	+	export mod native;
8	+	mod layout;
9	+
10	+	use std::mem;
11	+	use std::collections::dict;
12	+	use std::lang::alloc;
13	+	use std::lang::strings;
14	+	use std::lang::il;
15	+	use std::arch::rv64;
16	+	use std::arch::rv64::emit;
17	+
18	+	/// One explicitly permitted native function in the trusted execution environment.
19	+	export record Binding: Copy {
20	+	/// Exact qualified external declaration name.
21	+	name: *[u8],
22	+	/// Callable native address using the image's register and stack ABI.
23	+	address: u64,
24	+	}
25	+
26	+	/// Immutable native image entry and separately instantiated private state.
27	+	export record Descriptor: Copy {
28	+	/// Native entry receiving the domain Env pointer in a0.
29	+	entry: u64,
30	+	/// Read-only bytes copied into each domain's private allocation.
31	+	initial: u64,
32	+	/// Number of private state bytes required by one domain.
33	+	stateSize: u32,
34	+	/// Power-of-two alignment required by the private allocation.
35	+	stateAlignment: u32,
36	+	/// Repair private pointers after copying the initializer to its new base.
37	+	relocate: fn(u64),
38	+	}
39	+
40	+	/// Prepared native output borrowing caller-owned compiler storage until finish.
41	+	/// Its metadata must not be mutated by the caller; only size and alignment are
42	+	/// needed to obtain a destination allocation. No compiler storage escapes finish.
43	+	export record Prepared: Copy {
44	+	/// Minimum destination capacity in bytes.
45	+	size: u32,
46	+	/// Required power-of-two destination base alignment.
47	+	alignment: u32,
48	+	/// Final position-independent native instruction words.
49	+	code: *[u32],
50	+	/// Immutable initializer bytes containing allocation-relative pointers.
51	+	data: *[u8],
52	+	/// Flattened initializer declarations used to identify pointer slots.
53	+	items: *[il::Data],
54	+	/// Allocation-relative byte offset of native code.
55	+	codeOffset: u32,
56	+	/// Allocation-relative byte offset of the trusted entry wrapper.
57	+	entryOffset: u32,
58	+	/// Allocation-relative byte offset of the private initializer.
59	+	initialOffset: u32,
60	+	/// Allocation-relative byte offset of the private pointer repair helper.
61	+	relocateOffset: u32,
62	+	/// Number of separately allocated private state bytes.
63	+	stateSize: u32,
64	+	/// Alignment required by separately allocated private state.
65	+	stateAlignment: u32,
66	+	/// Immutable source range excluded from destination placement.
67	+	source: *[u8],
68	+	/// Persistent compiler arena range excluded from destination placement.
69	+	arena: *[u8],
70	+	/// Temporary compiler arena range excluded from destination placement.
71	+	scratch: *[u8],
72	+	/// Caller-owned interning pool range excluded from destination placement.
73	+	pool: *[u8],
74	+	}
75	+
76	+	/// Reject wrapping memory ranges before overlap or placement arithmetic.
77	+	fn range(bytes: *[u8]) throws (il::binary::Error) {
78	+	let base = bytes.ptr as u64;
79	+	if bytes.len as u64 > 0xFFFFFFFFFFFFFFFF - base {
80	+	throw il::binary::error(0, "image memory range wraps address space");
81	+	}
82	+	}
83	+
84	+	/// Detect overlap between already checked byte ranges without pointer ordering.
85	+	fn overlap(a: [u8], b: [u8]) -> bool {
86	+	if a.len == 0 or b.len == 0 { return false; }
87	+	let first = a.ptr as u64;
88	+	let second = b.ptr as u64;
89	+	return first < second + b.len as u64 and second < first + a.len as u64;
90	+	}
91	+
92	+	/// Require aligned, disjoint compiler storage and a caller-initialized empty pool.
93	+	fn storage(source: [u8], arena: alloc::Arena, scratch: alloc::Arena, pool: strings::Pool) throws (il::binary::Error) {
94	+	try range(source); try range(arena.data); try range(scratch.data);
95	+	let poolBytes = @sliceOf(pool as opaque as u8, @sizeOf(strings::Pool));
96	+	try range(poolBytes);
97	+	if arena.offset > arena.data.len or scratch.offset > scratch.data.len
98	+	or arena.data.len > 0x7FFFFFFF or scratch.data.len > 0x7FFFFFFF
99	+	or arena.data.ptr as u64 % 8 <> 0 or scratch.data.ptr as u64 % 8 <> 0 {
100	+	throw il::binary::error(0, "invalid image compiler arena");
101	+	}
102	+	if overlap(arena.data, scratch.data) or overlap(source, arena.data) or overlap(source, scratch.data)
103	+	or overlap(poolBytes, arena.data) or overlap(poolBytes, scratch.data) or overlap(poolBytes, source) {
104	+	throw il::binary::error(0, "image compiler storage overlaps");
105	+	}
106	+	if pool.count <> 0 { throw il::binary::error(0, "image compiler requires an empty string pool"); }
107	+	for i in 0..pool.table.len {
108	+	if pool.table[i].len <> 0 { throw il::binary::error(0, "image compiler requires an empty string pool"); }
109	+	}
110	+	}
111	+
112	+	/// Decode and compile a closed trusted image using the shared RV64 backend.
113	+	/// Source stays immutable through finish. Arena, scratch, and empty pool must
114	+	/// be disjoint and live through finish; arenas require eight-byte aligned bases.
115	+	/// The emitter reserves an eight-MiB code buffer plus fixed symbol/relocation
116	+	/// tables in arena. Remaining arena use depends on names and initialized data;
117	+	/// scratch holds the decoded graph and one function's register allocation at a
118	+	/// time. Every exhaustion is reported as a binary Error. Failed preparation may
119	+	/// consume both arenas and pool; reclaim them together before the next attempt.
120	+	export fn prepare(source: [u8], bindings: [Binding], arena: *mut alloc::Arena,
121	+	scratch: mut alloc::Arena, pool: mut strings::Pool) -> Prepared throws (il::binary::Error) {
122	+	if source.len > 16 * 1024 * 1024 { throw il::binary::error(0, "binary image exceeds capacity"); }
123	+	try storage(source, arena, scratch, pool);
124	+	if bindings.len > 4096 { throw il::binary::error(0, "too many native image bindings"); }
125	+	let mut bindingIndex = try il::binary::dictionary(scratch, bindings.len, 0);
126	+	for binding, i in bindings {
127	+	if binding.name.len <= 11 or not mem::eq(&binding.name[..11], "user::sys::")
128	+	or binding.address == 0 or binding.address % 4 <> 0 {
129	+	throw il::binary::error(0, "invalid native image binding");
130	+	}
131	+	if dict::get(&bindingIndex, binding.name) <> nil { throw il::binary::error(0, "duplicate native image binding"); }
132	+	dict::insert(&mut bindingIndex, binding.name, i as i32);
133	+	}
134	+	let image = try il::binary::decode(source, scratch, pool);
135	+	let plan = try graph::plan(&image, scratch);
136	+	let names = try graph::names(&image.program, 0, pool, arena);
137	+	let stateName = try graph::name("images::state_", 0, "", pool, arena);
138	+	let initial = try native::initializer(&plan, &names, stateName, arena, scratch);
139	+	let emitter = try emit::emitter(arena, false) catch { throw il::binary::error(0, "native image emitter arena exhausted"); };
140	+	let mut generator = rv64::Generator { e: emitter, entryPatch: rv64::EntryPatch::None };
141	+	for function, i in image.program.fns {
142	+	if not function.isExtern { continue; }
143	+	if function.params.len > rv64::ARG_REGS.len { throw il::binary::error(0, "image native declaration has too many arguments"); }
144	+	let binding = dict::get(&bindingIndex, function.name) else {
145	+	throw il::binary::error(0, "unresolved image sys native declaration");
146	+	};
147	+	try native::trampoline(&mut generator, names.functions[i], bindings[binding as u32].address);
148	+	}
149	+	let mut offsets = try il::binary::dictionary(scratch, plan.order.len, 0);
150	+	for item, i in plan.items {
151	+	if not item.readOnly { dict::insert(&mut offsets, names.data[i], plan.offsets[i] as i32); }
152	+	}
153	+	set generator.e.instanceData = &offsets;
154	+	for function, i in image.program.fns {
155	+	if function.isExtern { continue; }
156	+	let renamed = try graph::function(function, i, &plan, &names, scratch);
157	+	try native::function(&mut generator, &renamed, scratch);
158	+	}
159	+	set generator.e.instanceData = nil;
160	+	try native::helpers(&mut generator, 0, &plan, &names, &initial, pool, arena);
161	+	let placed = try layout::prepare(&mut generator, initial.items, arena, scratch);
162	+	let entryName = try graph::name("images::native_", 0, "", pool, arena);
163	+	let relocateName = try graph::name("images::relocate_", 0, "", pool, arena);
164	+	let entryOffset = placed.codeOffset + (try layout::functionOffset(&generator.e, entryName));
165	+	let relocateOffset = placed.codeOffset + (try layout::functionOffset(&generator.e, relocateName));
166	+	let initialOffset = dict::get(&placed.symbols.dict, stateName) else {
167	+	throw il::binary::error(0, "unresolved native image initializer");
168	+	};
169	+	return Prepared { size: placed.size, alignment: placed.alignment, code: emit::getCode(&generator.e),
170	+	data: placed.bytes, items: initial.items, codeOffset: placed.codeOffset, entryOffset,
171	+	initialOffset: initialOffset as u32, relocateOffset, stateSize: plan.size, stateAlignment: plan.alignment,
172	+	source, arena: arena.data, scratch: scratch.data, pool: @sliceOf(pool as opaque as u8, @sizeOf(strings::Pool)) };
173	+	}
174	+
175	+	/// Copy prepared native bytes and relocate persistent pointers at destination.
176	+	/// Destination must be aligned, large enough, and disjoint from compiler storage
177	+	/// and source. No allocation or compilation occurs here. The returned descriptor
178	+	/// borrows only destination, which must remain immutable for its execution life.
179	+	/// The caller must synchronize instruction fetch before any hart enters the code.
180	+	export fn finish(prepared: mut Prepared, destination: mut [u8]) -> Descriptor throws (il::binary::Error) {
181	+	try range(destination);
182	+	let base = destination.ptr as u64;
183	+	if base == 0 or prepared.alignment == 0 or prepared.alignment & (prepared.alignment - 1) <> 0
184	+	or base % prepared.alignment as u64 <> 0 or destination.len < prepared.size {
185	+	throw il::binary::error(0, "invalid native image destination");
186	+	}
187	+	let output = &mut destination[..prepared.size];
188	+	if overlap(output, prepared.source) or overlap(output, prepared.arena)
189	+	or overlap(output, prepared.scratch) or overlap(output, prepared.pool) {
190	+	throw il::binary::error(0, "native image destination overlaps compiler storage");
191	+	}
192	+	try! mem::copy(output, prepared.data);
193	+	for i in prepared.data.len..prepared.codeOffset { set output[i] = 0; }
194	+	let code = @sliceOf(prepared.code.ptr as u8, prepared.code.len rv64::INSTR_SIZE as u32);
195	+	try! mem::copy(&mut output[prepared.codeOffset..], code);
196	+	layout::rebase(prepared.items, output, base);
197	+	let relocateAddress = base + prepared.relocateOffset as u64;
198	+	let relocate = (&relocateAddress as opaque as *fn(u64));
199	+	return Descriptor { entry: base + prepared.entryOffset as u64, initial: base + prepared.initialOffset as u64,
200	+	stateSize: prepared.stateSize, stateAlignment: prepared.stateAlignment,
201	+	relocate };
202	+	}

compiler/radiance/images/graph.rad → lib/std/lang/il/images/graph.rad renamed +0 -0

lib/std/lang/il/images/layout.rad added +161 -0

1	+	//! Position-independent native layout and persistent pointer initialization.
2	+
3	+	use std::mem;
4	+	use std::collections::dict;
5	+	use std::lang::alloc;
6	+	use std::lang::il;
7	+	use std::lang::gen::data;
8	+	use std::arch::rv64;
9	+	use std::arch::rv64::emit;
10	+	use std::arch::rv64::encode;
11	+	use super::graph;
12	+
13	+	/// Native bytes and offsets, with data addresses relative to the allocation.
14	+	export record Layout: Copy {
15	+	/// Fully initialized read-only bytes with allocation-relative pointers.
16	+	bytes: *[u8],
17	+	/// Allocation-relative immutable symbol addresses.
18	+	symbols: data::DataSymMap,
19	+	/// Byte offset of the first native instruction.
20	+	codeOffset: u32,
21	+	/// Complete allocation size including native instructions.
22	+	size: u32,
23	+	/// Required power-of-two allocation alignment.
24	+	alignment: u32,
25	+	}
26	+
27	+	/// Resolve a function before invoking backend relocation routines.
28	+	export fn functionOffset(e: emit::Emitter, name: [u8]) -> u32 throws (il::binary::Error) {
29	+	let offset = dict::get(&e.labels.funcs, name) else {
30	+	throw il::binary::error(0, "unresolved native image function");
31	+	};
32	+	if offset < 0 or offset as u32 % 4 <> 0 or offset as u32 / 4 >= e.codeLen {
33	+	throw il::binary::error(0, "native image function offset exceeds code");
34	+	}
35	+	return offset as u32;
36	+	}
37	+
38	+	/// Require every reserved relocation instruction to belong to emitted code.
39	+	fn slots(e: *emit::Emitter, index: u32, count: u32) throws (il::binary::Error) {
40	+	if index > e.codeLen or count > e.codeLen - index {
41	+	throw il::binary::error(0, "native image relocation exceeds code");
42	+	}
43	+	}
44	+
45	+	/// Check all references before the shared emitter's infallible final patching.
46	+	fn references(e: emit::Emitter, items: [il::Data], symbols: *data::DataSymMap) throws (il::binary::Error) {
47	+	if e.pendingBranches.len <> 0 { throw il::binary::error(0, "unresolved native image block branch"); }
48	+	for pending in e.pendingCalls {
49	+	try slots(e, pending.index, 2);
50	+	let _ = try functionOffset(e, pending.target);
51	+	}
52	+	for pending in e.pendingJumps {
53	+	try slots(e, pending.index, 1);
54	+	let offset = try functionOffset(e, pending.target);
55	+	if not encode::isJumpImm(offset as i32 - pending.index as i32 * rv64::INSTR_SIZE) {
56	+	throw il::binary::error(0, "native image jump exceeds reach");
57	+	}
58	+	}
59	+	for pending in e.pendingAddrLoads {
60	+	try slots(e, pending.index, 2);
61	+	if pending.isData {
62	+	if data::lookupAddr(symbols, pending.target) == nil {
63	+	throw il::binary::error(0, "unresolved native image data address");
64	+	}
65	+	} else { let _ = try functionOffset(e, pending.target); }
66	+	}
67	+	for item in items {
68	+	for value in item.values {
69	+	if value.count == 0 { continue; }
70	+	match value.item {
71	+	case il::DataItem::Sym(name) => {
72	+	if data::lookupAddr(symbols, name) == nil {
73	+	throw il::binary::error(0, "unresolved native image data initializer");
74	+	}
75	+	}
76	+	case il::DataItem::Fn(name) => { let _ = try functionOffset(e, name); }
77	+	else => {},
78	+	}
79	+	}
80	+	}
81	+	}
82	+
83	+	/// Lay out flattened immutable data using the shared data placement and emitter.
84	+	/// Code and data address instructions use allocation-local PC-relative offsets.
85	+	export fn prepare(generator: mut rv64::Generator, items: [il::Data], arena: *mut alloc::Arena,
86	+	scratch: *mut alloc::Arena) -> Layout throws (il::binary::Error) {
87	+	let mut alignment: u32 = 8;
88	+	let mut extent: u64 = 0;
89	+	for item in items {
90	+	let a = item.alignment as u64;
91	+	if not item.readOnly or item.isZeroInit or a == 0 or a & (a - 1) <> 0 or a > graph::MAX_DATA as u64 {
92	+	throw il::binary::error(0, "invalid flattened native image data");
93	+	}
94	+	if item.alignment > alignment { set alignment = item.alignment; }
95	+	set extent = ((extent + a - 1) & ~(a - 1)) + item.size as u64;
96	+	if extent > graph::MAX_DATA as u64 { throw il::binary::error(0, "native image data exceeds capacity"); }
97	+	let mut size: u64 = 0;
98	+	for value in item.values {
99	+	set size += graph::width(value.item) as u64 * value.count as u64;
100	+	if size > item.size as u64 { throw il::binary::error(0, "native image initializer exceeds data"); }
101	+	}
102	+	if size <> item.size as u64 { throw il::binary::error(0, "native image initializer size mismatch"); }
103	+	}
104	+	let syms = try graph::storage(scratch, @sizeOf(data::DataSym), @alignOf(data::DataSym), items.len) as *mut [data::DataSym];
105	+	let mut count: u32 = 0;
106	+	let dataSize = data::layoutSection(items, syms, &mut count, 0, true);
107	+	let mut map = try il::binary::dictionary(scratch, count, 0);
108	+	for symbol in syms {
109	+	if dict::get(&map, symbol.name) <> nil { throw il::binary::error(0, "duplicate native image data symbol"); }
110	+	dict::insert(&mut map, symbol.name, symbol.addr as i32);
111	+	}
112	+	let symbols = data::DataSymMap { dict: map, syms };
113	+	let e = &mut generator.e;
114	+	let codeOffset = mem::alignUp(dataSize, 8);
115	+	let total = codeOffset as u64 + e.codeLen as u64 * rv64::INSTR_SIZE as u64;
116	+	// Keep every allocation-local AUIPC displacement strictly inside signed reach.
117	+	if total > 0x7FFFF000 { throw il::binary::error(0, "native image exceeds relative address reach"); }
118	+	try references(e, items, &symbols);
119	+	let bytes = try graph::storage(arena, 1, 1, dataSize) as *mut [u8];
120	+	for i in 0..bytes.len { set bytes[i] = 0; }
121	+	emit::patchCalls(e);
122	+	emit::patchJumps(e);
123	+	for pending in e.pendingAddrLoads {
124	+	let mut target: u32 = 0;
125	+	if pending.isData {
126	+	let address = data::lookupAddr(&symbols, pending.target) else {
127	+	throw il::binary::error(0, "unresolved native image data address");
128	+	};
129	+	set target = address;
130	+	} else { set target = codeOffset + (try functionOffset(e, pending.target)); }
131	+	let pc = codeOffset + pending.index * rv64::INSTR_SIZE as u32;
132	+	let split = emit::splitImm(target as i32 - pc as i32);
133	+	emit::patch(e, pending.index, encode::auipc(pending.rd, split.hi));
134	+	emit::patch(e, pending.index + 1, encode::addi(pending.rd, pending.rd, split.lo));
135	+	}
136	+	data::emitSection(items, &symbols, &e.labels, codeOffset, bytes, true);
137	+	return Layout { bytes, symbols, codeOffset, size: total as u32, alignment };
138	+	}
139	+
140	+	/// Convert allocation-relative initializer pointers to actual native addresses.
141	+	/// Scalar private offsets remain unchanged for the per-instance relocation helper.
142	+	export fn rebase(items: [il::Data], destination: mut [u8], base: u64) {
143	+	let mut offset: u32 = 0;
144	+	for item in items {
145	+	set offset = mem::alignUp(offset, item.alignment);
146	+	for value in item.values {
147	+	match value.item {
148	+	case il::DataItem::Sym(_), il::DataItem::Fn(_) => {
149	+	for _ in 0..value.count {
150	+	let mut relative: u64 = 0;
151	+	for byte in 0..8 { set relative \|= (destination[offset + byte] as u64) << (byte as u64 * 8); }
152	+	let address = base + relative;
153	+	for byte in 0..8 { set destination[offset + byte] = (address >> (byte as u64 * 8)) as u8; }
154	+	set offset += 8;
155	+	}
156	+	}
157	+	else => set offset += graph::width(value.item) * value.count,
158	+	}
159	+	}
160	+	}
161	+	}

compiler/radiance/images/native.rad → lib/std/lang/il/images/native.rad renamed +23 -4

        }
        case il::DataItem::Fn(symbol) => set item = il::DataItem::Fn(names.functions[try graph::index(&plan.fnIndex, symbol)]),
        case il::DataItem::Str(bytes) => set item = il::DataItem::Str(try graph::copy(arena, bytes)),
        else => {},
    }
    return il::DataValue { item, count: value.count };
    return il::DataValue { item, count: 0 if graph::width(item) == 0 else value.count };
}

/// Flatten private declarations into one RO initializer using their native layout.
export fn initializer(plan: *graph::Plan, names: *graph::Names, stateName: *[u8], arena: *mut alloc::Arena, scratch: *mut alloc::Arena) -> Initializer throws (il::binary::Error) {
    let mut shared: u32 = 0;

    for item in items { slice.append(item, allocator); }
    set *target = slice;
}

/// Check hard backend limits and a conservative expansion bound for one function.
fn bounds(e: *emit::Emitter, function: *il::Fn) throws (il::binary::Error) {
fn bounds(e: *emit::Emitter, function: *il::Fn) -> u32 throws (il::binary::Error) {
    if function.params.len > rv64::ARG_REGS.len { throw il::binary::error(0, "image function has too many arguments"); }
    let mut blocks = function.blocks.len as u64 + 1;
    let mut words: u64 = 256;
    let mut reserve: u64 = 0;
    let mut operands: u64 = 0;

        for item in block.instrs {
            set words += 64;
            set operands += 5;
            set branches += 2;
            match item {
                case il::Instr::Call { args, .. } => {
                case il::Instr::Call { func, args, .. } => {
                    match func {
                        case il::Val::FnAddr(_), il::Val::Reg(_) => {},
                        else => throw il::binary::error(0, "invalid image native call target"),
                    }
                    if args.len > rv64::ARG_REGS.len { throw il::binary::error(0, "image call has too many arguments"); }
                    set words += args.len as u64 * 32;
                    set operands += args.len as u64;
                    set calls += 1;
                }

    if e.funcs.len >= e.funcs.cap or operands + e.pendingAddrLoads.len as u64 > e.pendingAddrLoads.cap as u64
        or calls + e.pendingCalls.len as u64 > e.pendingCalls.cap as u64
        or branches + e.pendingBranches.len as u64 > e.pendingBranches.cap as u64 {
        throw il::binary::error(0, "native image relocation capacity exceeded");
    }
    return reserve as u32;
}

/// Shared liveness state used to bound the spill-candidate buffer.
record Pressure: Copy {
    /// Registers live at the inspected instruction.

    return regalloc::AllocResult { assignments: assignment.assignments, spill, usedCalleeSaved: assignment.usedCalleeSaved };
}

/// Emit through the existing allocator and selector, propagating scratch exhaustion.
export fn function(generator: *mut rv64::Generator, function: *il::Fn, scratch: *mut alloc::Arena) throws (il::binary::Error) {
    try bounds(&generator.e, function);
    let reserve = try bounds(&generator.e, function);
    let saved = alloc::save(scratch);
    let allocation = try allocate(function, scratch) catch error {
        alloc::restore(scratch, saved);
        throw error;
    };
    let frame = emit::computeFrame(allocation.spill.frameSize + reserve as i32, allocation.usedCalleeSaved,
        function.blocks.len, function.isLeaf, false);
    if frame.totalSize > 64 * 1024 {
        alloc::restore(scratch, saved);
        throw il::binary::error(0, "image function frame exceeds native stack");
    }
    rv64::isel::selectFn(&mut generator.e, &allocation, function);
    alloc::restore(scratch, saved);
}

/// Reserve a trusted leaf helper in the same global symbol and relocation tables.

    }
    emit::recordFunc(e, name);
    emit::recordFuncOffset(e, name);
}

/// Tail-call one explicitly bound primitive without limiting its physical address.
export fn trampoline(generator: *mut rv64::Generator, name: *[u8], address: u64) throws (il::binary::Error) {
    let e = &mut generator.e;
    try helper(e, name, 9);
    emit::loadImm(e, rv64::SCRATCH1, address as i64);
    emit::emit(e, encode::jalr(rv64::ZERO, rv64::SCRATCH1, 0));
}

/// Build trusted startup, address getters, and private-pointer relocation routines.
export fn helpers(generator: *mut rv64::Generator, id: u32, plan: *graph::Plan, names: *graph::Names,
    initial: *Initializer, pool: *mut strings::Pool, arena: *mut alloc::Arena) throws (il::binary::Error) {
    let stateName = initial.items[0].name;
    let entry = try graph::name("images::native_", id, "", pool, arena);

std.lib +4 -0

lib/std/lang/il/printer.rad
lib/std/lang/il/binary.rad
lib/std/lang/il/binary/instructions.rad
lib/std/lang/il/binary/records.rad
lib/std/lang/il/binary/structure.rad
lib/std/lang/il/images.rad
lib/std/lang/il/images/graph.rad
lib/std/lang/il/images/native.rad
lib/std/lang/il/images/layout.rad
lib/std/lang/resolver.rad
lib/std/lang/resolver/printer.rad
lib/std/lang/lower.rad
lib/std/lang/module.rad
lib/std/lang/module/printer.rad

1	1		//! Binary RIL image catalogs and native image append through the shared backend.
2	2
3		-	mod graph;
4		-	mod native;
	3	+	use std::lang::il::images::graph;
	4	+	use std::lang::il::images::native;
5	5
6	6		use std::mem;
7	7		use std::fmt;
8	8		use std::sys::unix;
9	9		use std::collections::dict;

2	2		EMU ?= $(or $(RAD_EMULATOR),emulator)
3	3		HOST_EMU ?= $(EMU)
4	4		COMPILER := ../bin/radiance.rv64.dev
5	5		COMPILE := $(HOST_EMU) -memory-size=385024 -data-size=348160 -stack-size=512 -run $(COMPILER)
6	6		MODULES := $(wildcard core/*.rad)
7		-	CORE_ASM := arch/atomic.ras arch/context.ras arch/clock.ras arch/mmio.ras arch/physical.ras arch/smp.ras
8		-	CORE := -pkg core -mod core.rad $(addprefix -mod ,$(MODULES) $(CORE_ASM))
	7	+	STD_FILES := $(addprefix ../,$(shell cat ../std.lib))
	8	+	STD := -pkg std $(addprefix -mod ,$(STD_FILES))
	9	+	CORE_ASM := arch/atomic.ras arch/context.ras arch/clock.ras arch/mmio.ras arch/physical.ras arch/smp.ras arch/loading.ras
	10	+	CORE := $(STD) -pkg core -mod core.rad $(addprefix -mod ,$(MODULES) $(CORE_ASM))
9	11		CHECK_MODULES := $(wildcard check/*.rad)
10	12		USER_INPUTS := user.rad $(wildcard user/.rad user//*.rad) core/abi.rad
11		-	USER_BASE := -pkg abi -mod core/abi.rad -pkg user -mod user.rad -mod user/sys.rad -pkg probe -mod user/probe.rad
	13	+	USER_BASE := -pkg abi -mod core/abi.rad -pkg user -mod user.rad -mod user/sys.rad -mod user/launch.rad -pkg probe -mod user/probe.rad
12	14		BASE_IMAGES := sample_root sample_control sample_scalars sample_memory sample_overflow sample_events sample_alias sample_instance
13	15		CONTROL_IMAGES := control flow exiting faulting spinning admin remote
14	16		BASE_RIL := $(addprefix build/,$(addsuffix .ril,$(BASE_IMAGES)))
15	17		CONTROL_RIL := $(addprefix build/,$(addsuffix .ril,$(CONTROL_IMAGES)))
16	18		IMAGE_MODULES_sample_control := user/sample_control/math.rad
17	19		IMAGE_MODULES_control := user/control/lifecycle.rad user/control/authority.rad
	20	+	ROOT_CATALOG := -zero-bss -pkg images -mod build/root/images.rad -mod user/sys.ras -image build/loader.ril
	21	+	LOADING_CATALOG := -zero-bss -pkg images -mod build/loading/images.rad -mod user/sys.ras -image build/load_check.ril
18	22		BASE_CATALOG := -zero-bss -pkg images -mod build/baseline/images.rad -mod user/sys.ras $(addprefix -image ,$(BASE_RIL))
19	23		CONTROL_CATALOG := -zero-bss -pkg images -mod build/control/images.rad -mod user/sys.ras $(addprefix -image ,$(CONTROL_RIL))
20		-	MACHINE := $(EMU) -machine -no-guard-stack -max-steps=1000000000 -count-instructions
	24	+	MACHINE := $(EMU) -machine -memory-size=196608 -no-guard-stack -max-steps=1000000000 -count-instructions
21	25
22	26		.PHONY: all check clean compiler-check
23	27		.DELETE_ON_ERROR:
24	28		all: kernel.rv64
25	29

40	44
41	45		build/control/images.rad: $(CONTROL_RIL) $(COMPILER)
42	46		mkdir -p $(@D)
43	47		$(COMPILE) -catalog $@ $(addprefix -image ,$(CONTROL_RIL))
44	48
45		-	kernel.rv64: main.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(COMPILER)
46		-	$(COMPILE) $(CONTROL_CATALOG) $(CORE) -pkg kernel -start arch/start.ras -mod main.rad -entry kernel -o $@
	49	+	build/root/images.rad: build/loader.ril $(COMPILER)
	50	+	mkdir -p $(@D)
	51	+	$(COMPILE) -catalog $@ -image $<
	52	+
	53	+	build/loading/images.rad: build/load_check.ril $(COMPILER)
	54	+	mkdir -p $(@D)
	55	+	$(COMPILE) -catalog $@ -image $<
47	56
48		-	check.rv64: check.rad core.rad $(MODULES) $(CORE_ASM) $(CHECK_MODULES) build/baseline/images.rad user/sys.ras $(COMPILER)
	57	+	# Keep the admission test input as raw RIL bytes.
	58	+	build/loading/input.ras: build/runtime.ril
	59	+	mkdir -p $(@D)
	60	+	{ printf '.text;\n.export @"loading::input";\n@"loading::input"\nla %%a0 @input;\nret;\n'; \
	61	+	printf '.export @"loading::inputSize";\n@"loading::inputSize"\nli %%a0 %s;\nret;\n.data;\n@input\n' "$$(wc -c < $<)"; \
	62	+	od -An -v -tu1 $< \| sed -e 's/^ //' -e 's/ /, /g' -e 's/^/.byte /' -e 's/$$/;/'; \
	63	+	} > $@
	64	+
	65	+	kernel.rv64: main.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/root/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
	66	+	$(COMPILE) $(ROOT_CATALOG) $(CORE) -pkg kernel -start arch/start.ras -mod main.rad -entry kernel -o $@
	67	+
	68	+	check.rv64: check.rad core.rad $(MODULES) $(CORE_ASM) $(CHECK_MODULES) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
49	69		$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg check -mod check.rad $(addprefix -mod ,$(CHECK_MODULES)) -entry check -o $@
50	70
51		-	context.rv64: context.rad context/wait.rad check/context.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(COMPILER)
	71	+	context.rv64: context.rad context/wait.rad check/context.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
52	72		$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg context -start arch/entry.ras -mod context.rad -mod context/wait.rad -mod check/context.ras -entry context -o $@
53	73
54		-	interrupt.rv64: interrupt.rad check/interrupt.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(COMPILER)
	74	+	interrupt.rv64: interrupt.rad check/interrupt.ras arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
55	75		$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg interrupt -start arch/entry.ras -mod interrupt.rad -mod check/interrupt.ras -entry interrupt -o $@
56	76
57		-	native.rv64: native.rad native/instances.rad arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(COMPILER)
	77	+	native.rv64: native.rad native/instances.rad arch/entry.ras core.rad $(MODULES) $(CORE_ASM) build/baseline/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
58	78		$(COMPILE) $(BASE_CATALOG) $(CORE) -pkg native -start arch/entry.ras -mod native.rad -mod native/instances.rad -entry native -o $@
59	79
60		-	parallel.rv64: parallel.rad parallel/mailboxes.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(COMPILER)
	80	+	parallel.rv64: parallel.rad parallel/mailboxes.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/control/images.rad user/sys.ras $(STD_FILES) $(COMPILER)
61	81		$(COMPILE) $(CONTROL_CATALOG) $(CORE) -pkg parallel -start arch/start.ras -mod parallel.rad -mod parallel/mailboxes.rad -entry parallel -o $@
62	82
63		-	check: all check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64
	83	+	loading.rv64: loading.rad arch/start.ras core.rad $(MODULES) $(CORE_ASM) build/loading/images.rad build/loading/input.ras user/sys.ras $(STD_FILES) $(COMPILER)
	84	+	$(COMPILE) $(LOADING_CATALOG) $(CORE) -pkg loading -start arch/start.ras -mod loading.rad -mod build/loading/input.ras -entry loading -o $@
	85	+
	86	+	check: all check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64 loading.rv64
64	87		$(HOST_EMU) -run check.rv64
65	88		$(MACHINE) -max-steps=1000000 -run context.rv64
66	89		$(MACHINE) -harts=2 -irq=3 -irq-at=4000000 -uart-rx=52 -uart-rx-at=8000000 -run interrupt.rv64
67	90		$(MACHINE) -run native.rv64
68	91		$(MACHINE) -harts=1 -run parallel.rv64
69	92		$(MACHINE) -harts=2 -run parallel.rv64
70	93		$(MACHINE) -harts=8 -run parallel.rv64
	94	+	$(MACHINE) -harts=1 -run loading.rv64
	95	+	$(MACHINE) -harts=2 -run loading.rv64
71	96		@output="$$( $(MACHINE) -harts=2 -run kernel.rv64 2>build/boot.log )"; status=$$?; \
72		-	if test "$$status" -ne 2 \|\| test "$$output" != CONTROL-OK; then cat build/boot.log; exit 1; fi
	97	+	if test "$$status" -ne 2 \|\| test "$$output" != RIL-READY; then cat build/boot.log; exit 1; fi
73	98
74	99		clean:
75	100		rm -rf build
76		-	rm -f kernel.rv64 check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64
	101	+	rm -f kernel.rv64 check.rv64 context.rv64 interrupt.rv64 native.rv64 parallel.rv64 loading.rv64

36	36		mret;
37	37		@idleLoop
38	38		wfi;
39	39		j @idleLoop;
40	40		@enterUser
	41	+	// Synchronize this hart with native images published under the kernel lock.
	42	+	fence.i;
41	43		mv %t6 %a0;
42	44		ld %t0 256(%t6);
43	45		csrw mepc %t0;
44	46		li %t0 0x80;
45	47		csrw mstatus %t0;

2	2		.text;
3	3		la %t0 @bootFault;
4	4		csrw mtvec %t0;
5	5		csrr %a0 mhartid;
6	6		bnez %a0 @bootIdle;
7		-	call @"::default";
	7	+	la %t0 @"::default";
	8	+	jalr %ra %t0 0;
8	9		li %t1 0x5555;
9	10		j @bootFinish;
10	11		@bootFault
11	12		csrr %t1 mcause;
12	13		slli %t1 %t1 16;

12	12		bgeu %a0 %t0 @bootFault;
13	13		la %sp @hartStacks;
14	14		addi %t0 %a0 1;
15	15		slli %t0 %t0 16;
16	16		add %sp %sp %t0;
17		-	call @"::default";
	17	+	la %t0 @"::default";
	18	+	jalr %ra %t0 0;
18	19		csrr %t0 mhartid;
19	20		bnez %t0 @bootIdle;
20	21		li %t1 0x5555;
21	22		j @bootFinish;
22	23		@bootFault

16	16		mod activation;
17	17		mod lifecycle;
18	18		mod calls;
19	19
20	20		/// Run the available kernel mechanism checks.
21		-	@default fn main() -> u32 {
	21	+	@default unsafe fn main() -> u32 {
22	22		frames::run();
23	23		boot::run();
24	24		handles::run();
25	25		domains::run();
26	26		capabilities::run();

12	12		use core::memory;
13	13		use core::pages;
14	14		use core::platform;
15	15		use core::resources;
16	16		use core::state;
17		-	use images;
	17	+	use core::images;
18	18
19	19		/// Test-owned physical memory for Page and private-stack allocations.
20	20		static RAM: [u8; 64 * frames::PAGE_SIZE] = undefined;
21	21		/// Domain slots include spare creation capacity unless explicitly exhausted.
22	22		static DOMAINS: [domains::Domain; 3] = undefined;

29	29		static PINS: [u16; 64] = undefined;
30	30		static ASSIGNED: [bool; 64] = undefined;
31	31		static GRANTS: [u64; 3] = undefined;
32	32		/// Shared state for the exercised public operations.
33	33		static KERNEL: state::State = undefined;
	34	+	/// Image identities owned by this isolated kernel state.
	35	+	static IMAGES: images::Registry = undefined;
34	36
35	37		/// Zero only the supplied simulated physical range.
36	38		fn clear(base: u64, size: u32) {
37	39		assert base <= RAM.len as u64 and size as u64 <= RAM.len as u64 - base;
38	40		for i in base as u32..base as u32 + size { set RAM[i] = 0; }

53	55		let mut ram: memory::Memory = undefined;
54	56		memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 3);
55	57		domains::init(&mut DOMAINS[..]);
56	58		resources::init(&mut OBJECTS[..]);
57	59		for i in 0..CONTEXTS.len { contexts::init(&mut CONTEXTS[i], i); }
58		-	set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
59		-	let handle = domains::root(&mut DOMAINS[..], images::ROOT);
	60	+	images::init(&mut IMAGES);
	61	+	set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
	62	+	let handle = domains::root(&mut DOMAINS[..], KERNEL.images.root);
60	63		let root = try! domains::resolve(&DOMAINS[..], 0, handle, abi::CREATE);
61		-	let child = try! domains::create(&mut DOMAINS[..], root, images::ROOT);
	64	+	let child = try! domains::create(&mut DOMAINS[..], root, KERNEL.images.root);
62	65		return handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
63	66		}
64	67
65	68		/// Read a rejected call's public error without accepting accidental success.
66	69		fn createError(authority: abi::Handle, image: u64) -> abi::Error {

86	89
87	90		/// A rejected image/capacity request cannot consume authority or an incarnation.
88	91		fn creation() {
89	92		let target = setup();
90	93		let self = abi::Handle { bits: 0 };
91		-	assert createError(self, images::COUNT as u64) == abi::Error::VerifyFailed;
	94	+	assert createError(self, KERNEL.images.count as u64) == abi::Error::VerifyFailed;
92	95		assert createError(self, 0x100000000) == abi::Error::VerifyFailed;
93		-	assert createError(target, images::ROOT as u64) == abi::Error::Denied;
	96	+	assert createError(target, KERNEL.images.root as u64) == abi::Error::Denied;
94	97		let child = try! domains::resolve(&DOMAINS[..], 0, target, abi::EXECUTE);
95	98		let foreign = handles::install(&mut DOMAINS[0].handles, 3, child, abi::CREATE);
96		-	assert createError(foreign, images::ROOT as u64) == abi::Error::Denied;
	99	+	assert createError(foreign, KERNEL.images.root as u64) == abi::Error::Denied;
97	100		set DOMAINS[0].events.reserved = events::CRITICAL;
98		-	assert createError(self, images::ROOT as u64) == abi::Error::Exhausted;
	101	+	assert createError(self, KERNEL.images.root as u64) == abi::Error::Exhausted;
99	102		assert DOMAINS[0].events.reserved == events::CRITICAL;
100	103		set DOMAINS[0].events.reserved = 1;
101	104		let root = try! domains::authority(&DOMAINS[..], 0, self, abi::CREATE);
102	105		for i in 4..abi::MAX_HANDLES {
103	106		let _handle = handles::install(&mut DOMAINS[0].handles, i, root, abi::EXECUTE);
104	107		}
105		-	assert createError(self, images::ROOT as u64) == abi::Error::Exhausted;
	108	+	assert createError(self, KERNEL.images.root as u64) == abi::Error::Exhausted;
106	109		for i in 4..abi::MAX_HANDLES { let _entry = handles::remove(&mut DOMAINS[0].handles, i); }
107	110		assert DOMAINS[2].epoch == 0 and DOMAINS[2].state == domains::Lifecycle::Dead;
108		-	let _last = try! domains::create(&mut DOMAINS[..], root, images::ROOT);
109		-	assert createError(self, images::ROOT as u64) == abi::Error::Exhausted;
	111	+	let _last = try! domains::create(&mut DOMAINS[..], root, KERNEL.images.root);
	112	+	assert createError(self, KERNEL.images.root as u64) == abi::Error::Exhausted;
110	113		assert POOL.available == 64;
111	114		assert domains::live(&DOMAINS[..], try! domains::resolve(&DOMAINS[..], 0, target, abi::EXECUTE));
112	115		}
113	116
114	117		/// Verify stack edges, adjacent args, allocation rollback, and context isolation.

150	153		assert DOMAINS[1].env.context == first;
151	154		assert CONTEXTS[a].env.stackBase == 0 and CONTEXTS[a].env.stackTop == 8192;
152	155		assert CONTEXTS[a].env.argsPointer == 4096 and CONTEXTS[a].env.argsSize == 8192;
153	156		assert CONTEXTS[a].env.eventsPointer == &DOMAINS[1].events.ring as u64;
154	157		assert CONTEXTS[a].env.privateStackTop - CONTEXTS[a].env.privateStackBase == activation::PRIVATE_STACK_SIZE as u64;
155		-	assert CONTEXTS[a].frame.pc == images::get(images::ROOT).entry;
	158	+	assert CONTEXTS[a].frame.pc == images::get(KERNEL.images, KERNEL.images.root).entry;
156	159		assert startError(target, 8192, 0, 0, true) == abi::Error::NotPending;
157	160		// Empty args need no Page, even for an otherwise invalid pointer.
158	161		let second = try! activation::context(&mut KERNEL, 0, target, 4096, 0xffffffffffffffff, 0, clear);
159	162		let b = try! contexts::resolve(&CONTEXTS[..], second);
160	163		assert first <> second and a <> b;

27	28		static ASSIGNED: [bool; 1] = undefined;
28	29		/// Empty-pool persistent domain grants.
29	30		static GRANTS: [u64; 2] = undefined;
30	31		/// Shared kernel mechanism storage.
31	32		static KERNEL: state::State = undefined;
	33	+	/// Image identities owned by this isolated kernel state.
	34	+	static IMAGES: images::Registry = undefined;
32	35
33	36		/// Reject a split without consuming any authority from the source.
34	37		fn reject(source: abi::Handle, expected: abi::Error) {
35	38		let _handle = try budget_caps::split(&mut KERNEL, 0, source, 1) catch error {
36	39		assert error == expected;

57	60		set machine.memoryCount = 0;
58	61		set machine.reservedCount = 0;
59	62		try! frames::init(&mut POOL, &machine);
60	63		let mut ram: memory::Memory = undefined;
61	64		memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 2);
62		-	set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
	65	+	images::init(&mut IMAGES);
	66	+	set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
63	67		let rootHandle = domains::root(&mut DOMAINS[..], 0);
64	68		let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
65	69		let child = try! domains::create(&mut DOMAINS[..], root, 0);
66	70		let target = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
67	71		set DOMAINS[0].state = domains::Lifecycle::Active;

13	13		use core::lifecycle;
14	14		use core::memory;
15	15		use core::platform;
16	16		use core::resources;
17	17		use core::state;
	18	+	use core::images;
	19	+	use core::loading;
18	20
19	21		static DOMAINS: [domains::Domain; 2] = undefined;
20	22		static OBJECTS: [resources::Slot; 2] = undefined;
21	23		static CONTEXTS: [contexts::Context; 1] = undefined;
22	24		static POOL: frames::Pool = undefined;
23	25		static PINS: [u16; 1] = undefined;
24	26		static ASSIGNED: [bool; 1] = undefined;
25	27		static GRANTS: [u64; 2] = undefined;
26	28		static KERNEL: state::State = undefined;
	29	+	/// Image identities owned by this isolated kernel state.
	30	+	static IMAGES: images::Registry = undefined;
27	31
28	32		/// Exercise a returning operation through exactly the public seven-word input.
29		-	fn reply(caller: u32, number: u64, args: [u64; 7], now: u64) -> [u64; 4] {
	33	+	unsafe fn reply(caller: u32, number: u64, args: [u64; 7], now: u64) -> [u64; 4] {
30	34		let result = try! calls::dispatch(&mut KERNEL, caller, number, &args[..], now);
31	35		match result {
32	36		case calls::Outcome::Reply { values } => return values,
33	37		else => panic "reply: expected completed direct call",
34	38		}
35	39		}
36	40
37	41		/// Observe a boundary rejection without substituting a lower-level helper.
38		-	fn reject(caller: u32, number: u64, args: [u64; 7], expected: abi::Error) {
	42	+	unsafe fn reject(caller: u32, number: u64, args: [u64; 7], expected: abi::Error) {
39	43		let _result = try calls::dispatch(&mut KERNEL, caller, number, &args[..], 100) catch error {
40	44		assert error == expected;
41	45		return;
42	46		};
43	47		panic "reject: invalid direct call succeeded";
44	48		}
45	49
46	50		/// Slot kinds, reserved bits, and MMIO widths/rights remain full-width inputs.
47		-	fn materialization(root: abi::Handle) {
	51	+	unsafe fn materialization(root: abi::Handle) {
48	52		let device = try! resources::create(&mut OBJECTS[..], resources::Value::Device(fdt::Range { base: 0x1000, size: 8 }));
49	53		resources::retain(&mut OBJECTS[..], device);
50	54		let handle = handles::install(&mut DOMAINS[0].handles, 3, device, abi::READ);
51	55		let readonly = reply(0, 10, [handle.bits, 0, 0, 0, 0, 0, 0], 0)[0];
52	56		let info = reply(0, 45, [readonly, 0, 0, 0, 0, 0, 0], 0);

71	75		reject(0, 62, [3, 3, 0, 0, 0, 0, 0], abi::Error::BadHandle);
72	76		set OBJECTS[device.index].epoch = device.epoch;
73	77		}
74	78
75	79		/// Running metadata is conservatively debited; Run keeps admission transactional.
76		-	fn execution(target: abi::Handle, child: abi::Object) {
	80	+	unsafe fn execution(target: abi::Handle, child: abi::Object) {
77	81		set DOMAINS[child.index].state = domains::Lifecycle::Active;
78	82		try! contexts::prepare(&mut CONTEXTS[0], child, 0, 0, DOMAINS[child.index].env);
79	83		let id = contexts::identity(&CONTEXTS[0]);
80	84		set DOMAINS[child.index].env.context = id;
81	85		let resource = try! resources::create(&mut OBJECTS[..], resources::Value::Budget(try! budgets::init(0, 1000)));

105	109		reject(0, 73, args, abi::Error::Busy);
106	110		set CONTEXTS[0].status = contexts::Status::Stopped;
107	111		reject(0, 75, [target.bits, id + (1 as u64 << 32), 0, 0, 0, 0, 0], abi::Error::InvalidArg);
108	112		}
109	113
	114	+	/// Admission rejects bad authority and byte ranges before touching physical RAM.
	115	+	unsafe fn admission(root: abi::Handle, target: abi::Handle) {
	116	+	let count = KERNEL.images.count;
	117	+	let available = POOL.available;
	118	+	reject(1, 77, [0, 4096, 4, 0, 0, 0, 0], abi::Error::Denied);
	119	+	reject(0, 77, [target.bits, 4096, 4, 0, 0, 0, 0], abi::Error::Denied);
	120	+	reject(0, 77, [root.bits, 4096, 0, 0, 0, 0, 0], abi::Error::InvalidArg);
	121	+	reject(0, 77, [root.bits, 4096, loading::MAX_BYTES as u64 + 1, 0, 0, 0, 0], abi::Error::InvalidArg);
	122	+	reject(0, 77, [root.bits, 0xfffffffffffffffe, 4, 0, 0, 0, 0], abi::Error::InvalidArg);
	123	+	reject(0, 77, [root.bits, 4096, 4, 0, 0, 0, 0], abi::Error::Denied);
	124	+	assert KERNEL.images.count == count and POOL.available == available;
	125	+	}
	126	+
110	127		/// Independent target-local administrative authority survives source death.
111		-	export fn run() {
	128	+	export unsafe fn run() {
112	129		domains::init(&mut DOMAINS[..]);
113	130		contexts::init(&mut CONTEXTS[0], 0);
114	131		resources::init(&mut OBJECTS[..]);
115	132		let mut machine: platform::Platform = undefined;
116	133		set machine.memoryCount = 0;
117	134		set machine.reservedCount = 0;
118	135		try! frames::init(&mut POOL, &machine);
119	136		let mut ram: memory::Memory = undefined;
120	137		memory::init(&mut ram, &mut POOL, &mut PINS[..], &mut ASSIGNED[..], &mut GRANTS[..], 2);
121		-	set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram };
	138	+	images::init(&mut IMAGES);
	139	+	set KERNEL = state::State { domains: &mut DOMAINS[..], resources: &mut OBJECTS[..], contexts: &mut CONTEXTS[..], memory: ram, images: &mut IMAGES };
122	140		let rootHandle = domains::root(&mut DOMAINS[..], 0);
123	141		let root = try! domains::resolve(&DOMAINS[..], 0, rootHandle, abi::CREATE);
124	142		let child = try! domains::create(&mut DOMAINS[..], root, 0);
125	143		let target = handles::install(&mut DOMAINS[0].handles, 2, child, abi::DOMAIN_RIGHTS);
	144	+	admission(rootHandle, target);
126	145		materialization(rootHandle);
127	146		execution(target, child);
128	147		reject(0, 76, [rootHandle.bits, target.bits, 0x100000020, 0, 0, 0, 0], abi::Error::InvalidArg);
129	148		reject(0, 76, [rootHandle.bits, target.bits, abi::DESTROY as u64, 0, 0, 0, 0], abi::Error::InvalidArg);
130	149		let createGrant = (abi::CREATE \| abi::GRANT) as u64;

26	27		static ASSIGNED: [bool; 1] = undefined;
27	28		/// Per-domain grant backing storage.
28	29		static GRANTS: [u64; 3] = undefined;
29	30		/// Shared mechanism state for each transaction.
30	31		static KERNEL: state::State = undefined;
	32	+	/// Image identities owned by this isolated kernel state.
	33	+	static IMAGES: images::Registry = undefined;
31	34
32	35		/// Return a grant error as a test value.
33	36		fn grantError(source: abi::Handle, target: abi::Handle) -> abi::Error {
34	37		let _handle = try capabilities::grant(&mut KERNEL, 0, source, target, abi::RIGHTS as u64) catch error {
35	38		return error;

27	28		static ASSIGNED: [bool; 1] = undefined;
28	29		/// Domain grant backing.
29	30		static GRANTS: [u64; 1] = undefined;
30	31		/// Shared state for the checked materialization entry point.
31	32		static KERNEL: state::State = undefined;
	33	+	/// Image identities owned by this isolated kernel state.
	34	+	static IMAGES: images::Registry = undefined;
32	35
33	36		/// Install a real device resource and read-only capability without touching MMIO.
34	37		fn device(base: u64, size: u64) -> abi::Handle {
35	38		let object = try! resources::create(&mut OBJECTS[..], resources::Value::Device(fdt::Range { base, size }));
36	39		resources::retain(&mut OBJECTS[..], object);

30	31		static PINS: [u16; 64] = undefined;
31	32		static ASSIGNED: [bool; 64] = undefined;
32	33		static GRANTS: [u64; 8] = undefined;
33	34		/// Public mechanism state shared by each isolated scenario.
34	35		static KERNEL: state::State = undefined;
	36	+	/// Image identities owned by this isolated kernel state.
	37	+	static IMAGES: images::Registry = undefined;
35	38
36	39		/// Clear only the allocator's simulated physical range.
37	40		fn clear(base: u64, size: u32) {
38	41		assert base <= RAM.len as u64 and size as u64 <= RAM.len as u64 - base;
39	42		for i in base as u32..base as u32 + size { set RAM[i] = 0; }

13	13		use core::memory;
14	14		use core::notifications;
15	15		use core::platform;
16	16		use core::resources;
17	17		use core::state;
	18	+	use core::images;
18	19
19	20		/// Independent sender and receiver domain storage.
20	21		static DOMAINS: [domains::Domain; 2] = undefined;
21	22		/// Reserve and execution-loan resource slots.
22	23		static OBJECTS: [resources::Slot; 3] = undefined;

30	31		static ASSIGNED: [bool; 1] = undefined;
31	32		/// Empty-pool grants.
32	33		static GRANTS: [u64; 2] = undefined;
33	34		/// Kernel mechanism storage.
34	35		static KERNEL: state::State = undefined;
	36	+	/// Image identities owned by this isolated kernel state.
	37	+	static IMAGES: images::Registry = undefined;
35	38
36	39		/// Consume the receiver's next Wakeup with acquire/release progress.
37	40		fn consume(token: u32, sender: u32) {
38	41		let queue = &mut DOMAINS[1].events;
39	42		let head = atomic::load(&queue.ring.head);

29	30		static ASSIGNED: [bool; 16] = undefined;
30	31		/// Per-domain persistent grant sets.
31	32		static GRANTS: [u64; 3] = undefined;
32	33		/// State shared by all tested control-plane operations.
33	34		static KERNEL: state::State = undefined;
	35	+	/// Image identities owned by this isolated kernel state.
	36	+	static IMAGES: images::Registry = undefined;
34	37
35	38		/// Clear a physical allocation before it is exposed through a handle.
36	39		fn clear(base: u64, size: u32) {
37	40		assert base <= RAM.len as u64 and size as u64 <= RAM.len as u64 - base;
38	41		for i in base as u32..base as u32 + size { set RAM[i] = 0; }

35	36		static GRANTS: [u64; 1] = undefined;
36	37		/// Private one-shot timer storage.
37	38		static TIMERS: timers::Store = undefined;
38	39		/// Shared kernel mechanisms.
39	40		static KERNEL: state::State = undefined;
	41	+	/// Image identities owned by this isolated kernel state.
	42	+	static IMAGES: images::Registry = undefined;
40	43
41	44		/// Resume one authorized user interval until a real trap and charge it once.
42	45		unsafe fn step() -> u64 {
43	46		let context = &mut CONTEXTS[0];
44	47		let budget = budget_caps::value(&mut KERNEL, context.budget);

31	31		export mod calls;
32	32		export mod execution;
33	33		export mod control;
34	34		export mod teardown;
35	35		export mod runtime;
	36	+	export mod images;
	37	+	export mod loading;

26	27		run: frames::Run,
27	28		/// Aligned mutable-state base within the owned run.
28	29		base: u64,
29	30		}
30	31
31		-	/// Allocate and copy a checked catalog initializer before domain installation.
	32	+	/// Allocate and copy a published image initializer before domain installation.
32	33		/// The caller installs the returned ownership or reclaims it on failure.
33		-	export fn initialize(pool: *mut frames::Pool, image: u32, clear: fn(u64, u32), copy: fn(u64, u64, u32)) -> ImageState throws (abi::Error) {
34		-	let descriptor = images::get(image);
	34	+	export fn initialize(pool: mut frames::Pool, registry: images::Registry, image: u32, clear: fn(u64, u32), copy: fn(u64, u64, u32)) -> ImageState throws (abi::Error) {
	35	+	let descriptor = images::get(registry, image);
35	36		let mut run = frames::Run { first: 0, count: 0 };
36	37		let mut base: u64 = 0;
37	38		if descriptor.stateSize <> 0 {
38	39		let alignment = descriptor.stateAlignment as u64;
39	40		// These are trusted compiler facts, not caller-provided metadata.

51	52		};
52	53		set run = frames::install(allocation);
53	54		let first = run.first as u64 * frames::PAGE_SIZE as u64;
54	55		set base = (first + alignment - 1) & ~(alignment - 1);
55	56		copy(base, descriptor.initial, descriptor.stateSize);
	57	+	// The immutable relocation helper may have been compiled on another hart.
	58	+	smp::syncInstructions();
56	59		descriptor.relocate(base);
57	60		}
58	61		return ImageState { run, base };
59	62		}
60	63
61		-	/// Instantiate an admitted catalog graph under the caller's Create authority.
62		-	/// image is only a catalog index, never an entry address or runtime descriptor.
	64	+	/// Instantiate an admitted image graph under the caller's Create authority.
	65	+	/// image is only a stable image ID, never an entry address or runtime descriptor.
63	66		/// clear(base, size) zeros exactly the supplied physical allocation;
64	67		/// copy(destination, source, size) copies exactly the immutable initializer.
65	68		/// Both callbacks obey the serialization and lifetime rules of this module.
66	69		/// All capacity checks precede allocation; failure installs no authority or
67	70		/// domain incarnation and returns any provisionally allocated private frames.
68	71		export fn create(kernel: *mut state::State, caller: u32, authority: abi::Handle, image: u64, clear: fn(u64, u32), copy: fn(u64, u64, u32)) -> abi::Handle throws (abi::Error) {
69	72		let creator = try domains::authority(kernel.domains, caller, authority, abi::CREATE);
70	73		if creator.index <> caller { throw abi::Error::Denied; }
71		-	if image >= images::COUNT as u64 { throw abi::Error::VerifyFailed; }
	74	+	if image >= kernel.images.count as u64 { throw abi::Error::VerifyFailed; }
72	75		let slot = try handles::vacant(&kernel.domains[caller].handles);
73	76		let _domainSlot = try domains::vacant(kernel.domains);
74	77		// Refresh only consumed event credits; do not reserve a child until commit.
75	78		try events::refresh(&mut kernel.domains[caller].events);
76	79		if kernel.domains[caller].events.reserved == events::CRITICAL {
77	80		throw abi::Error::Exhausted;
78	81		}
79		-	let initial = try initialize(kernel.memory.pool, image as u32, clear, copy);
	82	+	let initial = try initialize(kernel.memory.pool, kernel.images, image as u32, clear, copy);
80	83		// The consumer can publish another head during allocation. Revalidate it
81	84		// when reserving the terminal event; even that late failure returns RAM.
82	85		let object = try domains::create(kernel.domains, creator, image as u32) catch error {
83	86		if initial.run.count <> 0 { frames::reclaim(kernel.memory.pool, initial.run); }
84	87		throw error;

124	127		break;
125	128		}
126	129		}
127	130		if slot == kernel.contexts.len { throw abi::Error::Exhausted; }
128	131		let domain = &kernel.domains[owner.index];
129		-	assert domain.image < images::COUNT;
130		-	let descriptor = images::get(domain.image);
	132	+	assert domain.image < kernel.images.count;
	133	+	let descriptor = images::get(kernel.images, domain.image);
131	134		let allocation = try frames::allocate(kernel.memory.pool, PRIVATE_STACK_SIZE / frames::PAGE_SIZE, clear) catch {
132	135		throw abi::Error::OutOfMemory;
133	136		};
134	137		let private = frames::install(allocation);
135	138		let privateBase = private.first as u64 * frames::PAGE_SIZE as u64;

93	93		let contextStore = storage(pool, contexts::CAPACITY, @sizeOf(contexts::Context), @alignOf(contexts::Context));
94	94		let pinStore = storage(pool, frames::MAX_FRAMES, @sizeOf(u16), @alignOf(u16));
95	95		let objectStore = storage(pool, frames::MAX_FRAMES, @sizeOf(bool), @alignOf(bool));
96	96		let grantCount = frames::MAX_FRAMES / 64 * abi::MAX_DOMAINS;
97	97		let grantStore = storage(pool, grantCount, @sizeOf(u64), @alignOf(u64));
	98	+	let imageStore = storage(pool, 1, @sizeOf(images::Registry), @alignOf(images::Registry));
	99	+	set kernel.images = imageStore as *mut images::Registry;
	100	+	images::init(kernel.images);
98	101		set kernel.domains = @sliceOf(domainStore as *mut domains::Domain, abi::MAX_DOMAINS);
99	102		set kernel.resources = @sliceOf(resourceStore as *mut resources::Slot, RESOURCE_CAPACITY);
100	103		set kernel.contexts = @sliceOf(contextStore as *mut contexts::Context, contexts::CAPACITY);
101	104		domains::init(kernel.domains);
102	105		resources::init(kernel.resources);

139	142		export unsafe fn init(kernel: mut state::State, pool: mut frames::Pool, machine: platform::Platform, controller: mut interrupts::Controller) -> u64 {
140	143		assert pool.limit > 0 and pool.limit <= frames::MAX_FRAMES;
141	144		assert machine.deviceCount <= platform::MAX_DEVICES;
142	145		assert machine.irqCount > 0 and machine.irqCount < platform::MAX_IRQS;
143	146		assert machine.hartCount > 0 and machine.hartCount <= platform::MAX_HARTS;
144		-	assert machine.frequency > 0 and images::ROOT < images::COUNT;
	147	+	assert machine.frequency > 0;
145	148		let budgetBaseSlot = 2 + machine.deviceCount + machine.irqCount;
146	149		let bootstrapBudgetSlot = budgetBaseSlot + machine.hartCount;
147	150		assert bootstrapBudgetSlot + 3 <= abi::MAX_HANDLES;
148	151		let mut bootHartIndex = machine.hartCount;
149	152		for i in 0..machine.hartCount {