radiance · commit

Makefile +29 -4


# Emulator command used to invoke the self-hosted compiler.
EMU       := $(or $(RAD_EMULATOR),emulator)
EMU_FLAGS := -memory-size=385024 \
			 -data-size=348160 \
			 -stack-size=512 \
			 -stack-size=1024 \
			 -count-instructions
RADIANCE  := $(EMU) $(EMU_FLAGS) -run $(RAD_BIN)

# Verify the emulator binary exists.
EMU_PATH := $(shell command -v $(EMU) 2>/dev/null)

default: emulator $(RAD_BIN)
test: emulator seed-test std-test bin-test kernel-test module-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test termination-test runtime-test
test: emulator seed-test std-test bin-test kernel-test module-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test termination-test runtime-test bootstrap-test mmio-test scheduling-test

seed-test:
	@seed/test

module-test: emulator $(RAD_BIN)

	@$(RADIANCE) $(STD) $(KERNEL) -entry kernel -ril $(BIN_DIR)

$(BIN_DIR)/kernel.build.rv64: kernel/tools/build.rad $(STD_LIB) $(RAD_BIN)
	@$(RADIANCE) $(STD) -pkg build -mod $< -entry build -o $@

$(BIN_DIR)/kernel.rv64: $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.build.rv64 $(KERNEL_ASM)
# User packages share the ABI source through a staged package module tree.
$(BIN_DIR)/root.ril $(BIN_DIR)/scheduler.ril &: kernel/root.rad kernel/scheduler.rad kernel/mmio.rad kernel/kernel/abi.rad kernel/kernel/sys.rad $(RAD_BIN)
	@mkdir -p $(BIN_DIR)/user/root $(BIN_DIR)/user/scheduler
	@cp kernel/root.rad kernel/scheduler.rad $(BIN_DIR)/user/
	@cp kernel/kernel/abi.rad kernel/kernel/sys.rad $(BIN_DIR)/user/root/
	@cp kernel/kernel/abi.rad kernel/kernel/sys.rad kernel/mmio.rad $(BIN_DIR)/user/scheduler/
	@$(RADIANCE) -pkg root -mod $(BIN_DIR)/user/root.rad -mod $(BIN_DIR)/user/root/abi.rad -mod $(BIN_DIR)/user/root/sys.rad -entry root -ril $(BIN_DIR)
	@$(RADIANCE) -pkg scheduler -mod $(BIN_DIR)/user/scheduler.rad -mod $(BIN_DIR)/user/scheduler/abi.rad -mod $(BIN_DIR)/user/scheduler/sys.rad -mod $(BIN_DIR)/user/scheduler/mmio.rad -entry scheduler -ril $(BIN_DIR)

$(BIN_DIR)/kernel.rv64: $(BIN_DIR)/root.ril $(BIN_DIR)/scheduler.ril $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.build.rv64 $(KERNEL_ASM)
	@cat $(KERNEL_ASM) > $(BIN_DIR)/kernel.ras
	@$(EMU) $(EMU_FLAGS) -run $(BIN_DIR)/kernel.build.rv64 -- $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.ras $@
	@$(EMU) $(EMU_FLAGS) -run $(BIN_DIR)/kernel.build.rv64 -- $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.ras $@ $(BIN_DIR)/root.ril $(BIN_DIR)/scheduler.ril

# Separate binary package emission and decode checks.
package-test: emulator $(BIN_DIR)/packages.check.rv64
	@sh test/packages/run


# Scheduled compilation, cancellation, retry, and shared executable publication.
runtime-test: emulator $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/runtime/run

.PHONY: runtime-test

# Root lifetime and resource authority across the boot-package handoff.
bootstrap-test: emulator $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/bootstrap/run

.PHONY: bootstrap-test

# Production root and scheduler packages execute all authority changes in user mode.
.PHONY: scheduling-test
scheduling-test: $(BIN_DIR)/kernel.rv64
	@RAD_EMULATOR="$(EMU)" sh test/scheduling/run

# Checked user-mode register access, including rejection before device effects.
.PHONY: mmio-test
mmio-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run mmio

compiler/radiance.rad +2 -2


/// AST arena size (64 MB) - retains parsed nodes throughout compilation.
constant TEMP_ARENA_SIZE: u32 = 67108864;
/// Per-function lowering and register-allocation arena size (16 MB).
constant FN_ARENA_SIZE: u32 = 16777216;
/// Main arena size (128 MB) - lives throughout compilation.
/// Main arena size (160 MB) - lives throughout compilation.
/// Used for: resolver data, types, symbols, global IL data, and codegen output.
constant MAIN_ARENA_SIZE: u32 = 134217728;
constant MAIN_ARENA_SIZE: u32 = 167772160;

/// AST storage arena.
static TEMP_ARENA: [u8; TEMP_ARENA_SIZE] = [0; TEMP_ARENA_SIZE];
/// Scratch storage reclaimed after each generated function.
static FN_ARENA: [u8; FN_ARENA_SIZE] = [0; FN_ARENA_SIZE];

kernel/kernel.rad +2 -0

export mod trap;
export mod capability;
export mod transactions;
export mod events;
export mod interrupts;
export mod devices;
export mod plic;
export mod registry;
export mod frames;
export mod backing;
export mod pages;

export mod timers;
export mod remote;
export mod calls;
export mod sys;
export mod boot;
export mod startup;
@test export mod tests;

kernel/kernel/abi.rad +32 -0

    eventsHandle: u64,
    /// Address of the domain's shared event ring.
    eventsPointer: u64,
}

/// Initial root resources copied into root-owned startup memory.
export record BootInfo: Copy {
    /// Number of Device handles after the self and Events slots.
    devices: u64,
    /// Number of Interrupt handles following the Device handles.
    interrupts: u64,
    /// Bit mask of online harts, whose Budget handles follow in hart order.
    harts: u64,
    /// Machine timer ticks per second.
    timebase: u64,
    /// Image used to construct root and additional instances of its program.
    rootImage: Handle,
    /// Initial user-space scheduler image.
    schedulerImage: Handle,
    /// Read-only machine-time device, or zero when absent.
    clock: Handle,
    /// Platform shutdown device, or zero when absent.
    shutdown: Handle,
    /// Page that owns this header and the copied handle array.
    arguments: Handle,
    /// Copied capability values in their initial slot order.
    handles: *unsafe mut [u64],
}

/// Physical extent returned by QueryPage.
export record PageInfo: Copy {
    /// Address of the first physical frame.
    base: u64,
    /// Number of contiguous physical frames.
    count: u64,
}

/// Physical MMIO extent returned by QueryDevice.
export record DeviceInfo: Copy {
    /// Physical base address of the register region.
    base: u64,
    /// Size of the register region in bytes.
    size: u64,
}

/// Physical source returned by QueryInterrupt.
export record InterruptInfo: Copy {
    /// PLIC source number; zero does not name an interrupt.
    number: u32,
}

kernel/kernel/boot.rad +21 -1

//! Machine entry boundary and published platform state.

use super::platform;
use super::startup;
use super::fdt;
use super::range;
use super::limits;
use super::sync;
use super::trap;

use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use super::interrupts;
use super::devices;
use super::plic;
use std::arch::rv64::shared::catalog;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.
static READY: u64 = 0;
/// Number of harts that validated their machine stack.
static ARRIVED: u64 = 0;
/// Root resource publication before user dispatch begins.
static STARTED: u64 = 0;
/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {

        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
        try! plic::initialize(&PLATFORM);
        assert entryCount == 2;
        try! devices::initialize(&mut devices::STORE, &PLATFORM);
        assert entryCount >= 2 and entryCount <= limits::PACKAGES;
        registry::initialize(&mut registry::STORE);
        let bootCatalog = @sliceOf(entries, entryCount);
        try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
        let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";
        let kernel = registry::find(&registry::STORE, &"kernel"[..]) else panic "boot kernel";

    let mut count: u64 = 0;
    for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
    return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
}

/// Publish configured root resources once, then enter dispatch on every hart.
export unsafe fn launch(last: bool) -> ! {
    if last {
        if let root = registry::find(&registry::STORE, &"root"[..]) {
            let scheduler = registry::find(&registry::STORE, &"scheduler"[..]) else panic "missing scheduler image";
            let domain = try! startup::create(&PLATFORM, root, scheduler);
            print("kernel: root ready\n");
        }
        sync::storeRelease(&mut STARTED, 1);
    } else {
        while sync::loadAcquire(&STARTED) == 0 {}
    }
    run();
}

/// Enter dispatch after every online hart publishes its initialized state.
export unsafe fn run() -> ! {
    let mut count: u64 = 0;
    for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
    while sync::loadAcquire(&ARRIVED) < count {}

kernel/kernel/boot.ras +9 -1

.export @kernel::boot::read8;
.export @kernel::boot::write8;

@kernel::boot::start
    call @kernel::boot::initialize;
    call @kernel::boot::run;
    call @kernel::boot::launch;
    ebreak;

// Initialize machine CSRs and preserve the firmware stack across Radiance entry.
@kernel::boot::initialize
    csrw mie %zero;

@kernel::boot::write8
    fence iorw iorw;
    sb %a1 0(%a0);
    fence iorw iorw;
    ret;

.export @kernel::startup::info;
.export @kernel::startup::words;
// Resolve root-owned startup memory after physical allocation.
@kernel::startup::info
    ret;
@kernel::startup::words
    ret;

kernel/kernel/calls.rad +16 -1

//! Direct-call execution and checked copies into domain-owned memory.

use super::abi;
use std::lang::il;
use super::slots;
use super::limits;
use super::capability;
use super::pages;
use super::domains;

use super::dispatch;
use super::timers;
use super::sync;
use super::remote;
use super::interrupts;
use super::devices;
use super::plic;
use super::lifecycle;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {

    match entry.kind {
        case abi::Kind::Domain, abi::Kind::Events => {
            let object = try domains::get(&domains::STORE, entry.object);
            if object.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
        },
        case abi::Kind::Device => { let object = try devices::get(&devices::STORE, entry.object); },
        case abi::Kind::Image => { let object = try registry::get(&registry::STORE, entry.object); },
        else => throw abi::Error::BadHandle,
    }
}


    let handle = abi::Handle(arguments[0]);
    try allocationAuthority(&domain.memory.table, operation, handle);
    match operation {
        case 22, 43, 49 => { let stopped = try termination(owner, operation, arguments); return 0; },
        case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
        case il::DEVICE_ACCESS => {
            let flags = arguments[2];
            if (flags & ~0x1ff) <> 0 { throw abi::Error::InvalidArg; }
            return try devices::address(&devices::STORE, &domain.memory.table, handle,
                arguments[1], flags & 0xff, (flags & 0x100) <> 0);
        },
        case 12 => { try drop(&mut domain, handle); return 0; },
        case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,
            &registry::STORE, &mut domain.memory.table, handle, abi::Handle(arguments[1])),
        case 21 => {
            try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,

            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
            let page = try pages::get(&pages::STORE, entry.object);
            try reply(&domain, arguments, &[page.base, page.count as u64]);
            return 0;
        },
        case 45 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Device, abi::Rights(0));
            let device = try devices::get(&devices::STORE, entry.object);
            try reply(&domain, arguments, &[device.info.base, device.info.size]);
            return 0;
        },
        case 46 => {
            let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
            if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) { throw abi::Error::InvalidArg; }
            set *memory32(arguments[1]) = number;

    assert trap::fromUser(frame) and frame.cause == 8;
    assert trap::advanceCall(frame);
    let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
    let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
        let operation = frame.registers[17];
        if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
        if operation == il::DEVICE_ACCESS or (operation >= 44 and operation <= 48 and error == abi::Error::BadHandle) {
            try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
                assert stopped == abi::Error::BadHandle;
            };
        }
        set frame.registers[10] = (0 as u64) - error as u64;

kernel/kernel/devices.rad added +83 -0

1	+	//! Immutable MMIO regions and domain-relative device authority.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::platform;
5	+	use super::capability;
6	+
7	+	/// One platform register region available to user-space drivers.
8	+	export record Region: Copy {
9	+	/// Platform mechanism used during bootstrap handoff.
10	+	kind: platform::Kind,
11	+	/// Physical register extent returned by QueryDevice.
12	+	info: abi::DeviceInfo,
13	+	/// Maximum rights installed by bootstrap.
14	+	rights: abi::Rights,
15	+	}
16	+
17	+	/// Device identities retained for the complete boot lifetime.
18	+	export record Store: Copy {
19	+	/// Exported regions in device-tree order.
20	+	regions: [Region; limits::DEVICES],
21	+	/// Number of published regions.
22	+	count: u32,
23	+	}
24	+
25	+	/// Global device metadata published before any user context starts.
26	+	export unsafe static STORE: Store = undefined;
27	+
28	+	/// Publish driver regions and the read-only CLINT time register.
29	+	/// Interrupt-controller and timer-programming registers stay kernel-private.
30	+	export fn initialize(store: &mut Store, machine: &platform::Platform) throws (abi::Error) {
31	+	set store.count = 0;
32	+	if machine.deviceCount > limits::DEVICES { throw abi::Error::InvalidArg; }
33	+	let mut count: u32 = 0;
34	+	for i in 0..machine.deviceCount {
35	+	let device = machine.devices[i];
36	+	if device.memory.start >= device.memory.end { throw abi::Error::InvalidArg; }
37	+	if device.kind == platform::Kind::Plic { continue; }
38	+	let mut base = device.memory.start;
39	+	let mut size = device.memory.end - base;
40	+	let mut rights = abi::READ \| abi::WRITE \| abi::GRANT \| abi::TRANSFER;
41	+	if device.kind == platform::Kind::Clint {
42	+	if size < 0xc000 { throw abi::Error::InvalidArg; }
43	+	set base += 0xbff8;
44	+	set size = 8;
45	+	set rights &= ~abi::WRITE;
46	+	}
47	+	set store.regions[count] = Region {
48	+	kind: device.kind, info: abi::DeviceInfo { base, size }, rights: abi::Rights(rights),
49	+	};
50	+	set count += 1;
51	+	}
52	+	set store.count = count;
53	+	}
54	+
55	+	/// Resolve an immutable device identity; physical regions use generation one.
56	+	export fn get(store: &Store, object: abi::Ref) -> Region throws (abi::Error) {
57	+	if object.index >= store.count or object.generation <> 1 { throw abi::Error::BadHandle; }
58	+	return store.regions[object.index];
59	+	}
60	+
61	+	/// Install bootstrap authority for one published physical device.
62	+	export fn install(store: &Store, table: &mut capability::Table, index: u32) -> abi::Handle throws (abi::Error) {
63	+	let object = abi::Ref { index, generation: 1 };
64	+	let region = try get(store, object);
65	+	return try capability::install(table, capability::Entry { kind: abi::Kind::Device, object, rights: region.rights });
66	+	}
67	+
68	+	/// Validate one register access and return its physical address to generated code.
69	+	export fn address(store: &Store, table: &capability::Table, handle: abi::Handle,
70	+	offset: u64, width: u64, writing: bool) -> u64 throws (abi::Error)
71	+	{
72	+	let mut right = abi::READ;
73	+	if writing { set right = abi::WRITE; }
74	+	let entry = try capability::lookup(table, handle, abi::Kind::Device, abi::Rights(right));
75	+	let region = (try get(store, entry.object)).info;
76	+	if width <> 1 and width <> 2 and width <> 4 and width <> 8 { throw abi::Error::InvalidArg; }
77	+	if offset > region.size or width > region.size - offset or region.base > 0xffffffffffffffff - offset {
78	+	throw abi::Error::InvalidArg;
79	+	}
80	+	let address = region.base + offset;
81	+	if (address & (width - 1)) <> 0 { throw abi::Error::InvalidArg; }
82	+	return address;
83	+	}

kernel/kernel/domains.rad +57 -7

    match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
}

/// Initialize retained domain storage outside the metadata lock.
export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
    let mut storage = creation.storage;
    try prepareStorage(packages, creation.storage, creation.domain);
}

/// Initialize one domain's private allocations before capability publication.
unsafe fn prepareStorage(packages: &registry::Store, storage: Prepared, owner: abi::Ref) throws (abi::Error) {
    let mut storage = storage;
    zero(storage.memory as u64, storage.allocation.count);
    zero(storage.kernelStack.start, storage.kernelFrames.count);
    try instances::fill(packages, &storage.graph.graph, storage.graph.base);
    capability::initialize(&mut storage.memory.table, creation.domain);
    capability::initialize(&mut storage.memory.table, owner);
}

/// Check the unpublished capacity retained by one creation transaction.
fn require(store: &Store, table: &capability::Table, creation: &Creation) {
    assert table.owner == creation.owner;

    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
            discard(&mut backings.pool, creation.storage, release);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
        },
    }
}

/// Publish an initialized domain and its Events under metadata serialization.

    -> abi::Handle throws (abi::Error)
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            if not backing::domainLive(backings, creation.owner) {
            let root = creation.owner == creation.domain;
            if not root and not backing::domainLive(backings, creation.owner) {
                cancel(store, backings, table, Reservation::Held(creation));
                throw abi::Error::BadHandle;
            }
            let object = creation.domain;
            let initial = creation.context;

                owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.reclaimed[object.index] = false;
            set store.reaping[object.index] = false;
            let mut parent: ?abi::Ref = nil;
            if not root { set parent = creation.owner; }
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                state: Lifecycle::Pending, creator: creation.owner, parent, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
            };
            set store.ancestors[object.index] = store.ancestors[creation.owner.index];
            set store.ancestors[object.index][creation.owner.index / 32] |= 1 << (creation.owner.index % 32);
            if not root {
                set store.ancestors[object.index] = store.ancestors[creation.owner.index];
                set store.ancestors[object.index][creation.owner.index / 32] |= 1 << (creation.owner.index % 32);
            }
            let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
            let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
            return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
                kind: abi::Kind::Domain, object: domain, rights: creation.rights,
            });
        },
    }
}

/// Create the first domain with its self capability in slot zero and Events in slot one.
/// Boot calls this before any domain, context, or exposure membership is published.
export unsafe fn bootstrap(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store, image: abi::Ref)
    -> abi::Ref throws (abi::Error)
{
    for slot in store.slots { if slot.state <> slots::State::Free { throw abi::Error::Busy; } }
    let package = try registry::get(packages, image);
    let entry = package.entry else { throw abi::Error::VerifyFailed; };
    let domainSlot = try slots::reserve(&mut store.slots[..]);
    let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch error {
        try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
    };
    let object = slots::reference(&domainSlot);
    let mut storage = try allocate(&mut backings.pool, packages, image) catch error {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
    };
    try prepareStorage(packages, storage, object) catch error {
        discard(&mut backings.pool, storage, frames::release);
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
    };
    let handleSlot = try! slots::reserve(&mut storage.memory.table.slots[..]);
    assert slots::reference(&handleSlot).index == 0;
    match domainSlot {
        case slots::Reservation::Held(domain) => match contextSlot {
            case slots::Reservation::Held(context) => match handleSlot {
                case slots::Reservation::Held(handle) => {
                    let result = try publish(store, backings, &mut storage.memory.table, Reservation::Held(Creation {
                        owner: object, handle, domain, context,
                        rights: abi::Rights(MANAGEMENT_RIGHTS | abi::CREATE | abi::ALLOCATE), image, entry, storage,
                    }));
                    assert (try! abi::decode(result)).object.index == 0;
                    return object;
                },
            },
        },
    }
}

/// Create a private domain while the caller has exclusive access to metadata.
export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(store, backings, packages, table, authority, image);

kernel/kernel/startup.rad added +72 -0

1	+	//! Initial root resources and execution authority.
2	+	use super::abi;
3	+	use super::platform;
4	+	use super::limits;
5	+	use super::domains;
6	+	use super::pages;
7	+	use super::registry;
8	+	use super::devices;
9	+	use super::interrupts;
10	+	use super::budgets;
11	+	use super::dispatch;
12	+	use super::slots;
13	+	use super::capability;
14	+
15	+	/// Map a root-owned startup allocation to its ABI record.
16	+	fn info(address: u64) -> *mut abi::BootInfo;
17	+	/// Map an aligned root-owned startup extent to copied handle words.
18	+	fn words(address: u64) -> *unsafe mut u64;
19	+
20	+	/// Install root's fixed resource slots and activate its initial context on hart zero.
21	+	/// Boot calls this after global and per-hart initialization, before user dispatch.
22	+	export unsafe fn create(machine: &platform::Platform, image: abi::Ref, scheduler: abi::Ref) -> abi::Ref throws (abi::Error) {
23	+	if (machine.harts & 1) == 0 or machine.timebase == 0 { throw abi::Error::InvalidArg; }
24	+	let root = try domains::bootstrap(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, image);
25	+	let mut domain = try! domains::get(&domains::STORE, root);
26	+	let table = &mut domain.memory.table;
27	+	let self = try! abi::handle(abi::Kind::Domain, abi::Ref { index: 0, generation: table.slots[0].generation });
28	+	let mut clock = abi::Handle(0);
29	+	let mut shutdown = abi::Handle(0);
30	+	for i in 0..devices::STORE.count {
31	+	let handle = try devices::install(&devices::STORE, &mut *table, i);
32	+	if devices::STORE.regions[i].kind == platform::Kind::Clint { set clock = handle; }
33	+	if devices::STORE.regions[i].kind == platform::Kind::Finish { set shutdown = handle; }
34	+	}
35	+	for number in 1..interrupts::STORE.count + 1 {
36	+	let handle = try interrupts::seed(&mut interrupts::STORE, &mut *table, number);
37	+	}
38	+	let now = dispatch::clock();
39	+	let second = machine.timebase as u64;
40	+	if now > 0xffffffffffffffff - second * 61 { throw abi::Error::InvalidArg; }
41	+	let start = now + second;
42	+	let end = start + second * 60;
43	+	let mut count = 2 + devices::STORE.count + interrupts::STORE.count;
44	+	for hart in 0..limits::HARTS {
45	+	if (machine.harts & (1 << hart)) == 0 { continue; }
46	+	let handle = try budgets::seed(&mut budgets::STORE, &mut *table, hart, start, end);
47	+	if hart == 0 {
48	+	let bound = try budgets::bind(&mut budgets::STORE, &domains::STORE, &mut *table,
49	+	budgets::Binding { budget: handle, domain: self, context: domain.initial }, now);
50	+	}
51	+	set count += 1;
52	+	}
53	+	let rootImage = try registry::install(&registry::STORE, &mut *table, image);
54	+	let schedulerImage = try registry::install(&registry::STORE, &mut *table, scheduler);
55	+	let stack = try pages::allocate(&mut pages::STORE, &mut *table, self, 16);
56	+	let arguments = try pages::allocate(&mut pages::STORE, &mut *table, self, 1);
57	+	let stackPage = try! pages::get(&pages::STORE, (try! capability::get(&*table, stack)).object);
58	+	let argumentPage = try! pages::get(&pages::STORE, (try! capability::get(&*table, arguments)).object);
59	+	assert @sizeOf(abi::BootInfo) as u64 + count as u64 * 8 <= limits::FRAME_SIZE;
60	+	let handles = @sliceOf(words(argumentPage.base + @sizeOf(abi::BootInfo) as u64), count);
61	+	for i in 0..count {
62	+	assert table.slots[i].state == slots::State::Live;
63	+	set handles[i] = *try! abi::handle(table.entries[i].kind, abi::Ref { index: i, generation: table.slots[i].generation });
64	+	}
65	+	set *info(argumentPage.base) = abi::BootInfo {
66	+	devices: devices::STORE.count as u64, interrupts: interrupts::STORE.count as u64,
67	+	harts: machine.harts as u64, timebase: second, rootImage, schedulerImage, clock, shutdown, arguments, handles,
68	+	};
69	+	try domains::activate(&mut domains::STORE, &pages::STORE, &mut *table, self,
70	+	stackPage.base + stackPage.count as u64 * limits::FRAME_SIZE, argumentPage.base, @sizeOf(abi::BootInfo) as u64);
71	+	return root;
72	+	}

kernel/kernel/sys.rad +10 -0

        try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
        return info;
    }
}

/// Read Device metadata. An invalid handle is a program error.
export fn queryDevice(handle: abi::Handle) -> abi::DeviceInfo {
    unsafe {
        let mut info: abi::DeviceInfo = undefined;
        try! result(ecall(abi::Operation::QueryDevice as u32, *handle as i64,
            (&mut info) as i64, @sizeOf(abi::DeviceInfo) as i64, 0));
        return info;
    }
}

/// Read a physical interrupt source. An invalid handle is a program error.
export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
    unsafe {
        let mut info: abi::InterruptInfo = undefined;
        try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,

kernel/kernel/tests.rad +1 -0

export mod backing;
export mod pages;
export mod transactions;
export mod events;
export mod interrupts;
export mod devices;
export mod registry;
export mod loader;
export mod instances;
export mod domains;
export mod lifecycle;

kernel/kernel/tests/calls.rad +25 -1

use kernel::trap;
use kernel::budgets;
use kernel::remote;
use kernel::interrupts;
use kernel::events;
use kernel::devices;
use kernel::platform;

/// Event storage for administrative destruction targets.
unsafe static TARGETS: [domains::Memory; 2] = undefined;

/// Retained dead-domain records cannot be queried or delegated through handles.

    try testing::expect(budgets::STORE.slots[0].state == slots::State::Free);
    let installed = try! capability::get(&MEMORY.table, queue);
    try testing::expect(installed.kind == abi::Kind::Events);
}

/// Device queries copy only metadata into checked caller-owned memory.
@test unsafe fn deviceRecord() throws (testing::TestError) {
    let owner = initialize();
    let storage = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
    set devices::STORE.count = 1;
    set devices::STORE.regions[0] = devices::Region {
        kind: platform::Kind::Uart, info: { base: 0x10000000, size: 256 }, rights: abi::Rights(abi::READ),
    };
    let device = try! devices::install(&devices::STORE, &mut MEMORY.table, 0);
    let base = (&RAM[OFFSET]) as u64;
    set RAM[OFFSET + 2] = 123;
    assert try! calls::invoke(owner, 45, &[*device, base, 16, 0], 0) == 0;
    assert RAM[OFFSET] == 0x10000000 and RAM[OFFSET + 1] == 256 and RAM[OFFSET + 2] == 123;
    let mut rejected: u32 = 0;
    for args in &[[*device, base + 1, 16, 0], [*device, base, 8, 0], [*device, base, 16, 1]] {
        try calls::invoke(owner, 45, &args[..], 0) catch error {
            assert error == abi::Error::InvalidArg; set rejected += 1;
        };
    }
    assert rejected == 3 and RAM[OFFSET + 2] == 123;
}

/// A query copies exactly its record into writable memory owned by the caller.
@test unsafe fn records() throws (testing::TestError) {
    let owner = initialize();
    let handle = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
    let base = (&RAM[OFFSET]) as u64;

    try testing::expect(rejected == 6 and domains::STORE.contextSlots[0].state == slots::State::Free);
}

/// Invalid infallible queries terminate the caller at the syscall boundary.
@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
    for operation in [44 as u64, 46, 47, 48] {
    for operation in [44 as u64, 45, 46, 47, 48] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let mut frame = trap::Frame { registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
        set frame.registers[17] = operation;
        calls::handle(owner, &mut frame, 0, calls::invoke);

kernel/kernel/tests/devices.rad added +117 -0

1	+	//! Device identity, exported regions, and capability delegation.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::devices;
5	+	use kernel::platform;
6	+	use kernel::capability;
7	+	use kernel::calls;
8	+	use kernel::domains;
9	+	use kernel::slots;
10	+
11	+	/// Device metadata under test.
12	+	unsafe static STORE: devices::Store = undefined;
13	+	/// Synthetic validated platform regions.
14	+	unsafe static MACHINE: platform::Platform = undefined;
15	+	/// Independent caller and receiver capability storage.
16	+	unsafe static MEMORY: [domains::Memory; 2] = undefined;
17	+
18	+	/// Describe driver devices around kernel-owned interrupt and timer registers.
19	+	unsafe fn initialize() {
20	+	set MACHINE.deviceCount = 4;
21	+	set MACHINE.devices[0] = { kind: platform::Kind::Uart, memory: { start: 0x10000000, end: 0x10000100 } };
22	+	set MACHINE.devices[1] = { kind: platform::Kind::Plic, memory: { start: 0x0c000000, end: 0x10000000 } };
23	+	set MACHINE.devices[2] = { kind: platform::Kind::Clint, memory: { start: 0x02000000, end: 0x02010000 } };
24	+	set MACHINE.devices[3] = { kind: platform::Kind::Finish, memory: { start: 0x10001000, end: 0x10002000 } };
25	+	try! devices::initialize(&mut STORE, &MACHINE);
26	+	}
27	+
28	+	/// Driver regions retain tree order and expose only read-only timer state.
29	+	@test unsafe fn regions() throws (testing::TestError) {
30	+	initialize();
31	+	assert STORE.count == 3;
32	+	assert STORE.regions[0].kind == platform::Kind::Uart;
33	+	let timer = abi::DeviceInfo { base: 0x0200bff8, size: 8 };
34	+	assert STORE.regions[1].info == timer;
35	+	assert not abi::permits(STORE.regions[1].rights, abi::Rights(abi::WRITE));
36	+	assert STORE.regions[2].kind == platform::Kind::Finish;
37	+	let mut rejected: u32 = 0;
38	+	for object in [abi::Ref { index: 3, generation: 1 }, abi::Ref { index: 0, generation: 2 }] {
39	+	try devices::get(&STORE, object) catch error {
40	+	assert error == abi::Error::BadHandle; set rejected += 1;
41	+	};
42	+	}
43	+	assert rejected == 2;
44	+	set MACHINE.devices[2].memory.end = 0x0200bfff;
45	+	try devices::initialize(&mut STORE, &MACHINE) catch error {
46	+	assert error == abi::Error::InvalidArg; set rejected += 1;
47	+	};
48	+	assert rejected == 3 and STORE.count == 0;
49	+	}
50	+
51	+	/// Device authority can move to a surviving domain and can only lose rights.
52	+	@test unsafe fn delegation() throws (testing::TestError) {
53	+	initialize();
54	+	set devices::STORE = STORE;
55	+	domains::initialize(&mut domains::STORE);
56	+	let mut owners: [abi::Ref; 2] = undefined;
57	+	for i in 0..2 {
58	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
59	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
60	+	set owners[i] = owner;
61	+	capability::initialize(&mut MEMORY[i].table, owner);
62	+	set domains::STORE.records[owner.index].state = domains::Lifecycle::Active;
63	+	set domains::STORE.records[owner.index].memory = &mut MEMORY[i];
64	+	}
65	+	let target = try! capability::install(&mut MEMORY[0].table, capability::Entry {
66	+	kind: abi::Kind::Domain, object: owners[1], rights: abi::Rights(0),
67	+	});
68	+	let device = try! devices::install(&STORE, &mut MEMORY[0].table, 2);
69	+	let original = try! capability::get(&MEMORY[0].table, device);
70	+	let granted = abi::Handle(try! calls::invoke(owners[0], 10, &[device, target, abi::READ as u64, 0], 0));
71	+	let reader = try! capability::get(&MEMORY[1].table, granted);
72	+	assert reader.object == original.object and reader.rights == abi::Rights(abi::READ);
73	+	let moved = abi::Handle(try! calls::invoke(owners[0], 11, &[device, target, *original.rights as u64, 0], 0));
74	+	assert (try! capability::get(&MEMORY[1].table, moved)) == original;
75	+	let old = try! abi::decode(device);
76	+	assert MEMORY[0].table.slots[old.object.index].state == slots::State::Free;
77	+	let dropped = try! calls::invoke(owners[1], 12, &[*moved, 0, 0, 0], 0);
78	+	assert dropped == 0;
79	+	assert (try! devices::get(&devices::STORE, reader.object)).info.base == 0x10001000;
80	+	}
81	+
82	+	/// Every register access checks current authority and the complete physical extent.
83	+	@test unsafe fn access() throws (testing::TestError) {
84	+	initialize();
85	+	capability::initialize(&mut MEMORY[0].table, abi::Ref { index: 0, generation: 1 });
86	+	let device = try! devices::install(&STORE, &mut MEMORY[0].table, 0);
87	+	for width in [1 as u64, 2, 4, 8] {
88	+	assert (try! devices::address(&STORE, &MEMORY[0].table, device, 256 - width, width, false)) == 0x10000100 - width;
89	+	assert (try! devices::address(&STORE, &MEMORY[0].table, device, 0, width, true)) == 0x10000000;
90	+	}
91	+	let mut rejected: u32 = 0;
92	+	for offset in [1 as u64, 253, 256, 0xffffffffffffffff] {
93	+	try devices::address(&STORE, &MEMORY[0].table, device, offset, 4, false) catch error {
94	+	assert error == abi::Error::InvalidArg; set rejected += 1;
95	+	};
96	+	}
97	+	for width in [0 as u64, 3, 16, 0x100000004] {
98	+	try devices::address(&STORE, &MEMORY[0].table, device, 0, width, false) catch error {
99	+	assert error == abi::Error::InvalidArg; set rejected += 1;
100	+	};
101	+	}
102	+	let clock = try! devices::install(&STORE, &mut MEMORY[0].table, 1);
103	+	try devices::address(&STORE, &MEMORY[0].table, clock, 0, 8, true) catch error {
104	+	assert error == abi::Error::Denied; set rejected += 1;
105	+	};
106	+	let removed = try! capability::invalidate(&mut MEMORY[0].table, device);
107	+	let replacement = try! devices::install(&STORE, &mut MEMORY[0].table, 0);
108	+	assert replacement <> device;
109	+	try devices::address(&STORE, &MEMORY[0].table, device, 0, 1, false) catch error {
110	+	assert error == abi::Error::BadHandle; set rejected += 1;
111	+	};
112	+	set STORE.regions[0].info.base = 0x10000001;
113	+	try devices::address(&STORE, &MEMORY[0].table, replacement, 0, 4, false) catch error {
114	+	assert error == abi::Error::InvalidArg; set rejected += 1;
115	+	};
116	+	assert rejected == 11;
117	+	}

kernel/kernel/tests/domains.rad +51 -0

        template: &[7], memory: 8, alignment: 8, relocations: &[],
    });
    set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
}

/// Failed root construction restores capacity and permits a fresh attempt.
@test unsafe fn bootstrapRollback() throws (testing::TestError) {
    initialize(abi::CREATE);
    let image = (try! capability::get(&TABLE, IMAGE)).object;
    domains::initialize(&mut DOMAINS);
    set BACKINGS.domains[0] = 0;
    let capacity = BACKINGS.pool.count;
    for count in [0 as u32, 1, 8, 32] {
        set BACKINGS.pool.count = count;
        let mut failed = false;
        try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
            assert error == abi::Error::OutOfMemory; set failed = true;
        };
        assert failed and DOMAINS.slots[0].state == slots::State::Free;
        assert DOMAINS.contextSlots[0].state == slots::State::Free;
        assert BACKINGS.domains[0] == 0 and DOMAINS.events.queues[0].generation == 0;
        for i in 0..count { assert frames::available(&BACKINGS.pool, i); }
    }
    set BACKINGS.pool.count = capacity;
    let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
    assert root.index == 0 and root.generation > 1;
    assert (try! domains::get(&DOMAINS, root)).parent == nil;
}

/// Root construction owns slot zero and installs the fixed bootstrap handles.
@test unsafe fn bootstrap() throws (testing::TestError) {
    initialize(abi::CREATE);
    let image = (try! capability::get(&TABLE, IMAGE)).object;
    domains::initialize(&mut DOMAINS);
    set BACKINGS.domains[0] = 0;
    let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
    assert root.index == 0 and root.generation == 1;
    let domain = try! domains::get(&DOMAINS, root);
    assert domain.creator == root and domain.parent == nil and domain.state == domains::Lifecycle::Pending;
    assert domain.memory.table.slots[0].state == slots::State::Live;
    let self = domain.memory.table.entries[0];
    assert self.kind == abi::Kind::Domain and self.object == root;
    assert *self.rights == (domains::MANAGEMENT_RIGHTS | abi::CREATE | abi::ALLOCATE);
    let queue = try! abi::decode(domain.events);
    assert queue.object.index == 1 and queue.kind == abi::Kind::Events;
    assert domain.memory.table.entries[1].object == root;
    assert DOMAINS.contexts[domain.initial.index].frame.pc == 0x80000000;
    assert DOMAINS.contexts[domain.initial.index].frame.registers[3] == domain.graph.table.ptr as u64;
    for word in DOMAINS.ancestors[0] { assert word == 0; }
    let mut busy = false;
    try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy and DOMAINS.slots[1].state == slots::State::Free;
}

/// Interrupted creation returns all private storage after its caller releases its hart.
@test unsafe fn cancelledCreation() throws (testing::TestError) {
    initialize(abi::CREATE);
    lifecycle::initialize(&mut CALLS);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);

kernel/mmio.rad added +62 -0

1	+	//! Fixed-width register access through live Device capabilities.
2	+	use super::abi;
3	+
4	+	/// Compiler primitive for one checked 8-bit register read.
5	+	@intrinsic fn deviceRead8(handle: u64, offset: u64) -> u8;
6	+	/// Compiler primitive for one checked 8-bit register write.
7	+	@intrinsic fn deviceWrite8(handle: u64, offset: u64, value: u8);
8	+
9	+	/// Read an aligned 8-bit register at a byte offset within the device.
10	+	export fn read8(handle: abi::Handle, offset: u64) -> u8 {
11	+	return deviceRead8(*handle, offset);
12	+	}
13	+
14	+	/// Write an aligned 8-bit register at a byte offset within the device.
15	+	export fn write8(handle: abi::Handle, offset: u64, value: u8) {
16	+	deviceWrite8(*handle, offset, value);
17	+	}
18	+
19	+	/// Compiler primitive for one checked 16-bit register read.
20	+	@intrinsic fn deviceRead16(handle: u64, offset: u64) -> u16;
21	+	/// Compiler primitive for one checked 16-bit register write.
22	+	@intrinsic fn deviceWrite16(handle: u64, offset: u64, value: u16);
23	+
24	+	/// Read an aligned 16-bit register at a byte offset within the device.
25	+	export fn read16(handle: abi::Handle, offset: u64) -> u16 {
26	+	return deviceRead16(*handle, offset);
27	+	}
28	+
29	+	/// Write an aligned 16-bit register at a byte offset within the device.
30	+	export fn write16(handle: abi::Handle, offset: u64, value: u16) {
31	+	deviceWrite16(*handle, offset, value);
32	+	}
33	+
34	+	/// Compiler primitive for one checked 32-bit register read.
35	+	@intrinsic fn deviceRead32(handle: u64, offset: u64) -> u32;
36	+	/// Compiler primitive for one checked 32-bit register write.
37	+	@intrinsic fn deviceWrite32(handle: u64, offset: u64, value: u32);
38	+
39	+	/// Read an aligned 32-bit register at a byte offset within the device.
40	+	export fn read32(handle: abi::Handle, offset: u64) -> u32 {
41	+	return deviceRead32(*handle, offset);
42	+	}
43	+
44	+	/// Write an aligned 32-bit register at a byte offset within the device.
45	+	export fn write32(handle: abi::Handle, offset: u64, value: u32) {
46	+	deviceWrite32(*handle, offset, value);
47	+	}
48	+
49	+	/// Compiler primitive for one checked 64-bit register read.
50	+	@intrinsic fn deviceRead64(handle: u64, offset: u64) -> u64;
51	+	/// Compiler primitive for one checked 64-bit register write.
52	+	@intrinsic fn deviceWrite64(handle: u64, offset: u64, value: u64);
53	+
54	+	/// Read an aligned 64-bit register at a byte offset within the device.
55	+	export fn read64(handle: abi::Handle, offset: u64) -> u64 {
56	+	return deviceRead64(*handle, offset);
57	+	}
58	+
59	+	/// Write an aligned 64-bit register at a byte offset within the device.
60	+	export fn write64(handle: abi::Handle, offset: u64, value: u64) {
61	+	deviceWrite64(*handle, offset, value);
62	+	}

kernel/root.rad added +58 -0

1	+	//! Initial resource handoff to the user-space scheduler.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Root startup memory supplied by the kernel.
6	+	record Env: Copy {
7	+	/// Writable header and copied resource handles.
8	+	argsPointer: *unsafe mut abi::BootInfo,
9	+	/// Header size in bytes.
10	+	argsSize: u64,
11	+	/// Root's event capability.
12	+	eventsHandle: u64,
13	+	/// Root's event-ring address.
14	+	eventsPointer: u64,
15	+	}
16	+
17	+	/// Transfer root's resource bundle and parent duties before normal exit.
18	+	@default unsafe fn main(env: *Env) -> ! {
19	+	assert env.argsSize == @sizeOf(abi::BootInfo) as u64;
20	+	let mut info = env.argsPointer;
21	+	let self = abi::Handle(info.handles[0]);
22	+	let scheduler = try! sys::domainCreate(self, info.schedulerImage);
23	+	let context = try! sys::queryContext(scheduler, 0);
24	+	let authority = try! sys::capabilityGrant(scheduler, scheduler, abi::RIGHTS_MASK as u64);
25	+	let stack = try! sys::pageAllocate(self, 16);
26	+	let stackInfo = sys::queryPage(stack);
27	+	let sharedStack = try! sys::capabilityTransfer(stack, scheduler, abi::RIGHTS_MASK as u64);
28	+	let mut handles = info.handles;
29	+	let clock = info.clock;
30	+	let shutdown = info.shutdown;
31	+	let budgets = 2 + info.devices as u32 + info.interrupts as u32;
32	+	for i in 2..budgets {
33	+	let old = abi::Handle(handles[i]);
34	+	let moved = try! sys::capabilityTransfer(old, scheduler, abi::RIGHTS_MASK as u64);
35	+	set handles[i] = *moved;
36	+	if old == clock { set info.clock = moved; }
37	+	if old == shutdown { set info.shutdown = moved; }
38	+	}
39	+	for i in budgets..handles.len {
40	+	let budget = abi::Handle(handles[i]);
41	+	let window = try! sys::queryBudget(budget);
42	+	if window.hart == 0 {
43	+	let future = try! sys::budgetSplit(budget, window.start + info.timebase * 2);
44	+	set handles[i] = *try! sys::budgetBind(future, scheduler, context.context);
45	+	} else {
46	+	set handles[i] = *try! sys::capabilityTransfer(budget, scheduler, abi::RIGHTS_MASK as u64);
47	+	}
48	+	}
49	+	set info.rootImage = try! sys::capabilityTransfer(info.rootImage, scheduler, abi::RIGHTS_MASK as u64);
50	+	set info.schedulerImage = try! sys::capabilityTransfer(info.schedulerImage, scheduler, abi::RIGHTS_MASK as u64);
51	+	set info.arguments = try! sys::capabilityTransfer(info.arguments, scheduler, abi::RIGHTS_MASK as u64);
52	+	set handles[0] = *authority;
53	+	set handles[1] = 0;
54	+	try! sys::domainReparent(self, scheduler);
55	+	try! sys::domainActivate(scheduler, stackInfo.base + stackInfo.count * 4096,
56	+	info as *unsafe opaque, @sizeOf(abi::BootInfo) as u64);
57	+	sys::exit(0);
58	+	}

kernel/scheduler.rad added +47 -0

1	+	//! User-space CPU windows and bounded worker execution.
2	+	export mod abi;
3	+	export mod sys;
4	+	export mod mmio;
5	+
6	+	/// Scheduler or worker startup values.
7	+	record Env: Copy {
8	+	/// Shared resource bundle for the scheduler.
9	+	argsPointer: *abi::BootInfo,
10	+	/// Zero for a worker, or the resource header size for the scheduler.
11	+	argsSize: u64,
12	+	/// Installed event capability.
13	+	eventsHandle: u64,
14	+	/// Shared event-ring address.
15	+	eventsPointer: u64,
16	+	}
17	+
18	+	/// Run a worker or distribute four successive CPU windows from the boot bundle.
19	+	@default unsafe fn main(env: *Env) -> ! {
20	+	if env.argsSize == 0 {
21	+	sys::exit(7);
22	+	}
23	+	assert env.argsSize == @sizeOf(abi::BootInfo) as u64;
24	+	let info = env.argsPointer;
25	+	let self = abi::Handle(info.handles[0]);
26	+	let started = mmio::read64(info.clock, 0);
27	+	assert sys::queryDomain(self).state == 1;
28	+	for turn in 0..4 {
29	+	let current = sys::currentContext();
30	+	let budget = abi::Handle(current.budget);
31	+	let window = try! sys::queryBudget(budget);
32	+	let later = try! sys::budgetSplit(budget, window.start + info.timebase);
33	+	let worker = try! sys::domainCreate(self, info.schedulerImage);
34	+	let stack = try! sys::pageAllocate(self, 16);
35	+	let extent = sys::queryPage(stack);
36	+	let transferred = try! sys::capabilityTransfer(stack, worker, abi::RIGHTS_MASK as u64);
37	+	try! sys::domainActivate(worker, extent.base + extent.count * 4096, (&env) as unsafe opaque, 0);
38	+	try! sys::yield(worker);
39	+	assert sys::currentContext().budget == *later;
40	+	}
41	+	assert mmio::read64(info.clock, 0) > started;
42	+	if info.shutdown <> abi::Handle(0) {
43	+	mmio::write32(info.shutdown, 0, 0x5555);
44	+	panic "shutdown returned";
45	+	}
46	+	sys::exit(0);
47	+	}

kernel/tools/build.rad +43 -21

use std::arch::rv64::shared;
use std::arch::rv64::shared::catalog;

/// Native entry page, above the platform firmware data.
constant CODE_ADDRESS: u64 = 0x81000000;
/// Maximum shared packages in the boot image.
constant PACKAGES: u32 = 4;
/// Package code arenas retained until image output completes.
unsafe static CODE: [[u8; 16777216]; 2] = undefined;
unsafe static CODE: [[u8; 16777216]; PACKAGES] = undefined;
/// Reusable function workspace.
static SCRATCH: [u8; 16777216] = [0; 16777216];
/// Persistent decoded RIL storage.
static DECODE: [u8; 67108864] = [0; 67108864];
/// Exact package bytes retained in the native catalog.
unsafe static INPUT: [[u8; 8388608]; 2] = undefined;
unsafe static INPUT: [[u8; 8388608]; PACKAGES] = undefined;
/// Combined startup and boundary assembly source.
static SOURCE: [u8; 65536] = [0; 65536];
/// Assembler workspace.
static ASSEMBLY: [u8; 4194304] = [0; 4194304];
/// Assembled startup words.
static TEXT: [u32; 16384] = [0; 16384];
/// Assembly identifiers retained through package linking.
unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };
/// Per-package data layout workspaces.
unsafe static DATA: [[data::DataSym; 4096]; 2] = undefined;
unsafe static DATA: [[data::DataSym; 4096]; PACKAGES] = undefined;
/// Local function and data definitions.
unsafe static SYMBOLS: [[shared::Symbol; 4096]; 2] = undefined;
unsafe static SYMBOLS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
/// Public package symbols.
unsafe static EXPORTS: [[shared::Symbol; 4096]; 2] = undefined;
unsafe static EXPORTS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
/// Exports from dependencies compiled before the current package.
unsafe static IMPORTS: [shared::Symbol; 16384] = undefined;
/// Private-state relocation records.
unsafe static RELOCS: [[shared::Relocation; 4096]; 2] = undefined;
unsafe static RELOCS: [[shared::Relocation; 4096]; PACKAGES] = undefined;
/// Initialized package templates.
unsafe static TEMPLATES: [[u8; 1048576]; 2] = undefined;
unsafe static TEMPLATES: [[u8; 1048576]; PACKAGES] = undefined;
/// Read-only catalog and retained package payloads.
static RO: [u8; 16777216] = [0; 16777216];
/// Kernel state table and initialized private package graph.
static RW: [u8; 33554432] = [0; 33554432];
/// Contiguous native code segment, including entry trampoline and alignment padding.

            template: &mut TEMPLATES[slot][..], relocations: &mut RELOCS[slot][..],
        }, &mut arena, &mut scratch);
}

/// Emit a trampoline that preserves firmware arguments and supplies the native catalog.
unsafe fn trampoline(kernel: u64, ro: u64, rw: u64) {
unsafe fn trampoline(kernel: u64, ro: u64, rw: u64, count: u32) {
    let mut arena = alloc::new(&mut SCRATCH[..]);
    let mut e = try! emit::emitter(&mut arena, false);
    emit::loadImm(&mut e, rv64::GP, rw as i64);
    emit::loadImm(&mut e, rv64::A3, ro as i64);
    emit::loadImm(&mut e, rv64::A4, 2);
    emit::loadImm(&mut e, rv64::A4, count as i64);
    emit::loadImm(&mut e, rv64::T0, kernel as i64);
    emit::emit(&mut e, encode::jalr(rv64::ZERO, rv64::T0, 0));
    assert e.codeLen <= 1024;
    for word, i in emit::getCode(&e) { set NATIVE[i] = word; }
}

/// Link shared boot packages, retain their catalog, and instantiate the kernel graph.
@default unsafe fn main(env: *sys::Env) -> i32 {
    assert env.args.len == 5;
    assert env.args.len >= 5 and env.args.len <= PACKAGES + 3;
    let count = env.args.len - 3;
    let mut decoder = alloc::new(&mut DECODE[..]);
    let libraryBytesLength = unix::readFile(env.args[1], &mut INPUT[0][..]) else panic "std RIL";
    let libraryBytes = &INPUT[0][..libraryBytesLength];
    let libraryInput = try! program::decode(libraryBytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
    let kernelBytesLength = unix::readFile(env.args[2], &mut INPUT[1][..]) else panic "kernel RIL";

    let kernelInput = try! program::decode(kernelBytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
    assert mem::eq(libraryInput.name, "std") and mem::eq(kernelInput.name, "kernel");
    assert libraryInput.dependencies.len == 0;
    for dependency in kernelInput.dependencies { assert mem::eq(dependency, "std"); }
    let sourceLength = unix::readFile(env.args[3], &mut SOURCE[..]) else panic "kernel assembly";
    let source = &SOURCE[..sourceLength];
    let mut assemblyArena = alloc::new(&mut ASSEMBLY[..]);
    let empty: *mut [u8] = &mut [];
    let assembly = try! asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..sourceLength],
    let assembly = try! asm::assemble(asm::scanner::SourceKind::String, source,
        &mut TEXT[..], &mut empty[..], &mut assemblyArena, &mut STRINGS, 0);
    let library = try compile(&libraryInput, 0, CODE_ADDRESS + 4096, &[],
        asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error { report(error); return 1; };
    let kernelAddress = aligned(library.codeAddress + library.code.len as u64 * 4, 4096);
    let kernel = try compile(&kernelInput, 1, kernelAddress, library.exports, assembly) catch error { report(error); return 1; };
    let entries = [
        catalog::Entry { source: libraryBytes, package: library },
        catalog::Entry { source: kernelBytes, package: kernel },
    ];
    let codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
    let mut catalogEntries: [catalog::Entry; PACKAGES] = undefined;
    set catalogEntries[0] = catalog::Entry { source: libraryBytes, package: library };
    set catalogEntries[1] = catalog::Entry { source: kernelBytes, package: kernel };
    let mut codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
    for slot in 2..count {
        let bytesLength = unix::readFile(env.args[slot + 3], &mut INPUT[slot][..]) else panic "boot package RIL";
        let bytes = &INPUT[slot][..bytesLength];
        let input = try! program::decode(bytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
        let mut imported: u32 = 0;
        for previous in 0..slot {
            for symbol in catalogEntries[previous].package.exports {
                assert imported < IMPORTS.len;
                set IMPORTS[imported] = symbol;
                set imported += 1;
            }
        }
        let package = try compile(&input, slot, aligned(codeEnd, 4096), &IMPORTS[..imported],
            asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error { report(error); return 1; };
        set catalogEntries[slot] = catalog::Entry { source: bytes, package };
        set codeEnd = package.codeAddress + package.code.len as u64 * 4;
    }
    let codeSize = (codeEnd - CODE_ADDRESS) as u32;
    assert codeEnd - CODE_ADDRESS <= @sizeOf([u32; 4194304]) as u64;
    let roAddress = aligned(codeEnd, 4096);
    let roSize = try catalog::pack(&entries[..], roAddress, &mut RO[..]) catch error { report(error); return 1; };
    let roSize = try catalog::pack(&catalogEntries[..count], roAddress, &mut RO[..]) catch error { report(error); return 1; };
    let rwAddress = aligned(roAddress + roSize as u64, 4096);
    let mut bases: [u64; shared::MAX_PACKAGES] = [0; shared::MAX_PACKAGES];
    let mut rwSize: u32 = @sizeOf([u64; shared::MAX_PACKAGES]);
    for entry in &entries[..] {
    for entry in &catalogEntries[..2] {
        let at = aligned(rwAddress + rwSize as u64, entry.package.alignment);
        assert at - rwAddress <= RW.len as u64 and entry.package.memory <= RW.len - (at - rwAddress) as u32;
        set bases[entry.package.slot] = at;
        set rwSize = (at - rwAddress) as u32 + entry.package.memory;
    }
    try! mem::copy(&mut RW[..@sizeOf([u64; shared::MAX_PACKAGES])],
        @sliceOf(&bases[0] as *unsafe u8, @sizeOf([u64; shared::MAX_PACKAGES])));
    for entry in &entries[..] {
    for entry in &catalogEntries[..2] {
        let at = (bases[entry.package.slot] - rwAddress) as u32;
        try shared::instantiate(&entry.package, &bases[..], &mut RW[at..at + entry.package.memory])
            catch error { report(error); return 1; };
    }
    for i in 0..codeSize / 4 { set NATIVE[i] = encode::nop(); }
    for entry in &entries[..] {
    for entry in &catalogEntries[..count] {
        let at = ((entry.package.codeAddress - CODE_ADDRESS) / 4) as u32;
        for word, i in entry.package.code { set NATIVE[at + i] = word; }
    }
    trampoline(kernelAddress, roAddress, rwAddress);
    trampoline(kernelAddress, roAddress, rwAddress, count);
    let header = try! image::header(image::Layout {
        entry: CODE_ADDRESS,
        code: image::Segment { address: CODE_ADDRESS, initialized: codeSize, memory: codeSize },
        roData: image::Segment { address: roAddress, initialized: roSize, memory: roSize },
        rwData: image::Segment { address: rwAddress, initialized: rwSize, memory: rwSize },

lib/std/arch/rv64/isel.rad +32 -0


            // Result in A0.
            let ecallRd = getDstReg(s, dst, super::SCRATCH1);
            emitMv(s, ecallRd, super::A0);
        },
        case il::Instr::DeviceRead { typ, dst, handle, offset } => {
            deviceAddress(s, typ, handle, offset, il::Val::Imm(0), false);
            let rd = getDstReg(s, dst, super::SCRATCH1);
            match typ {
                case il::Type::W8 => emit::emit(s.e, encode::lbu(rd, super::A0, 0)),
                case il::Type::W16 => emit::emit(s.e, encode::lhu(rd, super::A0, 0)),
                case il::Type::W32 => emit::emit(s.e, encode::lwu(rd, super::A0, 0)),
                case il::Type::W64 => emit::emit(s.e, encode::ld(rd, super::A0, 0)),
            }
            emit::emit(s.e, encode::fence());
        },
        case il::Instr::DeviceWrite { typ, handle, offset, value } => {
            deviceAddress(s, typ, handle, offset, value, true);
            match typ {
                case il::Type::W8 => emit::emit(s.e, encode::sb(super::A3, super::A0, 0)),
                case il::Type::W16 => emit::emit(s.e, encode::sh(super::A3, super::A0, 0)),
                case il::Type::W32 => emit::emit(s.e, encode::sw(super::A3, super::A0, 0)),
                case il::Type::W64 => emit::emit(s.e, encode::sd(super::A3, super::A0, 0)),
            }
            emit::emit(s.e, encode::fence());
        },
        case il::Instr::Ebreak => {
            emit::emit(s.e, encode::ebreak());
        },
        case il::Instr::MemoryFence => {
            emit::emit(s.e, encode::fence());

    }
    if invert {
        emit::emit(s.e, encode::xori(rd, rd, 1));
    }
}

/// Resolve one live Device access before an ordered user-mode register instruction.
unsafe fn deviceAddress(s: &mut Selector, typ: il::Type, handle: il::Val, offset: il::Val, value: il::Val, writing: bool) {
    let mut flags = il::typeSize(typ) as i64;
    if writing { set flags |= 0x100; }
    let dsts: [gen::Reg; 5] = [super::A7, super::A0, super::A1, super::A2, super::A3];
    let args: [il::Val; 5] = [il::Val::Imm(il::DEVICE_ACCESS as i64), handle, offset, il::Val::Imm(flags), value];
    emitParallelMoves(s, &dsts[..], &args[..]);
    emit::emit(s.e, encode::ecall());
    emit::emit(s.e, encode::fence());
}

lib/std/lang/il.rad +36 -1

    /// Environment break: `ebreak;`.
    /// Triggers a breakpoint exception for debugging.
    Ebreak,
    /// Full acquire/release memory fence.
    MemoryFence,
    /// Read one register through a live Device handle.
    DeviceRead {
        /// Register access width.
        typ: Type,
        /// Destination for the zero-extended register value.
        dst: Reg,
        /// Domain-relative Device capability.
        handle: Val,
        /// Byte offset within the device region.
        offset: Val,
    },
    /// Write one register through a live Device handle.
    DeviceWrite {
        /// Register access width.
        typ: Type,
        /// Domain-relative Device capability.
        handle: Val,
        /// Byte offset within the device region.
        offset: Val,
        /// Value whose low bits are written at the selected width.
        value: Val,
    },
}

//////////////////////////
// Blocks and Functions //
//////////////////////////


///////////////////////
// Utility Functions //
///////////////////////

/// Compiler-generated metadata call for one MMIO access.
/// a0 is the handle, a1 the offset, and a2 the byte width plus bit 8 for a write.
/// a3 holds a write value and is preserved. Success returns the address in a0.
/// Any failure terminates the caller before the user-mode register access.
export constant DEVICE_ACCESS: u32 = 74;

/// Get the destination register of an instruction, if any.
export fn instrDst(instr: Instr) -> ?Reg {
    match instr {
        case Instr::Reserve { dst, .. } => return dst,
        case Instr::Load { dst, .. } => return dst,

        case Instr::UnOp { dst, .. } => return dst,
        case Instr::Zext { dst, .. } => return dst,
        case Instr::Sext { dst, .. } => return dst,
        case Instr::Call { dst, .. } => return dst,
        case Instr::Ecall { dst, .. } => return dst,
        case Instr::DeviceRead { dst, .. } => return dst,
        else => return nil,
    }
}

/// Check if an instruction is a function call.
export fn isCall(instr: Instr) -> bool {
    match instr {
        case Instr::Call { .. },
             Instr::Ecall { .. } => return true,
             Instr::Ecall { .. }, Instr::DeviceRead { .. }, Instr::DeviceWrite { .. } => return true,
        else => return false,
    }
}

/// Call a function for each register used by an instruction.

            withReg(a0, f, ctx);
            withReg(a1, f, ctx);
            withReg(a2, f, ctx);
            withReg(a3, f, ctx);
        },
        case Instr::DeviceRead { handle, offset, .. } => {
            withReg(handle, f, ctx); withReg(offset, f, ctx);
        },
        case Instr::DeviceWrite { handle, offset, value, .. } => {
            withReg(handle, f, ctx); withReg(offset, f, ctx); withReg(value, f, ctx);
        },
        case Instr::Unreachable,
             Instr::Ebreak,
             Instr::MemoryFence => {},
    }
}

lib/std/lang/il/binary.rad +4 -0

export constant INSTR_ECALL: u8 = 16;
/// Wire tag for instruction Ebreak.
export constant INSTR_EBREAK: u8 = 17;
/// Wire tag for instruction MemoryFence.
export constant INSTR_MEMORYFENCE: u8 = 18;
/// Wire tag for a checked device read.
export constant INSTR_DEVICE_READ: u8 = 19;
/// Wire tag for a checked device write.
export constant INSTR_DEVICE_WRITE: u8 = 20;

/// Wire tag for data Val.
export constant DATA_VAL: u8 = 0;
/// Wire tag for data Sym.
export constant DATA_SYM: u8 = 1;

lib/std/lang/il/binary/collect.rad +6 -0

            try value(names, owner, a0);
            try value(names, owner, a1);
            try value(names, owner, a2);
            try value(names, owner, a3);
        },
        case il::Instr::DeviceRead { handle, offset, .. } => {
            try value(names, owner, handle); try value(names, owner, offset);
        },
        case il::Instr::DeviceWrite { handle, offset, value: source, .. } => {
            try value(names, owner, handle); try value(names, owner, offset); try value(names, owner, source);
        },
        case il::Instr::Load { .. }, il::Instr::Sload { .. }, il::Instr::Unreachable,
             il::Instr::Ebreak, il::Instr::MemoryFence => {},
    }
}


lib/std/lang/il/binary/reader.rad +11 -0

            let va1 = try val(input);
            let va2 = try val(input);
            let va3 = try val(input);
            return il::Instr::Ecall { dst: vdst, num: vnum, a0: va0, a1: va1, a2: va2, a3: va3 };
        },
        case super::INSTR_DEVICE_READ => {
            let t = try typ(input);
            let dst = try reg(input);
            let handle = try val(input); let offset = try val(input);
            return il::Instr::DeviceRead { typ: t, dst, handle, offset };
        },
        case super::INSTR_DEVICE_WRITE => {
            let t = try typ(input);
            let handle = try val(input); let offset = try val(input); let value = try val(input);
            return il::Instr::DeviceWrite { typ: t, handle, offset, value };
        },
        case super::INSTR_EBREAK => {
            return il::Instr::Ebreak;
        },
        case super::INSTR_MEMORYFENCE => {
            return il::Instr::MemoryFence;

lib/std/lang/il/binary/tests.rad +12 -0

        dst: il::Reg { n: 1 }, num: il::Val::Undef, a0: il::Val::Undef, a1: il::Val::Undef,
        a2: il::Val::Undef, a3: il::Val::Undef,
    }, &[16, 1, 0, 0, 0, 4, 4, 4, 4, 4]);
    try instruction(il::Instr::Ebreak, &[17]);
    try instruction(il::Instr::MemoryFence, &[18]);
    for typ in [il::Type::W8, il::Type::W16, il::Type::W32, il::Type::W64] {
        let mut read: [u8; 8] = [19, 0, 1, 0, 0, 0, 4, 4];
        let mut write: [u8; 5] = [20, 0, 4, 4, 4];
        set read[1] = il::typeSize(typ) as u8;
        set write[1] = il::typeSize(typ) as u8;
        try instruction(il::Instr::DeviceRead {
            typ, dst: il::Reg { n: 1 }, handle: il::Val::Undef, offset: il::Val::Undef,
        }, &read[..]);
        try instruction(il::Instr::DeviceWrite {
            typ, handle: il::Val::Undef, offset: il::Val::Undef, value: il::Val::Undef,
        }, &write[..]);
    }
}

/// Check every value tag and empty and nonempty sequences.
@test unsafe fn values() throws (testing::TestError) {
    let mut buffer: [u8; 64] = [0; 64];

lib/std/lang/il/binary/writer.rad +11 -0

            try val(out, va0);
            try val(out, va1);
            try val(out, va2);
            try val(out, va3);
        },
        case il::Instr::DeviceRead { typ: t, dst, handle, offset } => {
            try integer(out, super::INSTR_DEVICE_READ as u64, 1);
            try typ(out, t);
            try integer(out, dst.n as u64, 4);
            try val(out, handle); try val(out, offset);
        },
        case il::Instr::DeviceWrite { typ: t, handle, offset, value } => {
            try integer(out, super::INSTR_DEVICE_WRITE as u64, 1);
            try typ(out, t);
            try val(out, handle); try val(out, offset); try val(out, value);
        },
        case il::Instr::Ebreak => {
            try integer(out, super::INSTR_EBREAK as u64, 1);
        },
        case il::Instr::MemoryFence => {
            try integer(out, super::INSTR_MEMORYFENCE as u64, 1);

lib/std/lang/il/printer.rad +8 -0

            write(out, " ");
            writeVal(out, a2);
            write(out, " ");
            writeVal(out, a3);
        }
        case super::Instr::DeviceRead { typ, dst, handle, offset } => {
            write(out, "device-read "); write(out, typeStr(typ)); write(out, " ");
            writeReg(out, dst); write(out, " "); writeVal(out, handle); write(out, " "); writeVal(out, offset);
        }
        case super::Instr::DeviceWrite { typ, handle, offset, value } => {
            write(out, "device-write "); write(out, typeStr(typ)); write(out, " ");
            writeVal(out, handle); write(out, " "); writeVal(out, offset); write(out, " "); writeVal(out, value);
        }
        case super::Instr::Ebreak => {
            write(out, "ebreak");
        }
        case super::Instr::MemoryFence => {
            write(out, "memory-fence");

lib/std/lang/lower.rad +34 -7

unsafe fn emit(self: &mut FnLowerer, instr: il::Instr) {
    let blk = self.currentBlock else panic;
    let mut block = getBlockMut(self, blk);

    // Track whether this function is a leaf.
    if self.isLeaf {
        match instr {
            case il::Instr::Call { .. },
                 il::Instr::Ecall { .. } => set self.isLeaf = false,
            else => {},
        }
    }
    if il::isCall(instr) { set self.isLeaf = false; }
    // Record source location alongside instruction when enabled.
    if self.low.options.debug {
        block.locs.append(self.srcLoc, self.allocator);
    }
    block.instrs.append(instr, self.allocator);

        return nil;
    }
    // Check for known intrinsic names.
    if mem::eq(sym.name, "ecall") {
        return try lowerEcall(self, call);
    } else if mem::eq(sym.name, "deviceRead8") {
        return try lowerDevice(self, call, il::Type::W8, false);
    } else if mem::eq(sym.name, "deviceWrite8") {
        return try lowerDevice(self, call, il::Type::W8, true);
    } else if mem::eq(sym.name, "deviceRead16") {
        return try lowerDevice(self, call, il::Type::W16, false);
    } else if mem::eq(sym.name, "deviceWrite16") {
        return try lowerDevice(self, call, il::Type::W16, true);
    } else if mem::eq(sym.name, "deviceRead32") {
        return try lowerDevice(self, call, il::Type::W32, false);
    } else if mem::eq(sym.name, "deviceWrite32") {
        return try lowerDevice(self, call, il::Type::W32, true);
    } else if mem::eq(sym.name, "deviceRead64") {
        return try lowerDevice(self, call, il::Type::W64, false);
    } else if mem::eq(sym.name, "deviceWrite64") {
        return try lowerDevice(self, call, il::Type::W64, true);
    } else if mem::eq(sym.name, "ebreak") {
        return try lowerEbreak(self, call);
    } else if mem::eq(sym.name, "memoryFence") {
        return try lowerMemoryFence(self, call);
    } else {

        case resolver::Type::Int => return il::Type::W64,
        case resolver::Type::Opaque => panic "ilType: opaque type must be behind a pointer",
        else => panic "ilType: type cannot be lowered",
    }
}

/// Lower one fixed-width MMIO operation without exposing a pointer value.
unsafe fn lowerDevice(self: &mut FnLowerer, call: ast::Call, typ: il::Type, writing: bool) -> il::Val throws (LowerError) {
    let mut count: u32 = 2;
    if writing { set count = 3; }
    if call.args.len <> count { throw LowerError::InvalidArgCount; }
    let handle = try lowerExpr(self, call.args[0]);
    let offset = try lowerExpr(self, call.args[1]);
    if writing {
        let value = try lowerExpr(self, call.args[2]);
        emit(self, il::Instr::DeviceWrite { typ, handle, offset, value });
        return il::Val::Undef;
    }
    let dst = nextReg(self);
    emit(self, il::Instr::DeviceRead { typ, dst, handle, offset });
    return il::Val::Reg(dst);
}

test/acceptance/replay added +27 -0

1	+	#!/bin/sh
2	+	# Compare complete output from two executions of the same native image.
3	+	set -eu
4	+	if [ "$#" -lt 4 ]; then
5	+	printf 'usage: replay profile emulator image harts [machine-options...]\n' >&2
6	+	exit 1
7	+	fi
8	+	profile=$1
9	+	emulator=$2
10	+	image=$3
11	+	harts=$4
12	+	shift 4
13	+	mkdir -p bin/acceptance/traces
14	+	prefix="bin/acceptance/traces/$profile-$harts"
15	+	for attempt in 1 2; do
16	+	if ! "$emulator" -machine -harts="$harts" -memory-size=262144 -max-steps=2000000000 "$@" -run "$image" > "$prefix.$attempt.log" 2>&1; then
17	+	cat "$prefix.$attempt.log"
18	+	exit 1
19	+	fi
20	+	done
21	+	if ! cmp -s "$prefix.1.log" "$prefix.2.log"; then
22	+	diff -u "$prefix.1.log" "$prefix.2.log" \|\| true
23	+	printf 'replay: %s %s-hart output differs\n' "$profile" "$harts" >&2
24	+	exit 1
25	+	fi
26	+	cat "$prefix.1.log"
27	+	printf 'replay: %s %s-hart output matched\n' "$profile" "$harts"

test/boot/run +5 -3

set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
cat test/boot/machine.ras kernel/kernel/*.ras > "$work/boot.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
for harts in 1 2 8; do
    # The deterministic emulator shares its tick budget across all harts.
    steps=$((10000000 * harts))
    if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then
        cat "$work/log" >&2

        cat "$work/log" >&2
        exit 1
    fi
    printf 'kernel boot: %s harts passed\n' "$harts"
done
cat kernel/kernel/*.ras > "$work/idle.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/idle.ras" "$work/idle.rv64"
status=0
"$emulator" -machine -harts=8 -max-steps=80000000 -run bin/kernel.rv64 > "$work/log" 2>&1 || status=$?
"$emulator" -machine -harts=8 -max-steps=80000000 -run "$work/idle.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
    cat "$work/log" >&2
    exit 1
fi
printf 'kernel startup: all eight harts reached machine idle without traps\n'
cat test/boot/trap.ras kernel/kernel/*.ras > "$work/trap.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
status=0
"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
    || [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then
    cat "$work/log" >&2

test/bootstrap/kernel/dispatchcheck.rad added +88 -0

1	+	//! Native root construction, exit, and continued resource authority.
2	+	use kernel::abi;
3	+	use kernel::boot;
4	+	use kernel::registry;
5	+	use kernel::domains;
6	+	use kernel::capability;
7	+	use kernel::pages;
8	+	use kernel::budgets;
9	+	use kernel::dispatch;
10	+	use kernel::calls;
11	+	use kernel::events;
12	+	use kernel::sync;
13	+	use kernel::startup;
14	+	use kernel::devices;
15	+	use kernel::interrupts;
16	+
17	+	/// Initial root identity retained by the terminal event.
18	+	unsafe static ROOT: abi::Ref = undefined;
19	+	/// Surviving controller domain with inherited resource authority.
20	+	unsafe static CONTROLLER: abi::Ref = undefined;
21	+	/// Image capability installed in the controller's own table.
22	+	unsafe static IMAGE: abi::Handle = undefined;
23	+	/// Current kernel package-state table.
24	+	fn kernelGp() -> u64;
25	+	/// Controller entry address.
26	+	fn completion() -> u64;
27	+	/// Signal successful completion to the emulator.
28	+	fn finish();
29	+	/// Map an assembly-compatible physical byte address.
30	+	fn memory(address: u64) -> *mut u8;
31	+
32	+	/// Construct root from the boot catalog and give its child a later CPU window.
33	+	export unsafe fn setup() {
34	+	let image = registry::find(&registry::STORE, &"loaded"[..]) else panic "missing boot user package";
35	+	let service = registry::find(&registry::STORE, &"service"[..]) else panic "missing boot service package";
36	+	set ROOT = try! startup::create(&boot::PLATFORM, image, service);
37	+	assert ROOT.index == 0;
38	+	let mut root = try! domains::get(&domains::STORE, ROOT);
39	+	let self = try! abi::handle(abi::Kind::Domain, abi::Ref { index: 0, generation: root.memory.table.slots[0].generation });
40	+	assert (try! abi::decode(root.events)).object.index == 1;
41	+	let executable = try! registry::install(&registry::STORE, &mut root.memory.table, image);
42	+	let childHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
43	+	&registry::STORE, &mut root.memory.table, self, executable);
44	+	set CONTROLLER = (try! capability::get(&root.memory.table, childHandle)).object;
45	+	let mut child = try! domains::get(&domains::STORE, CONTROLLER);
46	+	let inherited = domains::MANAGEMENT_RIGHTS \| abi::CREATE \| abi::ALLOCATE;
47	+	let authority = try! calls::invoke(ROOT, 10, &[childHandle, childHandle, inherited as u64, 0], 0);
48	+	assert service.index == 3 and image.index == 2;
49	+	set IMAGE = try! registry::install(&registry::STORE, &mut child.memory.table, service);
50	+	try! domains::reparent(&mut domains::STORE, &root.memory.table, self, childHandle);
51	+	set domains::STORE.records[CONTROLLER.index].state = domains::Lifecycle::Active;
52	+	let frame = &mut domains::STORE.contexts[child.initial.index].frame;
53	+	set frame.pc = completion(); set frame.status = 0x1880;
54	+	set frame.registers[2] = domains::STORE.contexts[child.initial.index].kernelStack.end;
55	+	set frame.registers[3] = kernelGp();
56	+	let now = dispatch::clock();
57	+	let index = 2 + devices::STORE.count + interrupts::STORE.count;
58	+	let budget = try! abi::handle(abi::Kind::Budget, abi::Ref { index, generation: root.memory.table.slots[index].generation });
59	+	let window = try! budgets::query(&budgets::STORE, &root.memory.table, budget);
60	+	let start = window.start;
61	+	let later = try! budgets::split(&mut budgets::STORE, &mut root.memory.table, budget, start + 10000000, now);
62	+	let running = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut root.memory.table,
63	+	budgets::Binding { budget, domain: self, context: root.initial }, now);
64	+	let transferred = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut root.memory.table,
65	+	budgets::Binding { budget: later, domain: childHandle, context: child.initial }, now);
66	+	}
67	+
68	+	/// Root exit leaves the survivor able to allocate and create new domains.
69	+	export unsafe fn verify() {
70	+	let guard = sync::enter();
71	+	let mut child = try! domains::get(&domains::STORE, CONTROLLER);
72	+	let terminal = try! events::pop(&mut child.memory.ring) else panic "missing root exit";
73	+	assert terminal.kind == events::CHILD_EXIT and terminal.code == 0 and terminal.value == abi::id(ROOT);
74	+	assert child.state == domains::Lifecycle::Active and child.parent == nil;
75	+	try! events::refresh(&mut domains::STORE.events, CONTROLLER, &child.memory.ring);
76	+	sync::leave(guard);
77	+	let page = abi::Handle(try! calls::synchronized(CONTROLLER, 30, &[0, 1, 0, 0], dispatch::clock()));
78	+	let domain = abi::Handle(try! calls::synchronized(CONTROLLER, 20, &[0, *IMAGE, 0, 0], dispatch::clock()));
79	+	let checking = sync::enter();
80	+	let allocated = try! capability::get(&child.memory.table, page);
81	+	assert (try! pages::get(&pages::STORE, allocated.object)).count == 1;
82	+	let object = (try! capability::get(&child.memory.table, domain)).object;
83	+	let created = try! domains::get(&domains::STORE, object);
84	+	assert created.creator == CONTROLLER and created.parent == CONTROLLER;
85	+	assert created.state == domains::Lifecycle::Pending;
86	+	sync::leave(checking);
87	+	finish();
88	+	}

test/bootstrap/machine.ras added +25 -0

1	+	//! Boot one hart and check user termination through production dispatch.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::setup;
5	+	call @kernel::boot::run;
6	+	.export @kernel::dispatchcheck::memory;
7	+	.export @kernel::dispatchcheck::kernelGp;
8	+	.export @kernel::dispatchcheck::completion;
9	+	.export @kernel::dispatchcheck::finish;
10	+	@kernel::dispatchcheck::memory
11	+	ret;
12	+	@kernel::dispatchcheck::kernelGp
13	+	mv %a0 %gp;
14	+	ret;
15	+	@kernel::dispatchcheck::completion
16	+	la %a0 @complete;
17	+	ret;
18	+	@complete
19	+	call @kernel::dispatchcheck::verify;
20	+	ebreak;
21	+	@kernel::dispatchcheck::finish
22	+	li %t0 0x10001000;
23	+	li %t1 0x5555;
24	+	sw %t1 0(%t0);
25	+	ebreak;

test/bootstrap/root.rad added +44 -0

1	+	//! Validate the root startup bundle from user mode.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Typed startup view used by the boot root package.
6	+	record Env: Copy {
7	+	/// Root-owned resource bundle.
8	+	argsPointer: *abi::BootInfo,
9	+	/// Byte size of the bundle header.
10	+	argsSize: u64,
11	+	/// Installed Events capability.
12	+	eventsHandle: u64,
13	+	/// Shared event-ring address.
14	+	eventsPointer: u64,
15	+	}
16	+
17	+	/// Check root identity, resource ordering, and image authority before returning.
18	+	@default unsafe fn main(env: *Env) {
19	+	assert env.argsSize == @sizeOf(abi::BootInfo) as u64;
20	+	let info = env.argsPointer;
21	+	let handles = info.handles;
22	+	let self = sys::queryDomain(abi::Handle(handles[0]));
23	+	assert (self.id & 0xffffffff) == 0 and self.creator == self.id;
24	+	assert handles[1] == env.eventsHandle and info.timebase > 0;
25	+	let mut harts: u64 = 0;
26	+	for hart in 0..8 { if (info.harts & (1 << hart as u64)) <> 0 { set harts += 1; } }
27	+	assert handles.len as u64 == 2 + info.devices + info.interrupts + harts;
28	+	for i in 0..info.devices as u32 {
29	+	let device = sys::queryDevice(abi::Handle(handles[2 + i]));
30	+	assert device.size > 0;
31	+	}
32	+	for i in 0..info.interrupts as u32 {
33	+	let irq = sys::queryInterrupt(abi::Handle(handles[2 + info.devices as u32 + i]));
34	+	assert irq.number == i + 1;
35	+	}
36	+	assert sys::queryDevice(info.clock).size == 8;
37	+	assert sys::queryDevice(info.shutdown).size >= 4;
38	+	assert (try! sys::queryImage(info.rootImage)).entry <> 0;
39	+	assert (try! sys::queryImage(info.schedulerImage)).entry <> 0;
40	+	let current = sys::currentContext();
41	+	let first = 2 + info.devices as u32 + info.interrupts as u32;
42	+	assert current.hart == 0 and current.budget == handles[first];
43	+	assert (try! sys::queryBudget(abi::Handle(handles[first]))).hart == 0;
44	+	}

test/bootstrap/run added +28 -0

1	+	#!/bin/sh
2	+	# Construct root from a boot package and retain authority after root exits.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	work=$(mktemp -d)
6	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
7	+	mkdir "$work/loaded"
8	+	cp test/bootstrap/root.rad "$work/loaded.rad"
9	+	cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/loaded/"
10	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
11	+	-pkg loaded -mod "$work/loaded.rad" -mod "$work/loaded/abi.rad" -mod "$work/loaded/sys.rad" -entry loaded -ril "$work"
12	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
13	+	-pkg service -mod test/bootstrap/service.rad -entry service -ril "$work"
14	+	cp kernel/kernel.rad "$work/kernel.rad"
15	+	printf '\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
16	+	mkdir "$work/kernel"
17	+	cp kernel/kernel/*.rad "$work/kernel/"
18	+	cp test/bootstrap/kernel/dispatchcheck.rad "$work/kernel/"
19	+	sh test/acceptance/compile "$emulator" "$work"
20	+	cat test/bootstrap/machine.ras kernel/kernel/*.ras > "$work/bootstrap.ras"
21	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
22	+	-- "$work/std.ril" "$work/kernel.ril" "$work/bootstrap.ras" "$work/bootstrap.rv64" "$work/loaded.ril" "$work/service.ril"
23	+	if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
24	+	sh test/acceptance/replay bootstrap "$emulator" "$work/bootstrap.rv64" 1
25	+	else
26	+	"$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/bootstrap.rv64"
27	+	fi
28	+	printf 'bootstrap: root exit preserves allocation and creation authority\n'

test/bootstrap/service.rad added +4 -0

1	+	//! Independent boot package used for creation after root exit.
2	+
3	+	/// Return a small status from the service entry.
4	+	@default fn main() -> u64 { return 7; }

test/dispatch/run +20 -4

work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
set -- -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad"
if [ "$fixture" = mmio ]; then
    cp kernel/mmio.rad "$work/spin/"
    set -- "$@" -mod "$work/spin/mmio.rad"
fi
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
    "$@" -entry spin -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"
cp "test/$fixture/kernel/dispatchcheck.rad" "$work/kernel/"
for source in "test/$fixture"/kernel/*.rad; do
    cp "$source" "$work/kernel/"
    module=${source##*/}
    module=${module%.rad}
    if [ "$module" != dispatchcheck ]; then
        printf '\nexport mod %s;\n' "$module" >> "$work/kernel.rad"
    fi
done
length=$(wc -c < "$work/spin.ril")
printf '//! Binary user loop.\n/// Complete trusted input package.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/dispatchinput.rad"
od -An -v -tu1 "$work/spin.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
set --
if [ "$irq" -ne 0 ]; then set -- "-irq=$irq" "-irq-at=1"; fi
for count in $harts; do
    if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
        sh test/acceptance/replay "$fixture" "$emulator" "$work/dispatch.rv64" "$count" "$@"
    else
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 "$@" -run "$work/dispatch.rv64"
    fi
    printf '%s: %s-hart execution passed\n' "$fixture" "$count"
done

test/loader/run +3 -3

# Build a kernel fixture and call code produced by its runtime loader.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev -pkg loaded -mod test/loader/loaded.rad -entry loaded -ril "$work"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev -pkg loaded -mod test/loader/loaded.rad -entry loaded -o "$work/hosted.rv64"
status=0
"$emulator" -run "$work/hosted.rv64" || status=$?
test "$status" -eq 43
cat kernel/kernel.rad > "$work/kernel.rad"

printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
printf '];\n' >> "$work/kernel/loadinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/loader/machine.ras kernel/kernel/*.ras > "$work/loader.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
printf 'runtime loader: hosted and loaded entries agree after workspace release\n'

test/mmio/kernel/dispatchcheck.rad added +131 -0

1	+	//! Native fixed-width MMIO and authority rejection before register access.
2	+	use std::mem;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::backing;
6	+	use kernel::pages;
7	+	use kernel::capability;
8	+	use kernel::registry;
9	+	use kernel::loader;
10	+	use kernel::domains;
11	+	use kernel::budgets;
12	+	use kernel::dispatch;
13	+	use kernel::boot;
14	+	use kernel::events;
15	+	use kernel::sync;
16	+	use kernel::dispatchinput;
17	+	use kernel::devices;
18	+	use kernel::platform;
19	+
20	+	/// Register storage exposed only through a synthetic Device capability.
21	+	static REGISTERS: [u64; 512] = [0; 512];
22	+
23	+	/// Bootstrap creation, allocation, and budget authority.
24	+	unsafe static TABLE: capability::Table = undefined;
25	+	/// Child domain identities retained by terminal events.
26	+	unsafe static CHILDREN: [abi::Ref; 9] = undefined;
27	+	/// Surviving event receiver and native completion checker.
28	+	unsafe static PARENT: abi::Ref = undefined;
29	+	/// Map a validated physical page.
30	+	fn memory(address: u64) -> *mut u8;
31	+	/// Current kernel package-state base.
32	+	fn kernelGp() -> u64;
33	+	/// Native entry for the completion checker.
34	+	fn completion() -> u64;
35	+	/// Signal successful completion to the machine fixture.
36	+	fn finish();
37	+
38	+	/// Give each child a disjoint window, followed by the surviving parent.
39	+	export unsafe fn setup() {
40	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
41	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
42	+	capability::initialize(&mut TABLE, owner);
43	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
44	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
45	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
46	+	});
47	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
48	+	let page = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, source)).object);
49	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
50	+	let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
51	+	let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
52	+	loader::Request { authority, source, offset: 0, length: length as u64 });
53	+	let parentHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
54	+	set PARENT = (try! capability::get(&TABLE, parentHandle)).object;
55	+	let parent = try! domains::get(&domains::STORE, PARENT);
56	+	set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;
57	+	set domains::STORE.contexts[parent.initial.index].frame.pc = completion();
58	+	set domains::STORE.contexts[parent.initial.index].frame.status = 0x1880;
59	+	set domains::STORE.contexts[parent.initial.index].frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;
60	+	set domains::STORE.contexts[parent.initial.index].frame.registers[3] = kernelGp();
61	+	let scratch = devices::STORE.count;
62	+	set devices::STORE.regions[scratch] = {
63	+	kind: platform::Kind::Uart,
64	+	info: { base: (&mut REGISTERS[0]) as u64, size: 4096 },
65	+	rights: abi::Rights(abi::READ \| abi::WRITE \| abi::GRANT \| abi::TRANSFER),
66	+	};
67	+	set devices::STORE.count += 1;
68	+	let mut clockDevice: u32 = 0;
69	+	let mut finish: u32 = 0;
70	+	for i in 0..scratch {
71	+	if devices::STORE.regions[i].kind == platform::Kind::Clint { set clockDevice = i; }
72	+	if devices::STORE.regions[i].kind == platform::Kind::Finish { set finish = i; }
73	+	}
74	+	let mut handles: [abi::Handle; 9] = undefined;
75	+	let mut contexts: [abi::Ref; 9] = undefined;
76	+	for i in 0..9 {
77	+	let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
78	+	let object = (try! capability::get(&TABLE, handle)).object;
79	+	let mut child = try! domains::get(&domains::STORE, object);
80	+	set CHILDREN[i] = object;
81	+	set handles[i] = handle;
82	+	set contexts[i] = child.initial;
83	+	try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);
84	+	let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
85	+	let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
86	+	let storage = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, stack)).object);
87	+	let mut region = scratch;
88	+	if i == 2 or i == 6 { set region = finish; }
89	+	let mut device = try! devices::install(&devices::STORE, &mut child.memory.table, region);
90	+	let slot = (try! abi::decode(device)).object.index;
91	+	if i == 2 { set child.memory.table.entries[slot].rights = abi::Rights(abi::READ); }
92	+	if i == 3 { set child.memory.table.entries[slot].rights = abi::Rights(abi::WRITE); }
93	+	if i == 1 { set device = abi::Handle(0); }
94	+	if i == 7 { set device = granted; }
95	+	let timerHandle = try! devices::install(&devices::STORE, &mut child.memory.table, clockDevice);
96	+	let args = @sliceOf(memory(storage.base) as *mut u64, 2);
97	+	set args[0] = device; set args[1] = timerHandle;
98	+	let size = 16 + i as u64;
99	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, size);
100	+	}
101	+	let timer = try! dispatch::timer(&boot::PLATFORM, 0);
102	+	let clock = dispatch::now(timer);
103	+	let start = clock + 1000000;
104	+	let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 20000000);
105	+	for i in 0..9 {
106	+	let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, remaining, start + (i as u64 + 1) * 2000000, clock);
107	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
108	+	budgets::Binding { budget: remaining, domain: handles[i], context: contexts[i] }, clock);
109	+	set remaining = next;
110	+	}
111	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
112	+	budgets::Binding { budget: remaining, domain: parentHandle, context: parent.initial }, clock);
113	+	}
114	+
115	+	/// Check each terminal kind, status, and generation while the parent continues to run.
116	+	export unsafe fn verify() {
117	+	let guard = sync::enter();
118	+	let mut parent = try! domains::get(&domains::STORE, PARENT);
119	+	let codes = [61 as u32, 256, 256, 256, 256, 256, 256, 256, 256];
120	+	for i in 0..9 {
121	+	let event = try! events::pop(&mut parent.memory.ring) else panic "missing termination event";
122	+	assert event.value == abi::id(CHILDREN[i]) and event.code == codes[i];
123	+	if i == 0 { assert event.kind == events::CHILD_EXIT; }
124	+	else { assert event.kind == events::FAULT; }
125	+	assert domains::STORE.records[CHILDREN[i].index].state == domains::Lifecycle::Dead;
126	+	}
127	+	try! events::refresh(&mut domains::STORE.events, PARENT, &parent.memory.ring);
128	+	assert try! events::pop(&mut parent.memory.ring) == nil;
129	+	sync::leave(guard);
130	+	finish();
131	+	}

test/mmio/machine.ras added +25 -0

1	+	//! Boot one hart and check user termination through production dispatch.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::setup;
5	+	call @kernel::boot::run;
6	+	.export @kernel::dispatchcheck::memory;
7	+	.export @kernel::dispatchcheck::kernelGp;
8	+	.export @kernel::dispatchcheck::completion;
9	+	.export @kernel::dispatchcheck::finish;
10	+	@kernel::dispatchcheck::memory
11	+	ret;
12	+	@kernel::dispatchcheck::kernelGp
13	+	mv %a0 %gp;
14	+	ret;
15	+	@kernel::dispatchcheck::completion
16	+	la %a0 @complete;
17	+	ret;
18	+	@complete
19	+	call @kernel::dispatchcheck::verify;
20	+	ebreak;
21	+	@kernel::dispatchcheck::finish
22	+	li %t0 0x10001000;
23	+	li %t1 0x5555;
24	+	sw %t1 0(%t0);
25	+	ebreak;

test/mmio/spin.rad added +51 -0

1	+	//! User-mode register widths and rejected MMIO authority.
2	+	export mod abi;
3	+	export mod sys;
4	+	export mod mmio;
5	+
6	+	/// Device values installed in the calling domain's table.
7	+	record Arguments: Copy {
8	+	/// Register interface under test.
9	+	device: abi::Handle,
10	+	/// Read-only machine timer.
11	+	clock: abi::Handle,
12	+	}
13	+
14	+	/// Case selection and readable arguments supplied by the test parent.
15	+	record Env: Copy {
16	+	/// Installed device handles.
17	+	argsPointer: *Arguments,
18	+	/// Sixteen plus the test case number.
19	+	argsSize: u64,
20	+	/// Installed event capability.
21	+	eventsHandle: u64,
22	+	/// Shared ring address.
23	+	eventsPointer: u64,
24	+	}
25	+
26	+	/// Execute each width, or require a kernel fault before an unauthorized access.
27	+	@default fn main(env: *Env) {
28	+	let device = env.argsPointer.device;
29	+	let scenario = env.argsSize - 16;
30	+	if scenario == 0 {
31	+	let before = mmio::read64(env.argsPointer.clock, 0);
32	+	mmio::write8(device, 0, 0x81);
33	+	mmio::write16(device, 2, 0x8123);
34	+	mmio::write32(device, 4, 0x81234567);
35	+	mmio::write64(device, 8, 0x8123456789abcdef);
36	+	assert mmio::read8(device, 0) == 0x81;
37	+	assert mmio::read16(device, 2) == 0x8123;
38	+	assert mmio::read32(device, 4) == 0x81234567;
39	+	assert mmio::read64(device, 8) == 0x8123456789abcdef;
40	+	assert mmio::read32(env.argsPointer.clock, 0) as u64 >= before;
41	+	sys::exit(61);
42	+	}
43	+	if scenario == 1 or scenario == 2 { mmio::write32(device, 0, 0x5555); }
44	+	if scenario == 3 { let value = mmio::read8(device, 0); }
45	+	if scenario == 4 { mmio::write32(device, 1, 1); }
46	+	if scenario == 5 { let value = mmio::read8(device, 4096); }
47	+	if scenario == 6 { try! sys::capabilityDrop(device); mmio::write32(device, 0, 0x5555); }
48	+	if scenario == 7 { let value = mmio::read8(device, 0); }
49	+	if scenario == 8 { mmio::write64(device, 0xfffffffffffffff8, 1); }
50	+	sys::exit(99);
51	+	}

test/modules/run +1 -1

    set -- "$@" -mod "$work/capacity/m$index.rad"
    index=$((index + 1))
done
printf 'use capacity::m126;\n/// Check a package root beyond module index 127.\n@default fn main() -> u64 { assert m126::value() == 126; return 0; }\n' > "$work/entry.rad"
set -- "$@" -pkg entry -mod "$work/entry.rad"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev "$@" -entry entry -o "$work/capacity.rv64"
"$emulator" -run "$work/capacity.rv64"
printf 'module scopes: references beyond index 127 passed\n'

test/packages/run +2 -2

emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/first" "$work/second"
for output in first second; do
    "$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    "$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
        -run bin/radiance.rv64.dev \
        -pkg support -mod test/packages/support.rad \
        -pkg app -mod test/packages/app.rad -entry app -ril "$work/$output"
done
cmp "$work/first/support.ril" "$work/second/support.ril"
cmp "$work/first/app.ril" "$work/second/app.ril"
"$emulator" -run bin/packages.check.rv64 -- "$work/first/support.ril" "$work/first/app.ril"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev \
    -pkg support -mod test/packages/support.rad \
    -pkg app -mod test/packages/app.rad -entry app -o "$work/hosted.rv64"
status=0
"$emulator" -run "$work/hosted.rv64" || status=$?

test/runtime/run +8 -4

work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
cp test/loader/loaded.rad "$work/loaded.rad"
index=0
while [ "$index" -lt 128 ]; do
    printf '\n/// Compute a distinct exported value.\nexport fn value%s(input: u64) -> u64 { return input + %s; }\n' "$index" "$index" >> "$work/loaded.rad"
    index=$((index + 1))
done
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
    -pkg loaded -mod "$work/loaded.rad" -entry loaded -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"

printf '\n/// Package compiled by scheduled user calls.\nexport static PAYLOAD: [u8; %s] = [\n' "$length" >> "$work/kernel/dispatchinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
"$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
    sh test/acceptance/replay runtime "$emulator" "$work/dispatch.rv64" 1
else
    "$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
fi
printf 'runtime: scheduled cancellation, retry, and publication passed\n'

test/scheduling/kernel/dispatchcheck.rad added +68 -0

1	+	//! Observe user-space bootstrap and scheduling at the trap boundary.
2	+	use kernel::abi;
3	+	use std::lang::il;
4	+	use kernel::trap;
5	+	use kernel::dispatch;
6	+	use kernel::domains;
7	+	use kernel::budgets;
8	+	use kernel::events;
9	+	use kernel::sync;
10	+
11	+	/// Root has requested normal termination.
12	+	static ROOT_EXITED: bool = false;
13	+	/// Workers that returned after root released its authority.
14	+	static WORKERS: u32 = 0;
15	+	/// Write one byte to the test UART.
16	+	fn put(value: u8);
17	+	/// Most recent user operation for fault diagnosis.
18	+	static LAST_CALL: u64 = 0;
19	+
20	+	/// Print a complete hexadecimal diagnostic word.
21	+	fn number(value: u64) {
22	+	let digits = "0123456789abcdef";
23	+	for i in 0..16 { put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]); }
24	+	put(10);
25	+	}
26	+
27	+	/// Check terminal calls without changing user-domain resources or execution.
28	+	export unsafe fn observe(frame: &trap::Frame) {
29	+	if not trap::fromUser(frame) { return; }
30	+	let cause = trap::classify(frame.cause);
31	+	if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External { return; }
32	+	if cause <> trap::Cause::UserCall {
33	+	number(frame.cause); number(frame.pc); number(LAST_CALL); number(WORKERS as u64);
34	+	assert false;
35	+	}
36	+	set LAST_CALL = frame.registers[17];
37	+	assert frame.registers[17] <> abi::Operation::Abort as u64;
38	+	let shutdown = frame.registers[17] == il::DEVICE_ACCESS as u64 and frame.registers[12] == 0x104;
39	+	if frame.registers[17] <> abi::Operation::Exit as u64 and not shutdown { return; }
40	+	let current = dispatch::current();
41	+	let context = abi::reference(current.context);
42	+	let guard = sync::enter();
43	+	let owner = domains::STORE.contexts[context.index].owner;
44	+	if owner.index == 0 {
45	+	assert frame.registers[10] == 0;
46	+	set ROOT_EXITED = true;
47	+	} else if owner.index == 1 {
48	+	assert ROOT_EXITED and WORKERS == 4 and shutdown and frame.registers[13] == 0x5555;
49	+	let scheduler = try! domains::get(&domains::STORE, owner);
50	+	assert scheduler.parent == nil;
51	+	let ring = &scheduler.memory.ring;
52	+	assert ring.tail - ring.head == 5;
53	+	assert ring.data[ring.head & ring.mask].kind == events::CHILD_EXIT;
54	+	assert ring.data[ring.head & ring.mask].code == 0;
55	+	for i in 1..5 {
56	+	let event = ring.data[(ring.head + i) & ring.mask];
57	+	assert event.kind == events::CHILD_EXIT and event.code == 7;
58	+	}
59	+	let future = try! budgets::query(&budgets::STORE, &scheduler.memory.table, abi::Handle(current.budget));
60	+	assert future.end > dispatch::clock();
61	+	sync::leave(guard);
62	+	return;
63	+	} else {
64	+	assert ROOT_EXITED and frame.registers[10] == 7;
65	+	set WORKERS += 1;
66	+	}
67	+	sync::leave(guard);
68	+	}

test/scheduling/machine.ras added +10 -0

1	+	//! Run production boot and finish after user-space scheduling checks.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::boot::launch;
5	+	ebreak;
6	+	.export @kernel::dispatchcheck::put;
7	+	@kernel::dispatchcheck::put
8	+	li %t0 0x10000000;
9	+	sb %a0 0(%t0);
10	+	ret;

test/scheduling/run added +25 -0

1	+	#!/bin/sh
2	+	# Execute root handoff and successive user-space yields with the production packages.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	harts=${1:-1 2 8}
6	+	work=$(mktemp -d)
7	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
8	+	cp kernel/kernel.rad "$work/kernel.rad"
9	+	printf '\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
10	+	mkdir "$work/kernel"
11	+	cp kernel/kernel/*.rad "$work/kernel/"
12	+	cp test/scheduling/kernel/dispatchcheck.rad "$work/kernel/"
13	+	sed -i '/let cause = trap::classify(frame.cause);/a\ super::dispatchcheck::observe(frame);' "$work/kernel/boot.rad"
14	+	sh test/acceptance/compile "$emulator" "$work"
15	+	cat test/scheduling/machine.ras kernel/kernel/*.ras > "$work/scheduling.ras"
16	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
17	+	-- "$work/std.ril" "$work/kernel.ril" "$work/scheduling.ras" "$work/scheduling.rv64" bin/root.ril bin/scheduler.ril
18	+	for count in $harts; do
19	+	if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
20	+	sh test/acceptance/replay scheduling "$emulator" "$work/scheduling.rv64" "$count"
21	+	else
22	+	"$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 -run "$work/scheduling.rv64"
23	+	fi
24	+	printf 'scheduling: %s-hart root handoff, four user-space yields, and checked shutdown passed\n' "$count"
25	+	done

test/shared/run +1 -1

# Compile two packages once and execute them with two private state graphs.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev \
    -pkg support -mod test/shared/support.rad \
    -pkg app -mod test/shared/app.rad -entry app -ril "$work"
"$emulator" -run bin/shared.build.rv64 -- "$work/support.ril" "$work/app.ril" "$work/native.rv64"
"$emulator" -machine -run "$work/native.rv64"

test/slots/run +1 -1

for name in duplicate abandon; do
    case "$name" in
        duplicate) diagnostic='linear value used after consumption' ;;
        abandon) diagnostic='linear value is not consumed' ;;
    esac
    if "$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    if "$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
        -run bin/radiance.rv64.dev "$@" -test \
        -pkg "$name" -mod "test/slots/$name.rad" -entry "$name" \
        -o "$work/test.rv64" > "$work/log" 2>&1; then
        printf 'reservation misuse compiled: %s\n' "$name" >&2
        exit 1

test/termination/kernel/dispatchcheck.rad +27 -11

use kernel::dispatch;
use kernel::boot;
use kernel::events;
use kernel::sync;
use kernel::dispatchinput;
use kernel::devices;
use kernel::platform;

/// Bootstrap creation, allocation, and budget authority.
unsafe static TABLE: capability::Table = undefined;
/// Child domain identities retained by terminal events.
static CHILDREN: [abi::Ref; 5] = undefined;
unsafe static CHILDREN: [abi::Ref; 6] = undefined;
/// Surviving event receiver and native completion checker.
unsafe static PARENT: abi::Ref = undefined;
/// Map a validated physical page.
fn memory(address: u64) -> *mut u8;
/// Current kernel package-state base.

/// Native entry for the completion checker.
fn completion() -> u64;
/// Signal successful completion to the machine fixture.
fn finish();

/// Give four children disjoint windows, followed by the surviving parent.
/// Give each child a disjoint window, followed by the surviving parent.
export unsafe fn setup() {
    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
    let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
    capability::initialize(&mut TABLE, owner);
    try! backing::registerDomain(&mut pages::STORE.backings, owner);

    set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;
    set domains::STORE.contexts[parent.initial.index].frame.pc = completion();
    set domains::STORE.contexts[parent.initial.index].frame.status = 0x1880;
    set domains::STORE.contexts[parent.initial.index].frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;
    set domains::STORE.contexts[parent.initial.index].frame.registers[3] = kernelGp();
    let mut handles: [abi::Handle; 5] = undefined;
    let mut contexts: [abi::Ref; 5] = undefined;
    for i in 0..5 {
    let mut handles: [abi::Handle; 6] = undefined;
    let mut contexts: [abi::Ref; 6] = undefined;
    for i in 0..6 {
        let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
        let object = (try! capability::get(&TABLE, handle)).object;
        let mut child = try! domains::get(&domains::STORE, object);
        set CHILDREN[i] = object;
        set handles[i] = handle;
        set contexts[i] = child.initial;
        try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);
        let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
        let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, stack, (abi::READ | abi::WRITE) as u64);
        let storage = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, stack)).object);
        try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, i as u64);
        let mut size = i as u64;
        if i == 5 {
            let mut installed = false;
            for index in 0..devices::STORE.count {
                if devices::STORE.regions[index].kind <> platform::Kind::Finish { continue; }
                let device = try! devices::install(&devices::STORE, &mut child.memory.table, index);
                let args = memory(storage.base) as *mut u64;
                set *args = *device;
                set installed = true;
                break;
            }
            assert installed;
            set size = 8;
        }
        try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, size);
    }
    let timer = try! dispatch::timer(&boot::PLATFORM, 0);
    let clock = dispatch::now(timer);
    let start = clock + 1000000;
    let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 12000000);
    for i in 0..5 {
    let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 14000000);
    for i in 0..6 {
        let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, remaining, start + (i as u64 + 1) * 2000000, clock);
        let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
            budgets::Binding { budget: remaining, domain: handles[i], context: contexts[i] }, clock);
        set remaining = next;
    }


/// Check each terminal kind, status, and generation while the parent continues to run.
export unsafe fn verify() {
    let guard = sync::enter();
    let mut parent = try! domains::get(&domains::STORE, PARENT);
    let codes = [37 as u32, abi::FAULT_ABORT, 3, 0, abi::FAULT_ABORT];
    for i in 0..5 {
    let codes = [37 as u32, abi::FAULT_ABORT, 3, 0, abi::FAULT_ABORT, 51];
    for i in 0..6 {
        let event = try! events::pop(&mut parent.memory.ring) else panic "missing termination event";
        assert event.value == abi::id(CHILDREN[i]) and event.code == codes[i];
        if i == 0 or i == 3 { assert event.kind == events::CHILD_EXIT; }
        if i == 0 or i == 3 or i == 5 { assert event.kind == events::CHILD_EXIT; }
        else { assert event.kind == events::FAULT; }
        assert domains::STORE.records[CHILDREN[i].index].state == domains::Lifecycle::Dead;
    }
    try! events::refresh(&mut domains::STORE.events, PARENT, &parent.memory.ring);
    assert try! events::pop(&mut parent.memory.ring) == nil;

test/termination/spin.rad +6 -1

@intrinsic fn ecall(operation: u32, a0: i64, a1: i64, a2: i64, a3: i64) -> i64;

/// Startup values used to select the termination operation.
record Env: Copy {
    /// Optional argument bytes supplied by the parent.
    argsPointer: *u8,
    argsPointer: *abi::Handle,
    /// Termination case selected by the parent.
    argsSize: u64,
    /// Installed event handle.
    eventsHandle: u64,
    /// Shared event-ring address.

    if env.argsSize == 3 { try! sys::domainDestroy(abi::Handle(0), 0); }
    if env.argsSize == 4 {
        let result = ecall(47, 0, 0, 0, 0);
        sys::exit(99);
    }
    if env.argsSize == 8 {
        let device = sys::queryDevice(*env.argsPointer);
        assert device.base == 0x10001000 and device.size >= 4;
        sys::exit(51);
    }
    assert false;
}

226	235		# Scheduled compilation, cancellation, retry, and shared executable publication.
227	236		runtime-test: emulator $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
228	237		@RAD_EMULATOR="$(EMU)" sh test/runtime/run
229	238
230	239		.PHONY: runtime-test
	240	+
	241	+	# Root lifetime and resource authority across the boot-package handoff.
	242	+	bootstrap-test: emulator $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	243	+	@RAD_EMULATOR="$(EMU)" sh test/bootstrap/run
	244	+
	245	+	.PHONY: bootstrap-test
	246	+
	247	+	# Production root and scheduler packages execute all authority changes in user mode.
	248	+	.PHONY: scheduling-test
	249	+	scheduling-test: $(BIN_DIR)/kernel.rv64
	250	+	@RAD_EMULATOR="$(EMU)" sh test/scheduling/run
	251	+
	252	+	# Checked user-mode register access, including rejection before device effects.
	253	+	.PHONY: mmio-test
	254	+	mmio-test: $(RAD_BIN) $(BIN_DIR)/kernel.build.rv64
	255	+	@RAD_EMULATOR="$(EMU)" sh test/dispatch/run mmio

229	238		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
230	239		let page = try pages::get(&pages::STORE, entry.object);
231	240		try reply(&domain, arguments, &[page.base, page.count as u64]);
232	241		return 0;
233	242		},
	243	+	case 45 => {
	244	+	let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Device, abi::Rights(0));
	245	+	let device = try devices::get(&devices::STORE, entry.object);
	246	+	try reply(&domain, arguments, &[device.info.base, device.info.size]);
	247	+	return 0;
	248	+	},
234	249		case 46 => {
235	250		let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
236	251		if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
237	252		or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) { throw abi::Error::InvalidArg; }
238	253		set *memory32(arguments[1]) = number;

519	534		assert trap::fromUser(frame) and frame.cause == 8;
520	535		assert trap::advanceCall(frame);
521	536		let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
522	537		let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
523	538		let operation = frame.registers[17];
524		-	if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
	539	+	if operation == il::DEVICE_ACCESS or (operation >= 44 and operation <= 48 and error == abi::Error::BadHandle) {
525	540		try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
526	541		assert stopped == abi::Error::BadHandle;
527	542		};
528	543		}
529	544		set frame.registers[10] = (0 as u64) - error as u64;

622	622		match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
623	623		}
624	624
625	625		/// Initialize retained domain storage outside the metadata lock.
626	626		export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
627		-	let mut storage = creation.storage;
	627	+	try prepareStorage(packages, creation.storage, creation.domain);
	628	+	}
	629	+
	630	+	/// Initialize one domain's private allocations before capability publication.
	631	+	unsafe fn prepareStorage(packages: &registry::Store, storage: Prepared, owner: abi::Ref) throws (abi::Error) {
	632	+	let mut storage = storage;
628	633		zero(storage.memory as u64, storage.allocation.count);
629	634		zero(storage.kernelStack.start, storage.kernelFrames.count);
630	635		try instances::fill(packages, &storage.graph.graph, storage.graph.base);
631		-	capability::initialize(&mut storage.memory.table, creation.domain);
	636	+	capability::initialize(&mut storage.memory.table, owner);
632	637		}
633	638
634	639		/// Check the unpublished capacity retained by one creation transaction.
635	640		fn require(store: &Store, table: &capability::Table, creation: &Creation) {
636	641		assert table.owner == creation.owner;

649	654		release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
650	655		{
651	656		match reservation {
652	657		case Reservation::Held(creation) => {
653	658		require(store, table, &creation);
	659	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
654	660		discard(&mut backings.pool, creation.storage, release);
655	661		try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
656	662		try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
657		-	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
658	663		},
659	664		}
660	665		}
661	666
662	667		/// Publish an initialized domain and its Events under metadata serialization.

13	13
14	14		# Emulator command used to invoke the self-hosted compiler.
15	15		EMU := $(or $(RAD_EMULATOR),emulator)
16	16		EMU_FLAGS := -memory-size=385024 \
17	17		-data-size=348160 \
18		-	-stack-size=512 \
	18	+	-stack-size=1024 \
19	19		-count-instructions
20	20		RADIANCE := $(EMU) $(EMU_FLAGS) -run $(RAD_BIN)
21	21
22	22		# Verify the emulator binary exists.
23	23		EMU_PATH := $(shell command -v $(EMU) 2>/dev/null)
24	24
25	25		default: emulator $(RAD_BIN)
26		-	test: emulator seed-test std-test bin-test kernel-test module-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test termination-test runtime-test
	26	+	test: emulator seed-test std-test bin-test kernel-test module-test package-test native-test shared-test sync-test kernel-boot-test trap-test page-test loader-test dispatch-test smp-test termination-test runtime-test bootstrap-test mmio-test scheduling-test
27	27
28	28		seed-test:
29	29		@seed/test
30	30
31	31		module-test: emulator $(RAD_BIN)

92	92		@$(RADIANCE) $(STD) $(KERNEL) -entry kernel -ril $(BIN_DIR)
93	93
94	94		$(BIN_DIR)/kernel.build.rv64: kernel/tools/build.rad $(STD_LIB) $(RAD_BIN)
95	95		@$(RADIANCE) $(STD) -pkg build -mod $< -entry build -o $@
96	96
97		-	$(BIN_DIR)/kernel.rv64: $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.build.rv64 $(KERNEL_ASM)
	97	+	# User packages share the ABI source through a staged package module tree.
	98	+	$(BIN_DIR)/root.ril $(BIN_DIR)/scheduler.ril &: kernel/root.rad kernel/scheduler.rad kernel/mmio.rad kernel/kernel/abi.rad kernel/kernel/sys.rad $(RAD_BIN)
	99	+	@mkdir -p $(BIN_DIR)/user/root $(BIN_DIR)/user/scheduler
	100	+	@cp kernel/root.rad kernel/scheduler.rad $(BIN_DIR)/user/
	101	+	@cp kernel/kernel/abi.rad kernel/kernel/sys.rad $(BIN_DIR)/user/root/
	102	+	@cp kernel/kernel/abi.rad kernel/kernel/sys.rad kernel/mmio.rad $(BIN_DIR)/user/scheduler/
	103	+	@$(RADIANCE) -pkg root -mod $(BIN_DIR)/user/root.rad -mod $(BIN_DIR)/user/root/abi.rad -mod $(BIN_DIR)/user/root/sys.rad -entry root -ril $(BIN_DIR)
	104	+	@$(RADIANCE) -pkg scheduler -mod $(BIN_DIR)/user/scheduler.rad -mod $(BIN_DIR)/user/scheduler/abi.rad -mod $(BIN_DIR)/user/scheduler/sys.rad -mod $(BIN_DIR)/user/scheduler/mmio.rad -entry scheduler -ril $(BIN_DIR)
	105	+
	106	+	$(BIN_DIR)/kernel.rv64: $(BIN_DIR)/root.ril $(BIN_DIR)/scheduler.ril $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.build.rv64 $(KERNEL_ASM)
98	107		@cat $(KERNEL_ASM) > $(BIN_DIR)/kernel.ras
99		-	@$(EMU) $(EMU_FLAGS) -run $(BIN_DIR)/kernel.build.rv64 -- $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.ras $@
	108	+	@$(EMU) $(EMU_FLAGS) -run $(BIN_DIR)/kernel.build.rv64 -- $(BIN_DIR)/std.ril $(BIN_DIR)/kernel.ril $(BIN_DIR)/kernel.ras $@ $(BIN_DIR)/root.ril $(BIN_DIR)/scheduler.ril
100	109
101	110		# Separate binary package emission and decode checks.
102	111		package-test: emulator $(BIN_DIR)/packages.check.rv64
103	112		@sh test/packages/run
104	113

40	40
41	41		/// AST arena size (64 MB) - retains parsed nodes throughout compilation.
42	42		constant TEMP_ARENA_SIZE: u32 = 67108864;
43	43		/// Per-function lowering and register-allocation arena size (16 MB).
44	44		constant FN_ARENA_SIZE: u32 = 16777216;
45		-	/// Main arena size (128 MB) - lives throughout compilation.
	45	+	/// Main arena size (160 MB) - lives throughout compilation.
46	46		/// Used for: resolver data, types, symbols, global IL data, and codegen output.
47		-	constant MAIN_ARENA_SIZE: u32 = 134217728;
	47	+	constant MAIN_ARENA_SIZE: u32 = 167772160;
48	48
49	49		/// AST storage arena.
50	50		static TEMP_ARENA: [u8; TEMP_ARENA_SIZE] = [0; TEMP_ARENA_SIZE];
51	51		/// Scratch storage reclaimed after each generated function.
52	52		static FN_ARENA: [u8; FN_ARENA_SIZE] = [0; FN_ARENA_SIZE];

12	12		export mod trap;
13	13		export mod capability;
14	14		export mod transactions;
15	15		export mod events;
16	16		export mod interrupts;
	17	+	export mod devices;
17	18		export mod plic;
18	19		export mod registry;
19	20		export mod frames;
20	21		export mod backing;
21	22		export mod pages;

28	29		export mod timers;
29	30		export mod remote;
30	31		export mod calls;
31	32		export mod sys;
32	33		export mod boot;
	34	+	export mod startup;
33	35		@test export mod tests;

15	15		eventsHandle: u64,
16	16		/// Address of the domain's shared event ring.
17	17		eventsPointer: u64,
18	18		}
19	19
	20	+	/// Initial root resources copied into root-owned startup memory.
	21	+	export record BootInfo: Copy {
	22	+	/// Number of Device handles after the self and Events slots.
	23	+	devices: u64,
	24	+	/// Number of Interrupt handles following the Device handles.
	25	+	interrupts: u64,
	26	+	/// Bit mask of online harts, whose Budget handles follow in hart order.
	27	+	harts: u64,
	28	+	/// Machine timer ticks per second.
	29	+	timebase: u64,
	30	+	/// Image used to construct root and additional instances of its program.
	31	+	rootImage: Handle,
	32	+	/// Initial user-space scheduler image.
	33	+	schedulerImage: Handle,
	34	+	/// Read-only machine-time device, or zero when absent.
	35	+	clock: Handle,
	36	+	/// Platform shutdown device, or zero when absent.
	37	+	shutdown: Handle,
	38	+	/// Page that owns this header and the copied handle array.
	39	+	arguments: Handle,
	40	+	/// Copied capability values in their initial slot order.
	41	+	handles: *unsafe mut [u64],
	42	+	}
	43	+
20	44		/// Physical extent returned by QueryPage.
21	45		export record PageInfo: Copy {
22	46		/// Address of the first physical frame.
23	47		base: u64,
24	48		/// Number of contiguous physical frames.
25	49		count: u64,
26	50		}
27	51
	52	+	/// Physical MMIO extent returned by QueryDevice.
	53	+	export record DeviceInfo: Copy {
	54	+	/// Physical base address of the register region.
	55	+	base: u64,
	56	+	/// Size of the register region in bytes.
	57	+	size: u64,
	58	+	}
	59	+
28	60		/// Physical source returned by QueryInterrupt.
29	61		export record InterruptInfo: Copy {
30	62		/// PLIC source number; zero does not name an interrupt.
31	63		number: u32,
32	64		}

1	1		//! Machine entry boundary and published platform state.
2	2
3	3		use super::platform;
	4	+	use super::startup;
4	5		use super::fdt;
5	6		use super::range;
6	7		use super::limits;
7	8		use super::sync;
8	9		use super::trap;

13	14		use super::budgets;
14	15		use super::dispatch;
15	16		use super::timers;
16	17		use super::remote;
17	18		use super::interrupts;
	19	+	use super::devices;
18	20		use super::plic;
19	21		use std::arch::rv64::shared::catalog;
20	22
21	23		/// Platform data published by hart zero before secondary initialization.
22	24		export unsafe static PLATFORM: platform::Platform = undefined;
23	25		/// Release/acquire publication flag for PLATFORM.
24	26		static READY: u64 = 0;
25	27		/// Number of harts that validated their machine stack.
26	28		static ARRIVED: u64 = 0;
	29	+	/// Root resource publication before user dispatch begins.
	30	+	static STARTED: u64 = 0;
27	31		/// Trap anchors owned by their physical harts.
28	32		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
29	33
30	34		/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
31	35		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {

87	91		budgets::initialize(&mut budgets::STORE);
88	92		timers::initialize(&mut timers::STORE);
89	93		remote::initialize(&mut remote::STORE, PLATFORM.harts);
90	94		interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
91	95		try! plic::initialize(&PLATFORM);
92		-	assert entryCount == 2;
	96	+	try! devices::initialize(&mut devices::STORE, &PLATFORM);
	97	+	assert entryCount >= 2 and entryCount <= limits::PACKAGES;
93	98		registry::initialize(&mut registry::STORE);
94	99		let bootCatalog = @sliceOf(entries, entryCount);
95	100		try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
96	101		let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";
97	102		let kernel = registry::find(&registry::STORE, &"kernel"[..]) else panic "boot kernel";

117	122		let mut count: u64 = 0;
118	123		for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
119	124		return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
120	125		}
121	126
	127	+	/// Publish configured root resources once, then enter dispatch on every hart.
	128	+	export unsafe fn launch(last: bool) -> ! {
	129	+	if last {
	130	+	if let root = registry::find(&registry::STORE, &"root"[..]) {
	131	+	let scheduler = registry::find(&registry::STORE, &"scheduler"[..]) else panic "missing scheduler image";
	132	+	let domain = try! startup::create(&PLATFORM, root, scheduler);
	133	+	print("kernel: root ready\n");
	134	+	}
	135	+	sync::storeRelease(&mut STARTED, 1);
	136	+	} else {
	137	+	while sync::loadAcquire(&STARTED) == 0 {}
	138	+	}
	139	+	run();
	140	+	}
	141	+
122	142		/// Enter dispatch after every online hart publishes its initialized state.
123	143		export unsafe fn run() -> ! {
124	144		let mut count: u64 = 0;
125	145		for id in 0..limits::HARTS { if (PLATFORM.harts & (1 << id)) <> 0 { set count += 1; } }
126	146		while sync::loadAcquire(&ARRIVED) < count {}

5	5		.export @kernel::boot::read8;
6	6		.export @kernel::boot::write8;
7	7
8	8		@kernel::boot::start
9	9		call @kernel::boot::initialize;
10		-	call @kernel::boot::run;
	10	+	call @kernel::boot::launch;
11	11		ebreak;
12	12
13	13		// Initialize machine CSRs and preserve the firmware stack across Radiance entry.
14	14		@kernel::boot::initialize
15	15		csrw mie %zero;

34	34		@kernel::boot::write8
35	35		fence iorw iorw;
36	36		sb %a1 0(%a0);
37	37		fence iorw iorw;
38	38		ret;
	39	+
	40	+	.export @kernel::startup::info;
	41	+	.export @kernel::startup::words;
	42	+	// Resolve root-owned startup memory after physical allocation.
	43	+	@kernel::startup::info
	44	+	ret;
	45	+	@kernel::startup::words
	46	+	ret;

1	1		//! Direct-call execution and checked copies into domain-owned memory.
2	2
3	3		use super::abi;
	4	+	use std::lang::il;
4	5		use super::slots;
5	6		use super::limits;
6	7		use super::capability;
7	8		use super::pages;
8	9		use super::domains;

50	52		match entry.kind {
51	53		case abi::Kind::Domain, abi::Kind::Events => {
52	54		let object = try domains::get(&domains::STORE, entry.object);
53	55		if object.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
54	56		},
	57	+	case abi::Kind::Device => { let object = try devices::get(&devices::STORE, entry.object); },
55	58		case abi::Kind::Image => { let object = try registry::get(&registry::STORE, entry.object); },
56	59		else => throw abi::Error::BadHandle,
57	60		}
58	61		}
59	62

191	194		let handle = abi::Handle(arguments[0]);
192	195		try allocationAuthority(&domain.memory.table, operation, handle);
193	196		match operation {
194	197		case 22, 43, 49 => { let stopped = try termination(owner, operation, arguments); return 0; },
195	198		case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
	199	+	case il::DEVICE_ACCESS => {
	200	+	let flags = arguments[2];
	201	+	if (flags & ~0x1ff) <> 0 { throw abi::Error::InvalidArg; }
	202	+	return try devices::address(&devices::STORE, &domain.memory.table, handle,
	203	+	arguments[1], flags & 0xff, (flags & 0x100) <> 0);
	204	+	},
196	205		case 12 => { try drop(&mut domain, handle); return 0; },
197	206		case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,
198	207		&registry::STORE, &mut domain.memory.table, handle, abi::Handle(arguments[1])),
199	208		case 21 => {
200	209		try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,

664	669		-> abi::Handle throws (abi::Error)
665	670		{
666	671		match reservation {
667	672		case Reservation::Held(creation) => {
668	673		require(store, table, &creation);
669		-	if not backing::domainLive(backings, creation.owner) {
	674	+	let root = creation.owner == creation.domain;
	675	+	if not root and not backing::domainLive(backings, creation.owner) {
670	676		cancel(store, backings, table, Reservation::Held(creation));
671	677		throw abi::Error::BadHandle;
672	678		}
673	679		let object = creation.domain;
674	680		let initial = creation.context;

692	698		owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
693	699		userStack: range::Range { start: 0, end: 0 }, frame,
694	700		};
695	701		set store.reclaimed[object.index] = false;
696	702		set store.reaping[object.index] = false;
	703	+	let mut parent: ?abi::Ref = nil;
	704	+	if not root { set parent = creation.owner; }
697	705		set store.records[object.index] = Domain {
698		-	state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
	706	+	state: Lifecycle::Pending, creator: creation.owner, parent, image: creation.image, initial,
699	707		allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
700	708		};
701		-	set store.ancestors[object.index] = store.ancestors[creation.owner.index];
702		-	set store.ancestors[object.index][creation.owner.index / 32] \|= 1 << (creation.owner.index % 32);
	709	+	if not root {
	710	+	set store.ancestors[object.index] = store.ancestors[creation.owner.index];
	711	+	set store.ancestors[object.index][creation.owner.index / 32] \|= 1 << (creation.owner.index % 32);
	712	+	}
703	713		let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
704	714		let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
705	715		return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
706	716		kind: abi::Kind::Domain, object: domain, rights: creation.rights,
707	717		});
708	718		},
709	719		}
710	720		}
711	721
	722	+	/// Create the first domain with its self capability in slot zero and Events in slot one.
	723	+	/// Boot calls this before any domain, context, or exposure membership is published.
	724	+	export unsafe fn bootstrap(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store, image: abi::Ref)
	725	+	-> abi::Ref throws (abi::Error)
	726	+	{
	727	+	for slot in store.slots { if slot.state <> slots::State::Free { throw abi::Error::Busy; } }
	728	+	let package = try registry::get(packages, image);
	729	+	let entry = package.entry else { throw abi::Error::VerifyFailed; };
	730	+	let domainSlot = try slots::reserve(&mut store.slots[..]);
	731	+	let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch error {
	732	+	try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
	733	+	};
	734	+	let object = slots::reference(&domainSlot);
	735	+	let mut storage = try allocate(&mut backings.pool, packages, image) catch error {
	736	+	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
	737	+	try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
	738	+	};
	739	+	try prepareStorage(packages, storage, object) catch error {
	740	+	discard(&mut backings.pool, storage, frames::release);
	741	+	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
	742	+	try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
	743	+	};
	744	+	let handleSlot = try! slots::reserve(&mut storage.memory.table.slots[..]);
	745	+	assert slots::reference(&handleSlot).index == 0;
	746	+	match domainSlot {
	747	+	case slots::Reservation::Held(domain) => match contextSlot {
	748	+	case slots::Reservation::Held(context) => match handleSlot {
	749	+	case slots::Reservation::Held(handle) => {
	750	+	let result = try publish(store, backings, &mut storage.memory.table, Reservation::Held(Creation {
	751	+	owner: object, handle, domain, context,
	752	+	rights: abi::Rights(MANAGEMENT_RIGHTS \| abi::CREATE \| abi::ALLOCATE), image, entry, storage,
	753	+	}));
	754	+	assert (try! abi::decode(result)).object.index == 0;
	755	+	return object;
	756	+	},
	757	+	},
	758	+	},
	759	+	}
	760	+	}
	761	+
712	762		/// Create a private domain while the caller has exclusive access to metadata.
713	763		export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
714	764		table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
715	765		{
716	766		let pending = try reserve(store, backings, packages, table, authority, image);

144	144		try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
145	145		return info;
146	146		}
147	147		}
148	148
	149	+	/// Read Device metadata. An invalid handle is a program error.
	150	+	export fn queryDevice(handle: abi::Handle) -> abi::DeviceInfo {
	151	+	unsafe {
	152	+	let mut info: abi::DeviceInfo = undefined;
	153	+	try! result(ecall(abi::Operation::QueryDevice as u32, *handle as i64,
	154	+	(&mut info) as i64, @sizeOf(abi::DeviceInfo) as i64, 0));
	155	+	return info;
	156	+	}
	157	+	}
	158	+
149	159		/// Read a physical interrupt source. An invalid handle is a program error.
150	160		export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
151	161		unsafe {
152	162		let mut info: abi::InterruptInfo = undefined;
153	163		try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,

14	14		use kernel::trap;
15	15		use kernel::budgets;
16	16		use kernel::remote;
17	17		use kernel::interrupts;
18	18		use kernel::events;
	19	+	use kernel::devices;
	20	+	use kernel::platform;
19	21
20	22		/// Event storage for administrative destruction targets.
21	23		unsafe static TARGETS: [domains::Memory; 2] = undefined;
22	24
23	25		/// Retained dead-domain records cannot be queried or delegated through handles.

373	375		try testing::expect(budgets::STORE.slots[0].state == slots::State::Free);
374	376		let installed = try! capability::get(&MEMORY.table, queue);
375	377		try testing::expect(installed.kind == abi::Kind::Events);
376	378		}
377	379
	380	+	/// Device queries copy only metadata into checked caller-owned memory.
	381	+	@test unsafe fn deviceRecord() throws (testing::TestError) {
	382	+	let owner = initialize();
	383	+	let storage = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
	384	+	set devices::STORE.count = 1;
	385	+	set devices::STORE.regions[0] = devices::Region {
	386	+	kind: platform::Kind::Uart, info: { base: 0x10000000, size: 256 }, rights: abi::Rights(abi::READ),
	387	+	};
	388	+	let device = try! devices::install(&devices::STORE, &mut MEMORY.table, 0);
	389	+	let base = (&RAM[OFFSET]) as u64;
	390	+	set RAM[OFFSET + 2] = 123;
	391	+	assert try! calls::invoke(owner, 45, &[*device, base, 16, 0], 0) == 0;
	392	+	assert RAM[OFFSET] == 0x10000000 and RAM[OFFSET + 1] == 256 and RAM[OFFSET + 2] == 123;
	393	+	let mut rejected: u32 = 0;
	394	+	for args in &[[device, base + 1, 16, 0], [device, base, 8, 0], [*device, base, 16, 1]] {
	395	+	try calls::invoke(owner, 45, &args[..], 0) catch error {
	396	+	assert error == abi::Error::InvalidArg; set rejected += 1;
	397	+	};
	398	+	}
	399	+	assert rejected == 3 and RAM[OFFSET + 2] == 123;
	400	+	}
	401	+
378	402		/// A query copies exactly its record into writable memory owned by the caller.
379	403		@test unsafe fn records() throws (testing::TestError) {
380	404		let owner = initialize();
381	405		let handle = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
382	406		let base = (&RAM[OFFSET]) as u64;

423	447		try testing::expect(rejected == 6 and domains::STORE.contextSlots[0].state == slots::State::Free);
424	448		}
425	449
426	450		/// Invalid infallible queries terminate the caller at the syscall boundary.
427	451		@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
428		-	for operation in [44 as u64, 46, 47, 48] {
	452	+	for operation in [44 as u64, 45, 46, 47, 48] {
429	453		let owner = initialize();
430	454		try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
431	455		let mut frame = trap::Frame { registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
432	456		set frame.registers[17] = operation;
433	457		calls::handle(owner, &mut frame, 0, calls::invoke);

71	71		template: &[7], memory: 8, alignment: 8, relocations: &[],
72	72		});
73	73		set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
74	74		}
75	75
	76	+	/// Failed root construction restores capacity and permits a fresh attempt.
	77	+	@test unsafe fn bootstrapRollback() throws (testing::TestError) {
	78	+	initialize(abi::CREATE);
	79	+	let image = (try! capability::get(&TABLE, IMAGE)).object;
	80	+	domains::initialize(&mut DOMAINS);
	81	+	set BACKINGS.domains[0] = 0;
	82	+	let capacity = BACKINGS.pool.count;
	83	+	for count in [0 as u32, 1, 8, 32] {
	84	+	set BACKINGS.pool.count = count;
	85	+	let mut failed = false;
	86	+	try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
	87	+	assert error == abi::Error::OutOfMemory; set failed = true;
	88	+	};
	89	+	assert failed and DOMAINS.slots[0].state == slots::State::Free;
	90	+	assert DOMAINS.contextSlots[0].state == slots::State::Free;
	91	+	assert BACKINGS.domains[0] == 0 and DOMAINS.events.queues[0].generation == 0;
	92	+	for i in 0..count { assert frames::available(&BACKINGS.pool, i); }
	93	+	}
	94	+	set BACKINGS.pool.count = capacity;
	95	+	let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
	96	+	assert root.index == 0 and root.generation > 1;
	97	+	assert (try! domains::get(&DOMAINS, root)).parent == nil;
	98	+	}
	99	+
	100	+	/// Root construction owns slot zero and installs the fixed bootstrap handles.
	101	+	@test unsafe fn bootstrap() throws (testing::TestError) {
	102	+	initialize(abi::CREATE);
	103	+	let image = (try! capability::get(&TABLE, IMAGE)).object;
	104	+	domains::initialize(&mut DOMAINS);
	105	+	set BACKINGS.domains[0] = 0;
	106	+	let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
	107	+	assert root.index == 0 and root.generation == 1;
	108	+	let domain = try! domains::get(&DOMAINS, root);
	109	+	assert domain.creator == root and domain.parent == nil and domain.state == domains::Lifecycle::Pending;
	110	+	assert domain.memory.table.slots[0].state == slots::State::Live;
	111	+	let self = domain.memory.table.entries[0];
	112	+	assert self.kind == abi::Kind::Domain and self.object == root;
	113	+	assert *self.rights == (domains::MANAGEMENT_RIGHTS \| abi::CREATE \| abi::ALLOCATE);
	114	+	let queue = try! abi::decode(domain.events);
	115	+	assert queue.object.index == 1 and queue.kind == abi::Kind::Events;
	116	+	assert domain.memory.table.entries[1].object == root;
	117	+	assert DOMAINS.contexts[domain.initial.index].frame.pc == 0x80000000;
	118	+	assert DOMAINS.contexts[domain.initial.index].frame.registers[3] == domain.graph.table.ptr as u64;
	119	+	for word in DOMAINS.ancestors[0] { assert word == 0; }
	120	+	let mut busy = false;
	121	+	try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
	122	+	assert error == abi::Error::Busy; set busy = true;
	123	+	};
	124	+	assert busy and DOMAINS.slots[1].state == slots::State::Free;
	125	+	}
	126	+
76	127		/// Interrupted creation returns all private storage after its caller releases its hart.
77	128		@test unsafe fn cancelledCreation() throws (testing::TestError) {
78	129		initialize(abi::CREATE);
79	130		lifecycle::initialize(&mut CALLS);
80	131		let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);

17	17		use std::arch::rv64::shared;
18	18		use std::arch::rv64::shared::catalog;
19	19
20	20		/// Native entry page, above the platform firmware data.
21	21		constant CODE_ADDRESS: u64 = 0x81000000;
	22	+	/// Maximum shared packages in the boot image.
	23	+	constant PACKAGES: u32 = 4;
22	24		/// Package code arenas retained until image output completes.
23		-	unsafe static CODE: [[u8; 16777216]; 2] = undefined;
	25	+	unsafe static CODE: [[u8; 16777216]; PACKAGES] = undefined;
24	26		/// Reusable function workspace.
25	27		static SCRATCH: [u8; 16777216] = [0; 16777216];
26	28		/// Persistent decoded RIL storage.
27	29		static DECODE: [u8; 67108864] = [0; 67108864];
28	30		/// Exact package bytes retained in the native catalog.
29		-	unsafe static INPUT: [[u8; 8388608]; 2] = undefined;
	31	+	unsafe static INPUT: [[u8; 8388608]; PACKAGES] = undefined;
30	32		/// Combined startup and boundary assembly source.
31	33		static SOURCE: [u8; 65536] = [0; 65536];
32	34		/// Assembler workspace.
33	35		static ASSEMBLY: [u8; 4194304] = [0; 4194304];
34	36		/// Assembled startup words.
35	37		static TEXT: [u32; 16384] = [0; 16384];
36	38		/// Assembly identifiers retained through package linking.
37	39		unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };
38	40		/// Per-package data layout workspaces.
39		-	unsafe static DATA: [[data::DataSym; 4096]; 2] = undefined;
	41	+	unsafe static DATA: [[data::DataSym; 4096]; PACKAGES] = undefined;
40	42		/// Local function and data definitions.
41		-	unsafe static SYMBOLS: [[shared::Symbol; 4096]; 2] = undefined;
	43	+	unsafe static SYMBOLS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
42	44		/// Public package symbols.
43		-	unsafe static EXPORTS: [[shared::Symbol; 4096]; 2] = undefined;
	45	+	unsafe static EXPORTS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
	46	+	/// Exports from dependencies compiled before the current package.
	47	+	unsafe static IMPORTS: [shared::Symbol; 16384] = undefined;
44	48		/// Private-state relocation records.
45		-	unsafe static RELOCS: [[shared::Relocation; 4096]; 2] = undefined;
	49	+	unsafe static RELOCS: [[shared::Relocation; 4096]; PACKAGES] = undefined;
46	50		/// Initialized package templates.
47		-	unsafe static TEMPLATES: [[u8; 1048576]; 2] = undefined;
	51	+	unsafe static TEMPLATES: [[u8; 1048576]; PACKAGES] = undefined;
48	52		/// Read-only catalog and retained package payloads.
49	53		static RO: [u8; 16777216] = [0; 16777216];
50	54		/// Kernel state table and initialized private package graph.
51	55		static RW: [u8; 33554432] = [0; 33554432];
52	56		/// Contiguous native code segment, including entry trampoline and alignment padding.

83	87		template: &mut TEMPLATES[slot][..], relocations: &mut RELOCS[slot][..],
84	88		}, &mut arena, &mut scratch);
85	89		}
86	90
87	91		/// Emit a trampoline that preserves firmware arguments and supplies the native catalog.
88		-	unsafe fn trampoline(kernel: u64, ro: u64, rw: u64) {
	92	+	unsafe fn trampoline(kernel: u64, ro: u64, rw: u64, count: u32) {
89	93		let mut arena = alloc::new(&mut SCRATCH[..]);
90	94		let mut e = try! emit::emitter(&mut arena, false);
91	95		emit::loadImm(&mut e, rv64::GP, rw as i64);
92	96		emit::loadImm(&mut e, rv64::A3, ro as i64);
93		-	emit::loadImm(&mut e, rv64::A4, 2);
	97	+	emit::loadImm(&mut e, rv64::A4, count as i64);
94	98		emit::loadImm(&mut e, rv64::T0, kernel as i64);
95	99		emit::emit(&mut e, encode::jalr(rv64::ZERO, rv64::T0, 0));
96	100		assert e.codeLen <= 1024;
97	101		for word, i in emit::getCode(&e) { set NATIVE[i] = word; }
98	102		}
99	103
100	104		/// Link shared boot packages, retain their catalog, and instantiate the kernel graph.
101	105		@default unsafe fn main(env: *sys::Env) -> i32 {
102		-	assert env.args.len == 5;
	106	+	assert env.args.len >= 5 and env.args.len <= PACKAGES + 3;
	107	+	let count = env.args.len - 3;
103	108		let mut decoder = alloc::new(&mut DECODE[..]);
104	109		let libraryBytesLength = unix::readFile(env.args[1], &mut INPUT[0][..]) else panic "std RIL";
105	110		let libraryBytes = &INPUT[0][..libraryBytesLength];
106	111		let libraryInput = try! program::decode(libraryBytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
107	112		let kernelBytesLength = unix::readFile(env.args[2], &mut INPUT[1][..]) else panic "kernel RIL";

109	114		let kernelInput = try! program::decode(kernelBytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
110	115		assert mem::eq(libraryInput.name, "std") and mem::eq(kernelInput.name, "kernel");
111	116		assert libraryInput.dependencies.len == 0;
112	117		for dependency in kernelInput.dependencies { assert mem::eq(dependency, "std"); }
113	118		let sourceLength = unix::readFile(env.args[3], &mut SOURCE[..]) else panic "kernel assembly";
	119	+	let source = &SOURCE[..sourceLength];
114	120		let mut assemblyArena = alloc::new(&mut ASSEMBLY[..]);
115	121		let empty: *mut [u8] = &mut [];
116		-	let assembly = try! asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..sourceLength],
	122	+	let assembly = try! asm::assemble(asm::scanner::SourceKind::String, source,
117	123		&mut TEXT[..], &mut empty[..], &mut assemblyArena, &mut STRINGS, 0);
118	124		let library = try compile(&libraryInput, 0, CODE_ADDRESS + 4096, &[],
119	125		asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error { report(error); return 1; };
120	126		let kernelAddress = aligned(library.codeAddress + library.code.len as u64 * 4, 4096);
121	127		let kernel = try compile(&kernelInput, 1, kernelAddress, library.exports, assembly) catch error { report(error); return 1; };
122		-	let entries = [
123		-	catalog::Entry { source: libraryBytes, package: library },
124		-	catalog::Entry { source: kernelBytes, package: kernel },
125		-	];
126		-	let codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
	128	+	let mut catalogEntries: [catalog::Entry; PACKAGES] = undefined;
	129	+	set catalogEntries[0] = catalog::Entry { source: libraryBytes, package: library };
	130	+	set catalogEntries[1] = catalog::Entry { source: kernelBytes, package: kernel };
	131	+	let mut codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
	132	+	for slot in 2..count {
	133	+	let bytesLength = unix::readFile(env.args[slot + 3], &mut INPUT[slot][..]) else panic "boot package RIL";
	134	+	let bytes = &INPUT[slot][..bytesLength];
	135	+	let input = try! program::decode(bytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
	136	+	let mut imported: u32 = 0;
	137	+	for previous in 0..slot {
	138	+	for symbol in catalogEntries[previous].package.exports {
	139	+	assert imported < IMPORTS.len;
	140	+	set IMPORTS[imported] = symbol;
	141	+	set imported += 1;
	142	+	}
	143	+	}
	144	+	let package = try compile(&input, slot, aligned(codeEnd, 4096), &IMPORTS[..imported],
	145	+	asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error { report(error); return 1; };
	146	+	set catalogEntries[slot] = catalog::Entry { source: bytes, package };
	147	+	set codeEnd = package.codeAddress + package.code.len as u64 * 4;
	148	+	}
127	149		let codeSize = (codeEnd - CODE_ADDRESS) as u32;
128	150		assert codeEnd - CODE_ADDRESS <= @sizeOf([u32; 4194304]) as u64;
129	151		let roAddress = aligned(codeEnd, 4096);
130		-	let roSize = try catalog::pack(&entries[..], roAddress, &mut RO[..]) catch error { report(error); return 1; };
	152	+	let roSize = try catalog::pack(&catalogEntries[..count], roAddress, &mut RO[..]) catch error { report(error); return 1; };
131	153		let rwAddress = aligned(roAddress + roSize as u64, 4096);
132	154		let mut bases: [u64; shared::MAX_PACKAGES] = [0; shared::MAX_PACKAGES];
133	155		let mut rwSize: u32 = @sizeOf([u64; shared::MAX_PACKAGES]);
134		-	for entry in &entries[..] {
	156	+	for entry in &catalogEntries[..2] {
135	157		let at = aligned(rwAddress + rwSize as u64, entry.package.alignment);
136	158		assert at - rwAddress <= RW.len as u64 and entry.package.memory <= RW.len - (at - rwAddress) as u32;
137	159		set bases[entry.package.slot] = at;
138	160		set rwSize = (at - rwAddress) as u32 + entry.package.memory;
139	161		}
140	162		try! mem::copy(&mut RW[..@sizeOf([u64; shared::MAX_PACKAGES])],
141	163		@sliceOf(&bases[0] as *unsafe u8, @sizeOf([u64; shared::MAX_PACKAGES])));
142		-	for entry in &entries[..] {
	164	+	for entry in &catalogEntries[..2] {
143	165		let at = (bases[entry.package.slot] - rwAddress) as u32;
144	166		try shared::instantiate(&entry.package, &bases[..], &mut RW[at..at + entry.package.memory])
145	167		catch error { report(error); return 1; };
146	168		}
147	169		for i in 0..codeSize / 4 { set NATIVE[i] = encode::nop(); }
148		-	for entry in &entries[..] {
	170	+	for entry in &catalogEntries[..count] {
149	171		let at = ((entry.package.codeAddress - CODE_ADDRESS) / 4) as u32;
150	172		for word, i in entry.package.code { set NATIVE[at + i] = word; }
151	173		}
152		-	trampoline(kernelAddress, roAddress, rwAddress);
	174	+	trampoline(kernelAddress, roAddress, rwAddress, count);
153	175		let header = try! image::header(image::Layout {
154	176		entry: CODE_ADDRESS,
155	177		code: image::Segment { address: CODE_ADDRESS, initialized: codeSize, memory: codeSize },
156	178		roData: image::Segment { address: roAddress, initialized: roSize, memory: roSize },
157	179		rwData: image::Segment { address: rwAddress, initialized: rwSize, memory: rwSize },

659	659
660	660		// Result in A0.
661	661		let ecallRd = getDstReg(s, dst, super::SCRATCH1);
662	662		emitMv(s, ecallRd, super::A0);
663	663		},
	664	+	case il::Instr::DeviceRead { typ, dst, handle, offset } => {
	665	+	deviceAddress(s, typ, handle, offset, il::Val::Imm(0), false);
	666	+	let rd = getDstReg(s, dst, super::SCRATCH1);
	667	+	match typ {
	668	+	case il::Type::W8 => emit::emit(s.e, encode::lbu(rd, super::A0, 0)),
	669	+	case il::Type::W16 => emit::emit(s.e, encode::lhu(rd, super::A0, 0)),
	670	+	case il::Type::W32 => emit::emit(s.e, encode::lwu(rd, super::A0, 0)),
	671	+	case il::Type::W64 => emit::emit(s.e, encode::ld(rd, super::A0, 0)),
	672	+	}
	673	+	emit::emit(s.e, encode::fence());
	674	+	},
	675	+	case il::Instr::DeviceWrite { typ, handle, offset, value } => {
	676	+	deviceAddress(s, typ, handle, offset, value, true);
	677	+	match typ {
	678	+	case il::Type::W8 => emit::emit(s.e, encode::sb(super::A3, super::A0, 0)),
	679	+	case il::Type::W16 => emit::emit(s.e, encode::sh(super::A3, super::A0, 0)),
	680	+	case il::Type::W32 => emit::emit(s.e, encode::sw(super::A3, super::A0, 0)),
	681	+	case il::Type::W64 => emit::emit(s.e, encode::sd(super::A3, super::A0, 0)),
	682	+	}
	683	+	emit::emit(s.e, encode::fence());
	684	+	},
664	685		case il::Instr::Ebreak => {
665	686		emit::emit(s.e, encode::ebreak());
666	687		},
667	688		case il::Instr::MemoryFence => {
668	689		emit::emit(s.e, encode::fence());

1195	1216		}
1196	1217		if invert {
1197	1218		emit::emit(s.e, encode::xori(rd, rd, 1));
1198	1219		}
1199	1220		}
	1221	+
	1222	+	/// Resolve one live Device access before an ordered user-mode register instruction.
	1223	+	unsafe fn deviceAddress(s: &mut Selector, typ: il::Type, handle: il::Val, offset: il::Val, value: il::Val, writing: bool) {
	1224	+	let mut flags = il::typeSize(typ) as i64;
	1225	+	if writing { set flags \|= 0x100; }
	1226	+	let dsts: [gen::Reg; 5] = [super::A7, super::A0, super::A1, super::A2, super::A3];
	1227	+	let args: [il::Val; 5] = [il::Val::Imm(il::DEVICE_ACCESS as i64), handle, offset, il::Val::Imm(flags), value];
	1228	+	emitParallelMoves(s, &dsts[..], &args[..]);
	1229	+	emit::emit(s.e, encode::ecall());
	1230	+	emit::emit(s.e, encode::fence());
	1231	+	}

296	296		/// Environment break: `ebreak;`.
297	297		/// Triggers a breakpoint exception for debugging.
298	298		Ebreak,
299	299		/// Full acquire/release memory fence.
300	300		MemoryFence,
	301	+	/// Read one register through a live Device handle.
	302	+	DeviceRead {
	303	+	/// Register access width.
	304	+	typ: Type,
	305	+	/// Destination for the zero-extended register value.
	306	+	dst: Reg,
	307	+	/// Domain-relative Device capability.
	308	+	handle: Val,
	309	+	/// Byte offset within the device region.
	310	+	offset: Val,
	311	+	},
	312	+	/// Write one register through a live Device handle.
	313	+	DeviceWrite {
	314	+	/// Register access width.
	315	+	typ: Type,
	316	+	/// Domain-relative Device capability.
	317	+	handle: Val,
	318	+	/// Byte offset within the device region.
	319	+	offset: Val,
	320	+	/// Value whose low bits are written at the selected width.
	321	+	value: Val,
	322	+	},
301	323		}
302	324
303	325		//////////////////////////
304	326		// Blocks and Functions //
305	327		//////////////////////////

398	420
399	421		///////////////////////
400	422		// Utility Functions //
401	423		///////////////////////
402	424
	425	+	/// Compiler-generated metadata call for one MMIO access.
	426	+	/// a0 is the handle, a1 the offset, and a2 the byte width plus bit 8 for a write.
	427	+	/// a3 holds a write value and is preserved. Success returns the address in a0.
	428	+	/// Any failure terminates the caller before the user-mode register access.
	429	+	export constant DEVICE_ACCESS: u32 = 74;
	430	+
403	431		/// Get the destination register of an instruction, if any.
404	432		export fn instrDst(instr: Instr) -> ?Reg {
405	433		match instr {
406	434		case Instr::Reserve { dst, .. } => return dst,
407	435		case Instr::Load { dst, .. } => return dst,

411	439		case Instr::UnOp { dst, .. } => return dst,
412	440		case Instr::Zext { dst, .. } => return dst,
413	441		case Instr::Sext { dst, .. } => return dst,
414	442		case Instr::Call { dst, .. } => return dst,
415	443		case Instr::Ecall { dst, .. } => return dst,
	444	+	case Instr::DeviceRead { dst, .. } => return dst,
416	445		else => return nil,
417	446		}
418	447		}
419	448
420	449		/// Check if an instruction is a function call.
421	450		export fn isCall(instr: Instr) -> bool {
422	451		match instr {
423	452		case Instr::Call { .. },
424		-	Instr::Ecall { .. } => return true,
	453	+	Instr::Ecall { .. }, Instr::DeviceRead { .. }, Instr::DeviceWrite { .. } => return true,
425	454		else => return false,
426	455		}
427	456		}
428	457
429	458		/// Call a function for each register used by an instruction.