radiance · commit

kernel/kernel.rad +2 -0

export mod trap;
export mod capability;
export mod transactions;
export mod events;
export mod interrupts;
export mod devices;
export mod plic;
export mod registry;
export mod frames;
export mod backing;
export mod pages;

export mod timers;
export mod remote;
export mod calls;
export mod sys;
export mod boot;
export mod startup;
@test export mod tests;

kernel/kernel/abi.rad +32 -0

    eventsHandle: u64,
    /// Address of the domain's shared event ring.
    eventsPointer: u64,
}

/// Initial root resources copied into root-owned startup memory.
export record BootInfo: Copy {
    /// Number of Device handles after the self and Events slots.
    devices: u64,
    /// Number of Interrupt handles following the Device handles.
    interrupts: u64,
    /// Bit mask of online harts, whose Budget handles follow in hart order.
    harts: u64,
    /// Machine timer ticks per second.
    timebase: u64,
    /// Image used to construct root and additional instances of its program.
    rootImage: Handle,
    /// Initial user-space scheduler image.
    schedulerImage: Handle,
    /// Read-only machine-time device, or zero when absent.
    clock: Handle,
    /// Platform shutdown device, or zero when absent.
    shutdown: Handle,
    /// Page that owns this header and the copied handle array.
    arguments: Handle,
    /// Copied capability values in their initial slot order.
    handles: *unsafe mut [u64],
}

/// Physical extent returned by QueryPage.
export record PageInfo: Copy {
    /// Address of the first physical frame.
    base: u64,
    /// Number of contiguous physical frames.
    count: u64,
}

/// Physical MMIO extent returned by QueryDevice.
export record DeviceInfo: Copy {
    /// Physical base address of the register region.
    base: u64,
    /// Size of the register region in bytes.
    size: u64,
}

/// Physical source returned by QueryInterrupt.
export record InterruptInfo: Copy {
    /// PLIC source number; zero does not name an interrupt.
    number: u32,
}

kernel/kernel/boot.rad +22 -1

//! Machine entry boundary and published platform state.

use super::platform;
use super::startup;
use super::fdt;
use super::range;
use super::limits;
use super::sync;
use super::trap;

use super::budgets;
use super::dispatch;
use super::timers;
use super::remote;
use super::interrupts;
use super::devices;
use super::plic;
use std::arch::rv64::shared::catalog;

/// Platform data published by hart zero before secondary initialization.
export unsafe static PLATFORM: platform::Platform = undefined;
/// Release/acquire publication flag for PLATFORM.
static READY: u64 = 0;
/// Number of harts that validated their machine stack.
static ARRIVED: u64 = 0;
/// Root resource publication before user dispatch begins.
static STARTED: u64 = 0;
/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {

        budgets::initialize(&mut budgets::STORE);
        timers::initialize(&mut timers::STORE);
        remote::initialize(&mut remote::STORE, PLATFORM.harts);
        interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
        try! plic::initialize(&PLATFORM);
        assert entryCount == 2;
        try! devices::initialize(&mut devices::STORE, &PLATFORM);
        assert entryCount >= 2 and entryCount <= limits::PACKAGES;
        registry::initialize(&mut registry::STORE);
        let bootCatalog = @sliceOf(entries, entryCount);
        try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
        let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";
        let kernel = registry::find(&registry::STORE, &"kernel"[..]) else panic "boot kernel";

        }
    }
    return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
}

/// Publish configured root resources once, then enter dispatch on every hart.
export unsafe fn launch(last: bool) -> ! {
    if last {
        if let root = registry::find(&registry::STORE, &"root"[..]) {
            let scheduler = registry::find(&registry::STORE, &"scheduler"[..]) else panic "missing scheduler image";
            let domain = try! startup::create(&PLATFORM, root, scheduler);
            print("kernel: root ready\n");
        }
        sync::storeRelease(&mut STARTED, 1);
    } else {
        while sync::loadAcquire(&STARTED) == 0 {
        }
    }
    run();
}

/// Enter dispatch after every online hart publishes its initialized state.
export unsafe fn run() -> ! {
    let mut count: u64 = 0;
    for id in 0..limits::HARTS {
        if (PLATFORM.harts & (1 << id)) <> 0 {

kernel/kernel/boot.ras +9 -1

.export @kernel::boot::read8;
.export @kernel::boot::write8;

@kernel::boot::start
    call @kernel::boot::initialize;
    call @kernel::boot::run;
    call @kernel::boot::launch;
    ebreak;

// Initialize machine CSRs and preserve the firmware stack across Radiance entry.
@kernel::boot::initialize
    csrw mie %zero;

@kernel::boot::write8
    fence iorw iorw;
    sb %a1 0(%a0);
    fence iorw iorw;
    ret;

.export @kernel::startup::info;
.export @kernel::startup::words;
// Resolve root-owned startup memory after physical allocation.
@kernel::startup::info
    ret;
@kernel::startup::words
    ret;

kernel/kernel/calls.rad +20 -1

//! Direct-call execution and checked copies into domain-owned memory.

use super::abi;
use std::lang::il;
use super::slots;
use super::limits;
use super::capability;
use super::pages;
use super::domains;

use super::dispatch;
use super::timers;
use super::sync;
use super::remote;
use super::interrupts;
use super::devices;
use super::plic;
use super::lifecycle;

/// Resolve the owning hart for an operation that can change CPU authority.
unsafe fn budgetHart(table: &capability::Table, operation: u64, arguments: &[u64]) -> ?u32 throws (abi::Error) {

            let object = try domains::get(&domains::STORE, entry.object);
            if object.state == domains::Lifecycle::Dead {
                throw abi::Error::BadHandle;
            }
        },
        case abi::Kind::Device => {
            let object = try devices::get(&devices::STORE, entry.object);
        },
        case abi::Kind::Image => {
            let object = try registry::get(&registry::STORE, entry.object);
        },
        else => throw abi::Error::BadHandle,
    }

        case 22, 43, 49 => {
            let stopped = try termination(owner, operation, arguments);
            return 0;
        },
        case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
        case il::DEVICE_ACCESS => {
            let flags = arguments[2];
            if (flags & ~0x1ff) <> 0 {
                throw abi::Error::InvalidArg;
            }
            return try devices::address(&devices::STORE, &domain.memory.table, handle,
                arguments[1], flags & 0xff, (flags & 0x100) <> 0);
        },
        case 12 => {
            try drop(&mut domain, handle);
            return 0;
        },
        case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,

            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
            let page = try pages::get(&pages::STORE, entry.object);
            try reply(&domain, arguments, &[page.base, page.count as u64]);
            return 0;
        },
        case 45 => {
            let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Device, abi::Rights(0));
            let device = try devices::get(&devices::STORE, entry.object);
            try reply(&domain, arguments, &[device.info.base, device.info.size]);
            return 0;
        },
        case 46 => {
            let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
            if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
                or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) {
                    throw abi::Error::InvalidArg;

    assert trap::fromUser(frame) and frame.cause == 8;
    assert trap::advanceCall(frame);
    let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
    let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
        let operation = frame.registers[17];
        if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
        if operation == il::DEVICE_ACCESS or (operation >= 44 and operation <= 48 and error == abi::Error::BadHandle) {
            try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
                assert stopped == abi::Error::BadHandle;
            };
        }
        set frame.registers[10] = (0 as u64) - error as u64;

kernel/kernel/devices.rad added +99 -0

1	+	//! Immutable MMIO regions and domain-relative device authority.
2	+	use super::abi;
3	+	use super::limits;
4	+	use super::platform;
5	+	use super::capability;
6	+
7	+	/// One platform register region available to user-space drivers.
8	+	export record Region: Copy {
9	+	/// Platform mechanism used during bootstrap handoff.
10	+	kind: platform::Kind,
11	+	/// Physical register extent returned by QueryDevice.
12	+	info: abi::DeviceInfo,
13	+	/// Maximum rights installed by bootstrap.
14	+	rights: abi::Rights,
15	+	}
16	+
17	+	/// Device identities retained for the complete boot lifetime.
18	+	export record Store: Copy {
19	+	/// Exported regions in device-tree order.
20	+	regions: [Region; limits::DEVICES],
21	+	/// Number of published regions.
22	+	count: u32,
23	+	}
24	+
25	+	/// Global device metadata published before any user context starts.
26	+	export unsafe static STORE: Store = undefined;
27	+
28	+	/// Publish driver regions and the read-only CLINT time register.
29	+	/// Interrupt-controller and timer-programming registers stay kernel-private.
30	+	export fn initialize(store: &mut Store, machine: &platform::Platform) throws (abi::Error) {
31	+	set store.count = 0;
32	+	if machine.deviceCount > limits::DEVICES {
33	+	throw abi::Error::InvalidArg;
34	+	}
35	+	let mut count: u32 = 0;
36	+	for i in 0..machine.deviceCount {
37	+	let device = machine.devices[i];
38	+	if device.memory.start >= device.memory.end {
39	+	throw abi::Error::InvalidArg;
40	+	}
41	+	if device.kind == platform::Kind::Plic {
42	+	continue;
43	+	}
44	+	let mut base = device.memory.start;
45	+	let mut size = device.memory.end - base;
46	+	let mut rights = abi::READ \| abi::WRITE \| abi::GRANT \| abi::TRANSFER;
47	+	if device.kind == platform::Kind::Clint {
48	+	if size < 0xc000 {
49	+	throw abi::Error::InvalidArg;
50	+	}
51	+	set base += 0xbff8;
52	+	set size = 8;
53	+	set rights &= ~abi::WRITE;
54	+	}
55	+	set store.regions[count] = Region {
56	+	kind: device.kind, info: abi::DeviceInfo { base, size }, rights: abi::Rights(rights),
57	+	};
58	+	set count += 1;
59	+	}
60	+	set store.count = count;
61	+	}
62	+
63	+	/// Resolve an immutable device identity; physical regions use generation one.
64	+	export fn get(store: &Store, object: abi::Ref) -> Region throws (abi::Error) {
65	+	if object.index >= store.count or object.generation <> 1 {
66	+	throw abi::Error::BadHandle;
67	+	}
68	+	return store.regions[object.index];
69	+	}
70	+
71	+	/// Install bootstrap authority for one published physical device.
72	+	export fn install(store: &Store, table: &mut capability::Table, index: u32) -> abi::Handle throws (abi::Error) {
73	+	let object = abi::Ref { index, generation: 1 };
74	+	let region = try get(store, object);
75	+	return try capability::install(table, capability::Entry { kind: abi::Kind::Device, object, rights: region.rights });
76	+	}
77	+
78	+	/// Validate one register access and return its physical address to generated code.
79	+	export fn address(store: &Store, table: &capability::Table, handle: abi::Handle,
80	+	offset: u64, width: u64, writing: bool) -> u64 throws (abi::Error)
81	+	{
82	+	let mut right = abi::READ;
83	+	if writing {
84	+	set right = abi::WRITE;
85	+	}
86	+	let entry = try capability::lookup(table, handle, abi::Kind::Device, abi::Rights(right));
87	+	let region = (try get(store, entry.object)).info;
88	+	if width <> 1 and width <> 2 and width <> 4 and width <> 8 {
89	+	throw abi::Error::InvalidArg;
90	+	}
91	+	if offset > region.size or width > region.size - offset or region.base > 0xffffffffffffffff - offset {
92	+	throw abi::Error::InvalidArg;
93	+	}
94	+	let address = region.base + offset;
95	+	if (address & (width - 1)) <> 0 {
96	+	throw abi::Error::InvalidArg;
97	+	}
98	+	return address;
99	+	}

kernel/kernel/domains.rad +65 -7

    match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
}

/// Initialize retained domain storage outside the metadata lock.
export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
    let mut storage = creation.storage;
    try prepareStorage(packages, creation.storage, creation.domain);
}

/// Initialize one domain's private allocations before capability publication.
unsafe fn prepareStorage(packages: &registry::Store, storage: Prepared, owner: abi::Ref) throws (abi::Error) {
    let mut storage = storage;
    zero(storage.memory as u64, storage.allocation.count);
    zero(storage.kernelStack.start, storage.kernelFrames.count);
    try instances::fill(packages, &storage.graph.graph, storage.graph.base);
    capability::initialize(&mut storage.memory.table, creation.domain);
    capability::initialize(&mut storage.memory.table, owner);
}

/// Check the unpublished capacity retained by one creation transaction.
fn require(store: &Store, table: &capability::Table, creation: &Creation) {
    assert table.owner == creation.owner;

    release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
            discard(&mut backings.pool, creation.storage, release);
            try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
            try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
            try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
        },
    }
}

/// Publish an initialized domain and its Events under metadata serialization.

    -> abi::Handle throws (abi::Error)
{
    match reservation {
        case Reservation::Held(creation) => {
            require(store, table, &creation);
            if not backing::domainLive(backings, creation.owner) {
            let root = creation.owner == creation.domain;
            if not root and not backing::domainLive(backings, creation.owner) {
                cancel(store, backings, table, Reservation::Held(creation));
                throw abi::Error::BadHandle;
            }
            let object = creation.domain;
            let initial = creation.context;

                owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
                userStack: range::Range { start: 0, end: 0 }, frame,
            };
            set store.reclaimed[object.index] = false;
            set store.reaping[object.index] = false;
            let mut parent: ?abi::Ref = nil;
            if not root {
                set parent = creation.owner;
            }
            set store.records[object.index] = Domain {
                state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
                state: Lifecycle::Pending, creator: creation.owner, parent, image: creation.image, initial,
                allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
            };
            set store.ancestors[object.index] = store.ancestors[creation.owner.index];
            set store.ancestors[object.index][creation.owner.index / 32] |= 1 << (creation.owner.index % 32);
            if not root {
                set store.ancestors[object.index] = store.ancestors[creation.owner.index];
                set store.ancestors[object.index][creation.owner.index / 32] |= 1 << (creation.owner.index % 32);
            }
            let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
            let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
            return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
                kind: abi::Kind::Domain, object: domain, rights: creation.rights,
            });
        },
    }
}

/// Create the first domain with its self capability in slot zero and Events in slot one.
/// Boot calls this before any domain, context, or exposure membership is published.
export unsafe fn bootstrap(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store, image: abi::Ref)
    -> abi::Ref throws (abi::Error)
{
    for slot in store.slots {
        if slot.state <> slots::State::Free {
            throw abi::Error::Busy;
        }
    }
    let package = try registry::get(packages, image);
    let entry = package.entry else {
        throw abi::Error::VerifyFailed;
    };
    let domainSlot = try slots::reserve(&mut store.slots[..]);
    let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch error {
        try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
    };
    let object = slots::reference(&domainSlot);
    let mut storage = try allocate(&mut backings.pool, packages, image) catch error {
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
    };
    try prepareStorage(packages, storage, object) catch error {
        discard(&mut backings.pool, storage, frames::release);
        try! slots::cancel(&mut store.contextSlots[..], contextSlot);
        try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
    };
    let handleSlot = try! slots::reserve(&mut storage.memory.table.slots[..]);
    assert slots::reference(&handleSlot).index == 0;
    match domainSlot {
        case slots::Reservation::Held(domain) => match contextSlot {
            case slots::Reservation::Held(context) => match handleSlot {
                case slots::Reservation::Held(handle) => {
                    let result = try publish(store, backings, &mut storage.memory.table, Reservation::Held(Creation {
                        owner: object, handle, domain, context,
                        rights: abi::Rights(MANAGEMENT_RIGHTS | abi::CREATE | abi::ALLOCATE), image, entry, storage,
                    }));
                    assert (try! abi::decode(result)).object.index == 0;
                    return object;
                },
            },
        },
    }
}

/// Create a private domain while the caller has exclusive access to metadata.
export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
    table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
{
    let pending = try reserve(store, backings, packages, table, authority, image);

kernel/kernel/startup.rad added +82 -0

1	+	//! Initial root resources and execution authority.
2	+	use super::abi;
3	+	use super::platform;
4	+	use super::limits;
5	+	use super::domains;
6	+	use super::pages;
7	+	use super::registry;
8	+	use super::devices;
9	+	use super::interrupts;
10	+	use super::budgets;
11	+	use super::dispatch;
12	+	use super::slots;
13	+	use super::capability;
14	+
15	+	/// Map a root-owned startup allocation to its ABI record.
16	+	fn info(address: u64) -> *mut abi::BootInfo;
17	+	/// Map an aligned root-owned startup extent to copied handle words.
18	+	fn words(address: u64) -> *unsafe mut u64;
19	+
20	+	/// Install root's fixed resource slots and activate its initial context on hart zero.
21	+	/// Boot calls this after global and per-hart initialization, before user dispatch.
22	+	export unsafe fn create(machine: &platform::Platform, image: abi::Ref, scheduler: abi::Ref) -> abi::Ref throws (abi::Error) {
23	+	if (machine.harts & 1) == 0 or machine.timebase == 0 {
24	+	throw abi::Error::InvalidArg;
25	+	}
26	+	let root = try domains::bootstrap(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, image);
27	+	let mut domain = try! domains::get(&domains::STORE, root);
28	+	let table = &mut domain.memory.table;
29	+	let self = try! abi::handle(abi::Kind::Domain, abi::Ref { index: 0, generation: table.slots[0].generation });
30	+	let mut clock = abi::Handle(0);
31	+	let mut shutdown = abi::Handle(0);
32	+	for i in 0..devices::STORE.count {
33	+	let handle = try devices::install(&devices::STORE, &mut *table, i);
34	+	if devices::STORE.regions[i].kind == platform::Kind::Clint {
35	+	set clock = handle;
36	+	}
37	+	if devices::STORE.regions[i].kind == platform::Kind::Finish {
38	+	set shutdown = handle;
39	+	}
40	+	}
41	+	for number in 1..interrupts::STORE.count + 1 {
42	+	let handle = try interrupts::seed(&mut interrupts::STORE, &mut *table, number);
43	+	}
44	+	let now = dispatch::clock();
45	+	let second = machine.timebase as u64;
46	+	if now > 0xffffffffffffffff - second * 61 {
47	+	throw abi::Error::InvalidArg;
48	+	}
49	+	let start = now + second;
50	+	let end = start + second * 60;
51	+	let mut count = 2 + devices::STORE.count + interrupts::STORE.count;
52	+	for hart in 0..limits::HARTS {
53	+	if (machine.harts & (1 << hart)) == 0 {
54	+	continue;
55	+	}
56	+	let handle = try budgets::seed(&mut budgets::STORE, &mut *table, hart, start, end);
57	+	if hart == 0 {
58	+	let bound = try budgets::bind(&mut budgets::STORE, &domains::STORE, &mut *table,
59	+	budgets::Binding { budget: handle, domain: self, context: domain.initial }, now);
60	+	}
61	+	set count += 1;
62	+	}
63	+	let rootImage = try registry::install(&registry::STORE, &mut *table, image);
64	+	let schedulerImage = try registry::install(&registry::STORE, &mut *table, scheduler);
65	+	let stack = try pages::allocate(&mut pages::STORE, &mut *table, self, 16);
66	+	let arguments = try pages::allocate(&mut pages::STORE, &mut *table, self, 1);
67	+	let stackPage = try! pages::get(&pages::STORE, (try! capability::get(&*table, stack)).object);
68	+	let argumentPage = try! pages::get(&pages::STORE, (try! capability::get(&*table, arguments)).object);
69	+	assert @sizeOf(abi::BootInfo) as u64 + count as u64 * 8 <= limits::FRAME_SIZE;
70	+	let handles = @sliceOf(words(argumentPage.base + @sizeOf(abi::BootInfo) as u64), count);
71	+	for i in 0..count {
72	+	assert table.slots[i].state == slots::State::Live;
73	+	set handles[i] = *try! abi::handle(table.entries[i].kind, abi::Ref { index: i, generation: table.slots[i].generation });
74	+	}
75	+	set *info(argumentPage.base) = abi::BootInfo {
76	+	devices: devices::STORE.count as u64, interrupts: interrupts::STORE.count as u64,
77	+	harts: machine.harts as u64, timebase: second, rootImage, schedulerImage, clock, shutdown, arguments, handles,
78	+	};
79	+	try domains::activate(&mut domains::STORE, &pages::STORE, &mut *table, self,
80	+	stackPage.base + stackPage.count as u64 * limits::FRAME_SIZE, argumentPage.base, @sizeOf(abi::BootInfo) as u64);
81	+	return root;
82	+	}

kernel/kernel/sys.rad +10 -0

        try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
        return info;
    }
}

/// Read Device metadata. An invalid handle is a program error.
export fn queryDevice(handle: abi::Handle) -> abi::DeviceInfo {
    unsafe {
        let mut info: abi::DeviceInfo = undefined;
        try! result(ecall(abi::Operation::QueryDevice as u32, *handle as i64,
            (&mut info) as i64, @sizeOf(abi::DeviceInfo) as i64, 0));
        return info;
    }
}

/// Read a physical interrupt source. An invalid handle is a program error.
export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
    unsafe {
        let mut info: abi::InterruptInfo = undefined;
        try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,

kernel/kernel/tests.rad +1 -0

export mod backing;
export mod pages;
export mod transactions;
export mod events;
export mod interrupts;
export mod devices;
export mod registry;
export mod loader;
export mod instances;
export mod domains;
export mod lifecycle;

kernel/kernel/tests/calls.rad +25 -1

use kernel::trap;
use kernel::budgets;
use kernel::remote;
use kernel::interrupts;
use kernel::events;
use kernel::devices;
use kernel::platform;

/// Event storage for administrative destruction targets.
unsafe static TARGETS: [domains::Memory; 2] = undefined;

/// Retained dead-domain records cannot be queried or delegated through handles.

    try testing::expect(budgets::STORE.slots[0].state == slots::State::Free);
    let installed = try! capability::get(&MEMORY.table, queue);
    try testing::expect(installed.kind == abi::Kind::Events);
}

/// Device queries copy only metadata into checked caller-owned memory.
@test unsafe fn deviceRecord() throws (testing::TestError) {
    let owner = initialize();
    let storage = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
    set devices::STORE.count = 1;
    set devices::STORE.regions[0] = devices::Region {
        kind: platform::Kind::Uart, info: { base: 0x10000000, size: 256 }, rights: abi::Rights(abi::READ),
    };
    let device = try! devices::install(&devices::STORE, &mut MEMORY.table, 0);
    let base = (&RAM[OFFSET]) as u64;
    set RAM[OFFSET + 2] = 123;
    assert try! calls::invoke(owner, 45, &[*device, base, 16, 0], 0) == 0;
    assert RAM[OFFSET] == 0x10000000 and RAM[OFFSET + 1] == 256 and RAM[OFFSET + 2] == 123;
    let mut rejected: u32 = 0;
    for args in &[[*device, base + 1, 16, 0], [*device, base, 8, 0], [*device, base, 16, 1]] {
        try calls::invoke(owner, 45, &args[..], 0) catch error {
            assert error == abi::Error::InvalidArg; set rejected += 1;
        };
    }
    assert rejected == 3 and RAM[OFFSET + 2] == 123;
}

/// A query copies exactly its record into writable memory owned by the caller.
@test unsafe fn records() throws (testing::TestError) {
    let owner = initialize();
    let handle = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
    let base = (&RAM[OFFSET]) as u64;

    try testing::expect(rejected == 6 and domains::STORE.contextSlots[0].state == slots::State::Free);
}

/// Invalid infallible queries terminate the caller at the syscall boundary.
@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
    for operation in [44 as u64, 46, 47, 48] {
    for operation in [44 as u64, 45, 46, 47, 48] {
        let owner = initialize();
        try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
        let mut frame = trap::Frame { registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
        set frame.registers[17] = operation;
        calls::handle(owner, &mut frame, 0, calls::invoke);

kernel/kernel/tests/devices.rad added +117 -0

1	+	//! Device identity, exported regions, and capability delegation.
2	+	use std::testing;
3	+	use kernel::abi;
4	+	use kernel::devices;
5	+	use kernel::platform;
6	+	use kernel::capability;
7	+	use kernel::calls;
8	+	use kernel::domains;
9	+	use kernel::slots;
10	+
11	+	/// Device metadata under test.
12	+	unsafe static STORE: devices::Store = undefined;
13	+	/// Synthetic validated platform regions.
14	+	unsafe static MACHINE: platform::Platform = undefined;
15	+	/// Independent caller and receiver capability storage.
16	+	unsafe static MEMORY: [domains::Memory; 2] = undefined;
17	+
18	+	/// Describe driver devices around kernel-owned interrupt and timer registers.
19	+	unsafe fn initialize() {
20	+	set MACHINE.deviceCount = 4;
21	+	set MACHINE.devices[0] = { kind: platform::Kind::Uart, memory: { start: 0x10000000, end: 0x10000100 } };
22	+	set MACHINE.devices[1] = { kind: platform::Kind::Plic, memory: { start: 0x0c000000, end: 0x10000000 } };
23	+	set MACHINE.devices[2] = { kind: platform::Kind::Clint, memory: { start: 0x02000000, end: 0x02010000 } };
24	+	set MACHINE.devices[3] = { kind: platform::Kind::Finish, memory: { start: 0x10001000, end: 0x10002000 } };
25	+	try! devices::initialize(&mut STORE, &MACHINE);
26	+	}
27	+
28	+	/// Driver regions retain tree order and expose only read-only timer state.
29	+	@test unsafe fn regions() throws (testing::TestError) {
30	+	initialize();
31	+	assert STORE.count == 3;
32	+	assert STORE.regions[0].kind == platform::Kind::Uart;
33	+	let timer = abi::DeviceInfo { base: 0x0200bff8, size: 8 };
34	+	assert STORE.regions[1].info == timer;
35	+	assert not abi::permits(STORE.regions[1].rights, abi::Rights(abi::WRITE));
36	+	assert STORE.regions[2].kind == platform::Kind::Finish;
37	+	let mut rejected: u32 = 0;
38	+	for object in [abi::Ref { index: 3, generation: 1 }, abi::Ref { index: 0, generation: 2 }] {
39	+	try devices::get(&STORE, object) catch error {
40	+	assert error == abi::Error::BadHandle; set rejected += 1;
41	+	};
42	+	}
43	+	assert rejected == 2;
44	+	set MACHINE.devices[2].memory.end = 0x0200bfff;
45	+	try devices::initialize(&mut STORE, &MACHINE) catch error {
46	+	assert error == abi::Error::InvalidArg; set rejected += 1;
47	+	};
48	+	assert rejected == 3 and STORE.count == 0;
49	+	}
50	+
51	+	/// Device authority can move to a surviving domain and can only lose rights.
52	+	@test unsafe fn delegation() throws (testing::TestError) {
53	+	initialize();
54	+	set devices::STORE = STORE;
55	+	domains::initialize(&mut domains::STORE);
56	+	let mut owners: [abi::Ref; 2] = undefined;
57	+	for i in 0..2 {
58	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
59	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
60	+	set owners[i] = owner;
61	+	capability::initialize(&mut MEMORY[i].table, owner);
62	+	set domains::STORE.records[owner.index].state = domains::Lifecycle::Active;
63	+	set domains::STORE.records[owner.index].memory = &mut MEMORY[i];
64	+	}
65	+	let target = try! capability::install(&mut MEMORY[0].table, capability::Entry {
66	+	kind: abi::Kind::Domain, object: owners[1], rights: abi::Rights(0),
67	+	});
68	+	let device = try! devices::install(&STORE, &mut MEMORY[0].table, 2);
69	+	let original = try! capability::get(&MEMORY[0].table, device);
70	+	let granted = abi::Handle(try! calls::invoke(owners[0], 10, &[device, target, abi::READ as u64, 0], 0));
71	+	let reader = try! capability::get(&MEMORY[1].table, granted);
72	+	assert reader.object == original.object and reader.rights == abi::Rights(abi::READ);
73	+	let moved = abi::Handle(try! calls::invoke(owners[0], 11, &[device, target, *original.rights as u64, 0], 0));
74	+	assert (try! capability::get(&MEMORY[1].table, moved)) == original;
75	+	let old = try! abi::decode(device);
76	+	assert MEMORY[0].table.slots[old.object.index].state == slots::State::Free;
77	+	let dropped = try! calls::invoke(owners[1], 12, &[*moved, 0, 0, 0], 0);
78	+	assert dropped == 0;
79	+	assert (try! devices::get(&devices::STORE, reader.object)).info.base == 0x10001000;
80	+	}
81	+
82	+	/// Every register access checks current authority and the complete physical extent.
83	+	@test unsafe fn access() throws (testing::TestError) {
84	+	initialize();
85	+	capability::initialize(&mut MEMORY[0].table, abi::Ref { index: 0, generation: 1 });
86	+	let device = try! devices::install(&STORE, &mut MEMORY[0].table, 0);
87	+	for width in [1 as u64, 2, 4, 8] {
88	+	assert (try! devices::address(&STORE, &MEMORY[0].table, device, 256 - width, width, false)) == 0x10000100 - width;
89	+	assert (try! devices::address(&STORE, &MEMORY[0].table, device, 0, width, true)) == 0x10000000;
90	+	}
91	+	let mut rejected: u32 = 0;
92	+	for offset in [1 as u64, 253, 256, 0xffffffffffffffff] {
93	+	try devices::address(&STORE, &MEMORY[0].table, device, offset, 4, false) catch error {
94	+	assert error == abi::Error::InvalidArg; set rejected += 1;
95	+	};
96	+	}
97	+	for width in [0 as u64, 3, 16, 0x100000004] {
98	+	try devices::address(&STORE, &MEMORY[0].table, device, 0, width, false) catch error {
99	+	assert error == abi::Error::InvalidArg; set rejected += 1;
100	+	};
101	+	}
102	+	let clock = try! devices::install(&STORE, &mut MEMORY[0].table, 1);
103	+	try devices::address(&STORE, &MEMORY[0].table, clock, 0, 8, true) catch error {
104	+	assert error == abi::Error::Denied; set rejected += 1;
105	+	};
106	+	let removed = try! capability::invalidate(&mut MEMORY[0].table, device);
107	+	let replacement = try! devices::install(&STORE, &mut MEMORY[0].table, 0);
108	+	assert replacement <> device;
109	+	try devices::address(&STORE, &MEMORY[0].table, device, 0, 1, false) catch error {
110	+	assert error == abi::Error::BadHandle; set rejected += 1;
111	+	};
112	+	set STORE.regions[0].info.base = 0x10000001;
113	+	try devices::address(&STORE, &MEMORY[0].table, replacement, 0, 4, false) catch error {
114	+	assert error == abi::Error::InvalidArg; set rejected += 1;
115	+	};
116	+	assert rejected == 11;
117	+	}

kernel/kernel/tests/domains.rad +55 -0

        template: &[7], memory: 8, alignment: 8, relocations: &[],
    });
    set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
}

/// Failed root construction restores capacity and permits a fresh attempt.
@test unsafe fn bootstrapRollback() throws (testing::TestError) {
    initialize(abi::CREATE);
    let image = (try! capability::get(&TABLE, IMAGE)).object;
    domains::initialize(&mut DOMAINS);
    set BACKINGS.domains[0] = 0;
    let capacity = BACKINGS.pool.count;
    for count in [0 as u32, 1, 8, 32] {
        set BACKINGS.pool.count = count;
        let mut failed = false;
        try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
            assert error == abi::Error::OutOfMemory; set failed = true;
        };
        assert failed and DOMAINS.slots[0].state == slots::State::Free;
        assert DOMAINS.contextSlots[0].state == slots::State::Free;
        assert BACKINGS.domains[0] == 0 and DOMAINS.events.queues[0].generation == 0;
        for i in 0..count {
            assert frames::available(&BACKINGS.pool, i);
        }
    }
    set BACKINGS.pool.count = capacity;
    let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
    assert root.index == 0 and root.generation > 1;
    assert (try! domains::get(&DOMAINS, root)).parent == nil;
}

/// Root construction owns slot zero and installs the fixed bootstrap handles.
@test unsafe fn bootstrap() throws (testing::TestError) {
    initialize(abi::CREATE);
    let image = (try! capability::get(&TABLE, IMAGE)).object;
    domains::initialize(&mut DOMAINS);
    set BACKINGS.domains[0] = 0;
    let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
    assert root.index == 0 and root.generation == 1;
    let domain = try! domains::get(&DOMAINS, root);
    assert domain.creator == root and domain.parent == nil and domain.state == domains::Lifecycle::Pending;
    assert domain.memory.table.slots[0].state == slots::State::Live;
    let self = domain.memory.table.entries[0];
    assert self.kind == abi::Kind::Domain and self.object == root;
    assert *self.rights == (domains::MANAGEMENT_RIGHTS | abi::CREATE | abi::ALLOCATE);
    let queue = try! abi::decode(domain.events);
    assert queue.object.index == 1 and queue.kind == abi::Kind::Events;
    assert domain.memory.table.entries[1].object == root;
    assert DOMAINS.contexts[domain.initial.index].frame.pc == 0x80000000;
    assert DOMAINS.contexts[domain.initial.index].frame.registers[3] == domain.graph.table.ptr as u64;
    for word in DOMAINS.ancestors[0] {
        assert word == 0;
    }
    let mut busy = false;
    try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
        assert error == abi::Error::Busy; set busy = true;
    };
    assert busy and DOMAINS.slots[1].state == slots::State::Free;
}

/// Interrupted creation returns all private storage after its caller releases its hart.
@test unsafe fn cancelledCreation() throws (testing::TestError) {
    initialize(abi::CREATE);
    lifecycle::initialize(&mut CALLS);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);

kernel/mmio.rad added +62 -0

1	+	//! Fixed-width register access through live Device capabilities.
2	+	use super::abi;
3	+
4	+	/// Compiler primitive for one checked 8-bit register read.
5	+	@intrinsic fn deviceRead8(handle: u64, offset: u64) -> u8;
6	+	/// Compiler primitive for one checked 8-bit register write.
7	+	@intrinsic fn deviceWrite8(handle: u64, offset: u64, value: u8);
8	+
9	+	/// Read an aligned 8-bit register at a byte offset within the device.
10	+	export fn read8(handle: abi::Handle, offset: u64) -> u8 {
11	+	return deviceRead8(*handle, offset);
12	+	}
13	+
14	+	/// Write an aligned 8-bit register at a byte offset within the device.
15	+	export fn write8(handle: abi::Handle, offset: u64, value: u8) {
16	+	deviceWrite8(*handle, offset, value);
17	+	}
18	+
19	+	/// Compiler primitive for one checked 16-bit register read.
20	+	@intrinsic fn deviceRead16(handle: u64, offset: u64) -> u16;
21	+	/// Compiler primitive for one checked 16-bit register write.
22	+	@intrinsic fn deviceWrite16(handle: u64, offset: u64, value: u16);
23	+
24	+	/// Read an aligned 16-bit register at a byte offset within the device.
25	+	export fn read16(handle: abi::Handle, offset: u64) -> u16 {
26	+	return deviceRead16(*handle, offset);
27	+	}
28	+
29	+	/// Write an aligned 16-bit register at a byte offset within the device.
30	+	export fn write16(handle: abi::Handle, offset: u64, value: u16) {
31	+	deviceWrite16(*handle, offset, value);
32	+	}
33	+
34	+	/// Compiler primitive for one checked 32-bit register read.
35	+	@intrinsic fn deviceRead32(handle: u64, offset: u64) -> u32;
36	+	/// Compiler primitive for one checked 32-bit register write.
37	+	@intrinsic fn deviceWrite32(handle: u64, offset: u64, value: u32);
38	+
39	+	/// Read an aligned 32-bit register at a byte offset within the device.
40	+	export fn read32(handle: abi::Handle, offset: u64) -> u32 {
41	+	return deviceRead32(*handle, offset);
42	+	}
43	+
44	+	/// Write an aligned 32-bit register at a byte offset within the device.
45	+	export fn write32(handle: abi::Handle, offset: u64, value: u32) {
46	+	deviceWrite32(*handle, offset, value);
47	+	}
48	+
49	+	/// Compiler primitive for one checked 64-bit register read.
50	+	@intrinsic fn deviceRead64(handle: u64, offset: u64) -> u64;
51	+	/// Compiler primitive for one checked 64-bit register write.
52	+	@intrinsic fn deviceWrite64(handle: u64, offset: u64, value: u64);
53	+
54	+	/// Read an aligned 64-bit register at a byte offset within the device.
55	+	export fn read64(handle: abi::Handle, offset: u64) -> u64 {
56	+	return deviceRead64(*handle, offset);
57	+	}
58	+
59	+	/// Write an aligned 64-bit register at a byte offset within the device.
60	+	export fn write64(handle: abi::Handle, offset: u64, value: u64) {
61	+	deviceWrite64(*handle, offset, value);
62	+	}

kernel/root.rad added +62 -0

1	+	//! Initial resource handoff to the user-space scheduler.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Root startup memory supplied by the kernel.
6	+	record Env: Copy {
7	+	/// Writable header and copied resource handles.
8	+	argsPointer: *unsafe mut abi::BootInfo,
9	+	/// Header size in bytes.
10	+	argsSize: u64,
11	+	/// Root's event capability.
12	+	eventsHandle: u64,
13	+	/// Root's event-ring address.
14	+	eventsPointer: u64,
15	+	}
16	+
17	+	/// Transfer root's resource bundle and parent duties before normal exit.
18	+	@default unsafe fn main(env: *Env) -> ! {
19	+	assert env.argsSize == @sizeOf(abi::BootInfo) as u64;
20	+	let mut info = env.argsPointer;
21	+	let self = abi::Handle(info.handles[0]);
22	+	let scheduler = try! sys::domainCreate(self, info.schedulerImage);
23	+	let context = try! sys::queryContext(scheduler, 0);
24	+	let authority = try! sys::capabilityGrant(scheduler, scheduler, abi::RIGHTS_MASK as u64);
25	+	let stack = try! sys::pageAllocate(self, 16);
26	+	let stackInfo = sys::queryPage(stack);
27	+	let sharedStack = try! sys::capabilityTransfer(stack, scheduler, abi::RIGHTS_MASK as u64);
28	+	let mut handles = info.handles;
29	+	let clock = info.clock;
30	+	let shutdown = info.shutdown;
31	+	let budgets = 2 + info.devices as u32 + info.interrupts as u32;
32	+	for i in 2..budgets {
33	+	let old = abi::Handle(handles[i]);
34	+	let moved = try! sys::capabilityTransfer(old, scheduler, abi::RIGHTS_MASK as u64);
35	+	set handles[i] = *moved;
36	+	if old == clock {
37	+	set info.clock = moved;
38	+	}
39	+	if old == shutdown {
40	+	set info.shutdown = moved;
41	+	}
42	+	}
43	+	for i in budgets..handles.len {
44	+	let budget = abi::Handle(handles[i]);
45	+	let window = try! sys::queryBudget(budget);
46	+	if window.hart == 0 {
47	+	let future = try! sys::budgetSplit(budget, window.start + info.timebase * 2);
48	+	set handles[i] = *try! sys::budgetBind(future, scheduler, context.context);
49	+	} else {
50	+	set handles[i] = *try! sys::capabilityTransfer(budget, scheduler, abi::RIGHTS_MASK as u64);
51	+	}
52	+	}
53	+	set info.rootImage = try! sys::capabilityTransfer(info.rootImage, scheduler, abi::RIGHTS_MASK as u64);
54	+	set info.schedulerImage = try! sys::capabilityTransfer(info.schedulerImage, scheduler, abi::RIGHTS_MASK as u64);
55	+	set info.arguments = try! sys::capabilityTransfer(info.arguments, scheduler, abi::RIGHTS_MASK as u64);
56	+	set handles[0] = *authority;
57	+	set handles[1] = 0;
58	+	try! sys::domainReparent(self, scheduler);
59	+	try! sys::domainActivate(scheduler, stackInfo.base + stackInfo.count * 4096,
60	+	info as *unsafe opaque, @sizeOf(abi::BootInfo) as u64);
61	+	sys::exit(0);
62	+	}

kernel/scheduler.rad added +47 -0

1	+	//! User-space CPU windows and bounded worker execution.
2	+	export mod abi;
3	+	export mod sys;
4	+	export mod mmio;
5	+
6	+	/// Scheduler or worker startup values.
7	+	record Env: Copy {
8	+	/// Shared resource bundle for the scheduler.
9	+	argsPointer: *abi::BootInfo,
10	+	/// Zero for a worker, or the resource header size for the scheduler.
11	+	argsSize: u64,
12	+	/// Installed event capability.
13	+	eventsHandle: u64,
14	+	/// Shared event-ring address.
15	+	eventsPointer: u64,
16	+	}
17	+
18	+	/// Run a worker or distribute four successive CPU windows from the boot bundle.
19	+	@default unsafe fn main(env: *Env) -> ! {
20	+	if env.argsSize == 0 {
21	+	sys::exit(7);
22	+	}
23	+	assert env.argsSize == @sizeOf(abi::BootInfo) as u64;
24	+	let info = env.argsPointer;
25	+	let self = abi::Handle(info.handles[0]);
26	+	let started = mmio::read64(info.clock, 0);
27	+	assert sys::queryDomain(self).state == 1;
28	+	for turn in 0..4 {
29	+	let current = sys::currentContext();
30	+	let budget = abi::Handle(current.budget);
31	+	let window = try! sys::queryBudget(budget);
32	+	let later = try! sys::budgetSplit(budget, window.start + info.timebase);
33	+	let worker = try! sys::domainCreate(self, info.schedulerImage);
34	+	let stack = try! sys::pageAllocate(self, 16);
35	+	let extent = sys::queryPage(stack);
36	+	let transferred = try! sys::capabilityTransfer(stack, worker, abi::RIGHTS_MASK as u64);
37	+	try! sys::domainActivate(worker, extent.base + extent.count * 4096, (&env) as unsafe opaque, 0);
38	+	try! sys::yield(worker);
39	+	assert sys::currentContext().budget == *later;
40	+	}
41	+	assert mmio::read64(info.clock, 0) > started;
42	+	if info.shutdown <> abi::Handle(0) {
43	+	mmio::write32(info.shutdown, 0, 0x5555);
44	+	panic "shutdown returned";
45	+	}
46	+	sys::exit(0);
47	+	}

kernel/tools/build.rad +46 -21

use std::arch::rv64::shared;
use std::arch::rv64::shared::catalog;

/// Native entry page, above the platform firmware data.
constant CODE_ADDRESS: u64 = 0x81000000;
/// Maximum shared packages in the boot image.
constant PACKAGES: u32 = 4;
/// Package code arenas retained until image output completes.
unsafe static CODE: [[u8; 16777216]; 2] = undefined;
unsafe static CODE: [[u8; 16777216]; PACKAGES] = undefined;
/// Reusable function workspace.
static SCRATCH: [u8; 16777216] = [0; 16777216];
/// Persistent decoded RIL storage.
static DECODE: [u8; 67108864] = [0; 67108864];
/// Exact package bytes retained in the native catalog.
unsafe static INPUT: [[u8; 8388608]; 2] = undefined;
unsafe static INPUT: [[u8; 8388608]; PACKAGES] = undefined;
/// Combined startup and boundary assembly source.
static SOURCE: [u8; 65536] = [0; 65536];
/// Assembler workspace.
static ASSEMBLY: [u8; 4194304] = [0; 4194304];
/// Assembled startup words.
static TEXT: [u32; 16384] = [0; 16384];
/// Assembly identifiers retained through package linking.
unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };
/// Per-package data layout workspaces.
unsafe static DATA: [[data::DataSym; 4096]; 2] = undefined;
unsafe static DATA: [[data::DataSym; 4096]; PACKAGES] = undefined;
/// Local function and data definitions.
unsafe static SYMBOLS: [[shared::Symbol; 4096]; 2] = undefined;
unsafe static SYMBOLS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
/// Public package symbols.
unsafe static EXPORTS: [[shared::Symbol; 4096]; 2] = undefined;
unsafe static EXPORTS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
/// Exports from dependencies compiled before the current package.
unsafe static IMPORTS: [shared::Symbol; 16384] = undefined;
/// Private-state relocation records.
unsafe static RELOCS: [[shared::Relocation; 4096]; 2] = undefined;
unsafe static RELOCS: [[shared::Relocation; 4096]; PACKAGES] = undefined;
/// Initialized package templates.
unsafe static TEMPLATES: [[u8; 1048576]; 2] = undefined;
unsafe static TEMPLATES: [[u8; 1048576]; PACKAGES] = undefined;
/// Read-only catalog and retained package payloads.
static RO: [u8; 16777216] = [0; 16777216];
/// Kernel state table and initialized private package graph.
static RW: [u8; 33554432] = [0; 33554432];
/// Contiguous native code segment, including entry trampoline and alignment padding.

            template: &mut TEMPLATES[slot][..], relocations: &mut RELOCS[slot][..],
        }, &mut arena, &mut scratch);
}

/// Emit a trampoline that preserves firmware arguments and supplies the native catalog.
unsafe fn trampoline(kernel: u64, ro: u64, rw: u64) {
unsafe fn trampoline(kernel: u64, ro: u64, rw: u64, count: u32) {
    let mut arena = alloc::new(&mut SCRATCH[..]);
    let mut e = try! emit::emitter(&mut arena, false);
    emit::loadImm(&mut e, rv64::GP, rw as i64);
    emit::loadImm(&mut e, rv64::A3, ro as i64);
    emit::loadImm(&mut e, rv64::A4, 2);
    emit::loadImm(&mut e, rv64::A4, count as i64);
    emit::loadImm(&mut e, rv64::T0, kernel as i64);
    emit::emit(&mut e, encode::jalr(rv64::ZERO, rv64::T0, 0));
    assert e.codeLen <= 1024;
    for word, i in emit::getCode(&e) {
        set NATIVE[i] = word;
    }
}

/// Link shared boot packages, retain their catalog, and instantiate the kernel graph.
@default unsafe fn main(env: *sys::Env) -> i32 {
    assert env.args.len == 5;
    assert env.args.len >= 5 and env.args.len <= PACKAGES + 3;
    let count = env.args.len - 3;
    let mut decoder = alloc::new(&mut DECODE[..]);
    let libraryBytesLength = unix::readFile(env.args[1], &mut INPUT[0][..]) else panic "std RIL";
    let libraryBytes = &INPUT[0][..libraryBytesLength];
    let libraryInput = try! program::decode(libraryBytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
    let kernelBytesLength = unix::readFile(env.args[2], &mut INPUT[1][..]) else panic "kernel RIL";

    assert libraryInput.dependencies.len == 0;
    for dependency in kernelInput.dependencies {
        assert mem::eq(dependency, "std");
    }
    let sourceLength = unix::readFile(env.args[3], &mut SOURCE[..]) else panic "kernel assembly";
    let source = &SOURCE[..sourceLength];
    let mut assemblyArena = alloc::new(&mut ASSEMBLY[..]);
    let empty: *mut [u8] = &mut [];
    let assembly = try! asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..sourceLength],
    let assembly = try! asm::assemble(asm::scanner::SourceKind::String, source,
        &mut TEXT[..], &mut empty[..], &mut assemblyArena, &mut STRINGS, 0);
    let library = try compile(&libraryInput, 0, CODE_ADDRESS + 4096, &[],
        asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error {
            report(error);
            return 1;

    let kernelAddress = aligned(library.codeAddress + library.code.len as u64 * 4, 4096);
    let kernel = try compile(&kernelInput, 1, kernelAddress, library.exports, assembly) catch error {
        report(error);
        return 1;
    };
    let entries = [
        catalog::Entry { source: libraryBytes, package: library },
        catalog::Entry { source: kernelBytes, package: kernel },
    ];
    let codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
    let mut catalogEntries: [catalog::Entry; PACKAGES] = undefined;
    set catalogEntries[0] = catalog::Entry { source: libraryBytes, package: library };
    set catalogEntries[1] = catalog::Entry { source: kernelBytes, package: kernel };
    let mut codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
    for slot in 2..count {
        let bytesLength = unix::readFile(env.args[slot + 3], &mut INPUT[slot][..]) else panic "boot package RIL";
        let bytes = &INPUT[slot][..bytesLength];
        let input = try! program::decode(bytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
        let mut imported: u32 = 0;
        for previous in 0..slot {
            for symbol in catalogEntries[previous].package.exports {
                assert imported < IMPORTS.len;
                set IMPORTS[imported] = symbol;
                set imported += 1;
            }
        }
        let package = try compile(&input, slot, aligned(codeEnd, 4096), &IMPORTS[..imported],
            asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error {
                report(error);
                return 1;
            };
        set catalogEntries[slot] = catalog::Entry { source: bytes, package };
        set codeEnd = package.codeAddress + package.code.len as u64 * 4;
    }
    let codeSize = (codeEnd - CODE_ADDRESS) as u32;
    assert codeEnd - CODE_ADDRESS <= @sizeOf([u32; 4194304]) as u64;
    let roAddress = aligned(codeEnd, 4096);
    let roSize = try catalog::pack(&entries[..], roAddress, &mut RO[..]) catch error {
    let roSize = try catalog::pack(&catalogEntries[..count], roAddress, &mut RO[..]) catch error {
        report(error);
        return 1;
    };
    let rwAddress = aligned(roAddress + roSize as u64, 4096);
    let mut bases: [u64; shared::MAX_PACKAGES] = [0; shared::MAX_PACKAGES];
    let mut rwSize: u32 = @sizeOf([u64; shared::MAX_PACKAGES]);
    for entry in &entries[..] {
    for entry in &catalogEntries[..2] {
        let at = aligned(rwAddress + rwSize as u64, entry.package.alignment);
        assert at - rwAddress <= RW.len as u64 and entry.package.memory <= RW.len - (at - rwAddress) as u32;
        set bases[entry.package.slot] = at;
        set rwSize = (at - rwAddress) as u32 + entry.package.memory;
    }
    try! mem::copy(&mut RW[..@sizeOf([u64; shared::MAX_PACKAGES])],
        @sliceOf(&bases[0] as *unsafe u8, @sizeOf([u64; shared::MAX_PACKAGES])));
    for entry in &entries[..] {
    for entry in &catalogEntries[..2] {
        let at = (bases[entry.package.slot] - rwAddress) as u32;
        try shared::instantiate(&entry.package, &bases[..], &mut RW[at..at + entry.package.memory])
            catch error {
                report(error);
                return 1;
            };
    }
    for i in 0..codeSize / 4 {
        set NATIVE[i] = encode::nop();
    }
    for entry in &entries[..] {
    for entry in &catalogEntries[..count] {
        let at = ((entry.package.codeAddress - CODE_ADDRESS) / 4) as u32;
        for word, i in entry.package.code {
            set NATIVE[at + i] = word;
        }
    }
    trampoline(kernelAddress, roAddress, rwAddress);
    trampoline(kernelAddress, roAddress, rwAddress, count);
    let header = try! image::header(image::Layout {
        entry: CODE_ADDRESS,
        code: image::Segment { address: CODE_ADDRESS, initialized: codeSize, memory: codeSize },
        roData: image::Segment { address: roAddress, initialized: roSize, memory: roSize },
        rwData: image::Segment { address: rwAddress, initialized: rwSize, memory: rwSize },

test/acceptance/replay added +27 -0

1	+	#!/bin/sh
2	+	# Compare complete output from two executions of the same native image.
3	+	set -eu
4	+	if [ "$#" -lt 4 ]; then
5	+	printf 'usage: replay profile emulator image harts [machine-options...]\n' >&2
6	+	exit 1
7	+	fi
8	+	profile=$1
9	+	emulator=$2
10	+	image=$3
11	+	harts=$4
12	+	shift 4
13	+	mkdir -p bin/acceptance/traces
14	+	prefix="bin/acceptance/traces/$profile-$harts"
15	+	for attempt in 1 2; do
16	+	if ! "$emulator" -machine -harts="$harts" -memory-size=262144 -max-steps=2000000000 "$@" -run "$image" > "$prefix.$attempt.log" 2>&1; then
17	+	cat "$prefix.$attempt.log"
18	+	exit 1
19	+	fi
20	+	done
21	+	if ! cmp -s "$prefix.1.log" "$prefix.2.log"; then
22	+	diff -u "$prefix.1.log" "$prefix.2.log" \|\| true
23	+	printf 'replay: %s %s-hart output differs\n' "$profile" "$harts" >&2
24	+	exit 1
25	+	fi
26	+	cat "$prefix.1.log"
27	+	printf 'replay: %s %s-hart output matched\n' "$profile" "$harts"

test/boot/run +5 -3

set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
cat test/boot/machine.ras kernel/kernel/*.ras > "$work/boot.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
for harts in 1 2 8; do
    # The deterministic emulator shares its tick budget across all harts.
    steps=$((10000000 * harts))
    if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then
        cat "$work/log" >&2

        cat "$work/log" >&2
        exit 1
    fi
    printf 'kernel boot: %s harts passed\n' "$harts"
done
cat kernel/kernel/*.ras > "$work/idle.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/idle.ras" "$work/idle.rv64"
status=0
"$emulator" -machine -harts=8 -max-steps=80000000 -run bin/kernel.rv64 > "$work/log" 2>&1 || status=$?
"$emulator" -machine -harts=8 -max-steps=80000000 -run "$work/idle.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
    cat "$work/log" >&2
    exit 1
fi
printf 'kernel startup: all eight harts reached machine idle without traps\n'
cat test/boot/trap.ras kernel/kernel/*.ras > "$work/trap.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
status=0
"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
    || [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then
    cat "$work/log" >&2

test/bootstrap/kernel/dispatchcheck.rad added +88 -0

1	+	//! Native root construction, exit, and continued resource authority.
2	+	use kernel::abi;
3	+	use kernel::boot;
4	+	use kernel::registry;
5	+	use kernel::domains;
6	+	use kernel::capability;
7	+	use kernel::pages;
8	+	use kernel::budgets;
9	+	use kernel::dispatch;
10	+	use kernel::calls;
11	+	use kernel::events;
12	+	use kernel::sync;
13	+	use kernel::startup;
14	+	use kernel::devices;
15	+	use kernel::interrupts;
16	+
17	+	/// Initial root identity retained by the terminal event.
18	+	unsafe static ROOT: abi::Ref = undefined;
19	+	/// Surviving controller domain with inherited resource authority.
20	+	unsafe static CONTROLLER: abi::Ref = undefined;
21	+	/// Image capability installed in the controller's own table.
22	+	unsafe static IMAGE: abi::Handle = undefined;
23	+	/// Current kernel package-state table.
24	+	fn kernelGp() -> u64;
25	+	/// Controller entry address.
26	+	fn completion() -> u64;
27	+	/// Signal successful completion to the emulator.
28	+	fn finish();
29	+	/// Map an assembly-compatible physical byte address.
30	+	fn memory(address: u64) -> *mut u8;
31	+
32	+	/// Construct root from the boot catalog and give its child a later CPU window.
33	+	export unsafe fn setup() {
34	+	let image = registry::find(&registry::STORE, &"loaded"[..]) else panic "missing boot user package";
35	+	let service = registry::find(&registry::STORE, &"service"[..]) else panic "missing boot service package";
36	+	set ROOT = try! startup::create(&boot::PLATFORM, image, service);
37	+	assert ROOT.index == 0;
38	+	let mut root = try! domains::get(&domains::STORE, ROOT);
39	+	let self = try! abi::handle(abi::Kind::Domain, abi::Ref { index: 0, generation: root.memory.table.slots[0].generation });
40	+	assert (try! abi::decode(root.events)).object.index == 1;
41	+	let executable = try! registry::install(&registry::STORE, &mut root.memory.table, image);
42	+	let childHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings,
43	+	&registry::STORE, &mut root.memory.table, self, executable);
44	+	set CONTROLLER = (try! capability::get(&root.memory.table, childHandle)).object;
45	+	let mut child = try! domains::get(&domains::STORE, CONTROLLER);
46	+	let inherited = domains::MANAGEMENT_RIGHTS \| abi::CREATE \| abi::ALLOCATE;
47	+	let authority = try! calls::invoke(ROOT, 10, &[childHandle, childHandle, inherited as u64, 0], 0);
48	+	assert service.index == 3 and image.index == 2;
49	+	set IMAGE = try! registry::install(&registry::STORE, &mut child.memory.table, service);
50	+	try! domains::reparent(&mut domains::STORE, &root.memory.table, self, childHandle);
51	+	set domains::STORE.records[CONTROLLER.index].state = domains::Lifecycle::Active;
52	+	let frame = &mut domains::STORE.contexts[child.initial.index].frame;
53	+	set frame.pc = completion(); set frame.status = 0x1880;
54	+	set frame.registers[2] = domains::STORE.contexts[child.initial.index].kernelStack.end;
55	+	set frame.registers[3] = kernelGp();
56	+	let now = dispatch::clock();
57	+	let index = 2 + devices::STORE.count + interrupts::STORE.count;
58	+	let budget = try! abi::handle(abi::Kind::Budget, abi::Ref { index, generation: root.memory.table.slots[index].generation });
59	+	let window = try! budgets::query(&budgets::STORE, &root.memory.table, budget);
60	+	let start = window.start;
61	+	let later = try! budgets::split(&mut budgets::STORE, &mut root.memory.table, budget, start + 10000000, now);
62	+	let running = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut root.memory.table,
63	+	budgets::Binding { budget, domain: self, context: root.initial }, now);
64	+	let transferred = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut root.memory.table,
65	+	budgets::Binding { budget: later, domain: childHandle, context: child.initial }, now);
66	+	}
67	+
68	+	/// Root exit leaves the survivor able to allocate and create new domains.
69	+	export unsafe fn verify() {
70	+	let guard = sync::enter();
71	+	let mut child = try! domains::get(&domains::STORE, CONTROLLER);
72	+	let terminal = try! events::pop(&mut child.memory.ring) else panic "missing root exit";
73	+	assert terminal.kind == events::CHILD_EXIT and terminal.code == 0 and terminal.value == abi::id(ROOT);
74	+	assert child.state == domains::Lifecycle::Active and child.parent == nil;
75	+	try! events::refresh(&mut domains::STORE.events, CONTROLLER, &child.memory.ring);
76	+	sync::leave(guard);
77	+	let page = abi::Handle(try! calls::synchronized(CONTROLLER, 30, &[0, 1, 0, 0], dispatch::clock()));
78	+	let domain = abi::Handle(try! calls::synchronized(CONTROLLER, 20, &[0, *IMAGE, 0, 0], dispatch::clock()));
79	+	let checking = sync::enter();
80	+	let allocated = try! capability::get(&child.memory.table, page);
81	+	assert (try! pages::get(&pages::STORE, allocated.object)).count == 1;
82	+	let object = (try! capability::get(&child.memory.table, domain)).object;
83	+	let created = try! domains::get(&domains::STORE, object);
84	+	assert created.creator == CONTROLLER and created.parent == CONTROLLER;
85	+	assert created.state == domains::Lifecycle::Pending;
86	+	sync::leave(checking);
87	+	finish();
88	+	}

test/bootstrap/machine.ras added +25 -0

1	+	//! Boot one hart and check user termination through production dispatch.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::setup;
5	+	call @kernel::boot::run;
6	+	.export @kernel::dispatchcheck::memory;
7	+	.export @kernel::dispatchcheck::kernelGp;
8	+	.export @kernel::dispatchcheck::completion;
9	+	.export @kernel::dispatchcheck::finish;
10	+	@kernel::dispatchcheck::memory
11	+	ret;
12	+	@kernel::dispatchcheck::kernelGp
13	+	mv %a0 %gp;
14	+	ret;
15	+	@kernel::dispatchcheck::completion
16	+	la %a0 @complete;
17	+	ret;
18	+	@complete
19	+	call @kernel::dispatchcheck::verify;
20	+	ebreak;
21	+	@kernel::dispatchcheck::finish
22	+	li %t0 0x10001000;
23	+	li %t1 0x5555;
24	+	sw %t1 0(%t0);
25	+	ebreak;

test/bootstrap/root.rad added +48 -0

1	+	//! Validate the root startup bundle from user mode.
2	+	export mod abi;
3	+	export mod sys;
4	+
5	+	/// Typed startup view used by the boot root package.
6	+	record Env: Copy {
7	+	/// Root-owned resource bundle.
8	+	argsPointer: *abi::BootInfo,
9	+	/// Byte size of the bundle header.
10	+	argsSize: u64,
11	+	/// Installed Events capability.
12	+	eventsHandle: u64,
13	+	/// Shared event-ring address.
14	+	eventsPointer: u64,
15	+	}
16	+
17	+	/// Check root identity, resource ordering, and image authority before returning.
18	+	@default unsafe fn main(env: *Env) {
19	+	assert env.argsSize == @sizeOf(abi::BootInfo) as u64;
20	+	let info = env.argsPointer;
21	+	let handles = info.handles;
22	+	let self = sys::queryDomain(abi::Handle(handles[0]));
23	+	assert (self.id & 0xffffffff) == 0 and self.creator == self.id;
24	+	assert handles[1] == env.eventsHandle and info.timebase > 0;
25	+	let mut harts: u64 = 0;
26	+	for hart in 0..8 {
27	+	if (info.harts & (1 << hart as u64)) <> 0 {
28	+	set harts += 1;
29	+	}
30	+	}
31	+	assert handles.len as u64 == 2 + info.devices + info.interrupts + harts;
32	+	for i in 0..info.devices as u32 {
33	+	let device = sys::queryDevice(abi::Handle(handles[2 + i]));
34	+	assert device.size > 0;
35	+	}
36	+	for i in 0..info.interrupts as u32 {
37	+	let irq = sys::queryInterrupt(abi::Handle(handles[2 + info.devices as u32 + i]));
38	+	assert irq.number == i + 1;
39	+	}
40	+	assert sys::queryDevice(info.clock).size == 8;
41	+	assert sys::queryDevice(info.shutdown).size >= 4;
42	+	assert (try! sys::queryImage(info.rootImage)).entry <> 0;
43	+	assert (try! sys::queryImage(info.schedulerImage)).entry <> 0;
44	+	let current = sys::currentContext();
45	+	let first = 2 + info.devices as u32 + info.interrupts as u32;
46	+	assert current.hart == 0 and current.budget == handles[first];
47	+	assert (try! sys::queryBudget(abi::Handle(handles[first]))).hart == 0;
48	+	}

test/bootstrap/run added +28 -0

1	+	#!/bin/sh
2	+	# Construct root from a boot package and retain authority after root exits.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	work=$(mktemp -d)
6	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
7	+	mkdir "$work/loaded"
8	+	cp test/bootstrap/root.rad "$work/loaded.rad"
9	+	cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/loaded/"
10	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
11	+	-pkg loaded -mod "$work/loaded.rad" -mod "$work/loaded/abi.rad" -mod "$work/loaded/sys.rad" -entry loaded -ril "$work"
12	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
13	+	-pkg service -mod test/bootstrap/service.rad -entry service -ril "$work"
14	+	cp kernel/kernel.rad "$work/kernel.rad"
15	+	printf '\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
16	+	mkdir "$work/kernel"
17	+	cp kernel/kernel/*.rad "$work/kernel/"
18	+	cp test/bootstrap/kernel/dispatchcheck.rad "$work/kernel/"
19	+	sh test/acceptance/compile "$emulator" "$work"
20	+	cat test/bootstrap/machine.ras kernel/kernel/*.ras > "$work/bootstrap.ras"
21	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
22	+	-- "$work/std.ril" "$work/kernel.ril" "$work/bootstrap.ras" "$work/bootstrap.rv64" "$work/loaded.ril" "$work/service.ril"
23	+	if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
24	+	sh test/acceptance/replay bootstrap "$emulator" "$work/bootstrap.rv64" 1
25	+	else
26	+	"$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/bootstrap.rv64"
27	+	fi
28	+	printf 'bootstrap: root exit preserves allocation and creation authority\n'

test/bootstrap/service.rad added +6 -0

1	+	//! Independent boot package used for creation after root exit.
2	+
3	+	/// Return a small status from the service entry.
4	+	@default fn main() -> u64 {
5	+	return 7;
6	+	}

test/dispatch/run +20 -4

work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
set -- -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad"
if [ "$fixture" = mmio ]; then
    cp kernel/mmio.rad "$work/spin/"
    set -- "$@" -mod "$work/spin/mmio.rad"
fi
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
    "$@" -entry spin -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"
cp "test/$fixture/kernel/dispatchcheck.rad" "$work/kernel/"
for source in "test/$fixture"/kernel/*.rad; do
    cp "$source" "$work/kernel/"
    module=${source##*/}
    module=${module%.rad}
    if [ "$module" != dispatchcheck ]; then
        printf '\nexport mod %s;\n' "$module" >> "$work/kernel.rad"
    fi
done
length=$(wc -c < "$work/spin.ril")
printf '//! Binary user loop.\n/// Complete trusted input package.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/dispatchinput.rad"
od -An -v -tu1 "$work/spin.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
set --
if [ "$irq" -ne 0 ]; then set -- "-irq=$irq" "-irq-at=1"; fi
for count in $harts; do
    if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
        sh test/acceptance/replay "$fixture" "$emulator" "$work/dispatch.rv64" "$count" "$@"
    else
    "$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 "$@" -run "$work/dispatch.rv64"
    fi
    printf '%s: %s-hart execution passed\n' "$fixture" "$count"
done

test/loader/run +3 -3

# Build a kernel fixture and call code produced by its runtime loader.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev -pkg loaded -mod test/loader/loaded.rad -entry loaded -ril "$work"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev -pkg loaded -mod test/loader/loaded.rad -entry loaded -o "$work/hosted.rv64"
status=0
"$emulator" -run "$work/hosted.rv64" || status=$?
test "$status" -eq 43
cat kernel/kernel.rad > "$work/kernel.rad"

printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
printf '];\n' >> "$work/kernel/loadinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/loader/machine.ras kernel/kernel/*.ras > "$work/loader.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
printf 'runtime loader: hosted and loaded entries agree after workspace release\n'

test/mmio/kernel/dispatchcheck.rad added +149 -0

1	+	//! Native fixed-width MMIO and authority rejection before register access.
2	+	use std::mem;
3	+	use kernel::abi;
4	+	use kernel::slots;
5	+	use kernel::backing;
6	+	use kernel::pages;
7	+	use kernel::capability;
8	+	use kernel::registry;
9	+	use kernel::loader;
10	+	use kernel::domains;
11	+	use kernel::budgets;
12	+	use kernel::dispatch;
13	+	use kernel::boot;
14	+	use kernel::events;
15	+	use kernel::sync;
16	+	use kernel::dispatchinput;
17	+	use kernel::devices;
18	+	use kernel::platform;
19	+
20	+	/// Register storage exposed only through a synthetic Device capability.
21	+	static REGISTERS: [u64; 512] = [0; 512];
22	+
23	+	/// Bootstrap creation, allocation, and budget authority.
24	+	unsafe static TABLE: capability::Table = undefined;
25	+	/// Child domain identities retained by terminal events.
26	+	unsafe static CHILDREN: [abi::Ref; 9] = undefined;
27	+	/// Surviving event receiver and native completion checker.
28	+	unsafe static PARENT: abi::Ref = undefined;
29	+	/// Map a validated physical page.
30	+	fn memory(address: u64) -> *mut u8;
31	+	/// Current kernel package-state base.
32	+	fn kernelGp() -> u64;
33	+	/// Native entry for the completion checker.
34	+	fn completion() -> u64;
35	+	/// Signal successful completion to the machine fixture.
36	+	fn finish();
37	+
38	+	/// Give each child a disjoint window, followed by the surviving parent.
39	+	export unsafe fn setup() {
40	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
41	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
42	+	capability::initialize(&mut TABLE, owner);
43	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
44	+	let authority = try! capability::install(&mut TABLE, capability::Entry {
45	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
46	+	});
47	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
48	+	let page = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, source)).object);
49	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
50	+	let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
51	+	let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
52	+	loader::Request { authority, source, offset: 0, length: length as u64 });
53	+	let parentHandle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
54	+	set PARENT = (try! capability::get(&TABLE, parentHandle)).object;
55	+	let parent = try! domains::get(&domains::STORE, PARENT);
56	+	set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;
57	+	set domains::STORE.contexts[parent.initial.index].frame.pc = completion();
58	+	set domains::STORE.contexts[parent.initial.index].frame.status = 0x1880;
59	+	set domains::STORE.contexts[parent.initial.index].frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;
60	+	set domains::STORE.contexts[parent.initial.index].frame.registers[3] = kernelGp();
61	+	let scratch = devices::STORE.count;
62	+	set devices::STORE.regions[scratch] = {
63	+	kind: platform::Kind::Uart,
64	+	info: { base: (&mut REGISTERS[0]) as u64, size: 4096 },
65	+	rights: abi::Rights(abi::READ \| abi::WRITE \| abi::GRANT \| abi::TRANSFER),
66	+	};
67	+	set devices::STORE.count += 1;
68	+	let mut clockDevice: u32 = 0;
69	+	let mut finish: u32 = 0;
70	+	for i in 0..scratch {
71	+	if devices::STORE.regions[i].kind == platform::Kind::Clint {
72	+	set clockDevice = i;
73	+	}
74	+	if devices::STORE.regions[i].kind == platform::Kind::Finish {
75	+	set finish = i;
76	+	}
77	+	}
78	+	let mut handles: [abi::Handle; 9] = undefined;
79	+	let mut contexts: [abi::Ref; 9] = undefined;
80	+	for i in 0..9 {
81	+	let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
82	+	let object = (try! capability::get(&TABLE, handle)).object;
83	+	let mut child = try! domains::get(&domains::STORE, object);
84	+	set CHILDREN[i] = object;
85	+	set handles[i] = handle;
86	+	set contexts[i] = child.initial;
87	+	try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);
88	+	let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
89	+	let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
90	+	let storage = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, stack)).object);
91	+	let mut region = scratch;
92	+	if i == 2 or i == 6 {
93	+	set region = finish;
94	+	}
95	+	let mut device = try! devices::install(&devices::STORE, &mut child.memory.table, region);
96	+	let slot = (try! abi::decode(device)).object.index;
97	+	if i == 2 {
98	+	set child.memory.table.entries[slot].rights = abi::Rights(abi::READ);
99	+	}
100	+	if i == 3 {
101	+	set child.memory.table.entries[slot].rights = abi::Rights(abi::WRITE);
102	+	}
103	+	if i == 1 {
104	+	set device = abi::Handle(0);
105	+	}
106	+	if i == 7 {
107	+	set device = granted;
108	+	}
109	+	let timerHandle = try! devices::install(&devices::STORE, &mut child.memory.table, clockDevice);
110	+	let args = @sliceOf(memory(storage.base) as *mut u64, 2);
111	+	set args[0] = device; set args[1] = timerHandle;
112	+	let size = 16 + i as u64;
113	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, size);
114	+	}
115	+	let timer = try! dispatch::timer(&boot::PLATFORM, 0);
116	+	let clock = dispatch::now(timer);
117	+	let start = clock + 1000000;
118	+	let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 20000000);
119	+	for i in 0..9 {
120	+	let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, remaining, start + (i as u64 + 1) * 2000000, clock);
121	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
122	+	budgets::Binding { budget: remaining, domain: handles[i], context: contexts[i] }, clock);
123	+	set remaining = next;
124	+	}
125	+	let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
126	+	budgets::Binding { budget: remaining, domain: parentHandle, context: parent.initial }, clock);
127	+	}
128	+
129	+	/// Check each terminal kind, status, and generation while the parent continues to run.
130	+	export unsafe fn verify() {
131	+	let guard = sync::enter();
132	+	let mut parent = try! domains::get(&domains::STORE, PARENT);
133	+	let codes = [61 as u32, 256, 256, 256, 256, 256, 256, 256, 256];
134	+	for i in 0..9 {
135	+	let event = try! events::pop(&mut parent.memory.ring) else panic "missing termination event";
136	+	assert event.value == abi::id(CHILDREN[i]) and event.code == codes[i];
137	+	if i == 0 {
138	+	assert event.kind == events::CHILD_EXIT;
139	+	}
140	+	else {
141	+	assert event.kind == events::FAULT;
142	+	}
143	+	assert domains::STORE.records[CHILDREN[i].index].state == domains::Lifecycle::Dead;
144	+	}
145	+	try! events::refresh(&mut domains::STORE.events, PARENT, &parent.memory.ring);
146	+	assert try! events::pop(&mut parent.memory.ring) == nil;
147	+	sync::leave(guard);
148	+	finish();
149	+	}

test/mmio/machine.ras added +25 -0

1	+	//! Boot one hart and check user termination through production dispatch.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::dispatchcheck::setup;
5	+	call @kernel::boot::run;
6	+	.export @kernel::dispatchcheck::memory;
7	+	.export @kernel::dispatchcheck::kernelGp;
8	+	.export @kernel::dispatchcheck::completion;
9	+	.export @kernel::dispatchcheck::finish;
10	+	@kernel::dispatchcheck::memory
11	+	ret;
12	+	@kernel::dispatchcheck::kernelGp
13	+	mv %a0 %gp;
14	+	ret;
15	+	@kernel::dispatchcheck::completion
16	+	la %a0 @complete;
17	+	ret;
18	+	@complete
19	+	call @kernel::dispatchcheck::verify;
20	+	ebreak;
21	+	@kernel::dispatchcheck::finish
22	+	li %t0 0x10001000;
23	+	li %t1 0x5555;
24	+	sw %t1 0(%t0);
25	+	ebreak;

test/mmio/spin.rad added +66 -0

1	+	//! User-mode register widths and rejected MMIO authority.
2	+	export mod abi;
3	+	export mod sys;
4	+	export mod mmio;
5	+
6	+	/// Device values installed in the calling domain's table.
7	+	record Arguments: Copy {
8	+	/// Register interface under test.
9	+	device: abi::Handle,
10	+	/// Read-only machine timer.
11	+	clock: abi::Handle,
12	+	}
13	+
14	+	/// Case selection and readable arguments supplied by the test parent.
15	+	record Env: Copy {
16	+	/// Installed device handles.
17	+	argsPointer: *Arguments,
18	+	/// Sixteen plus the test case number.
19	+	argsSize: u64,
20	+	/// Installed event capability.
21	+	eventsHandle: u64,
22	+	/// Shared ring address.
23	+	eventsPointer: u64,
24	+	}
25	+
26	+	/// Execute each width, or require a kernel fault before an unauthorized access.
27	+	@default fn main(env: *Env) {
28	+	let device = env.argsPointer.device;
29	+	let scenario = env.argsSize - 16;
30	+	if scenario == 0 {
31	+	let before = mmio::read64(env.argsPointer.clock, 0);
32	+	mmio::write8(device, 0, 0x81);
33	+	mmio::write16(device, 2, 0x8123);
34	+	mmio::write32(device, 4, 0x81234567);
35	+	mmio::write64(device, 8, 0x8123456789abcdef);
36	+	assert mmio::read8(device, 0) == 0x81;
37	+	assert mmio::read16(device, 2) == 0x8123;
38	+	assert mmio::read32(device, 4) == 0x81234567;
39	+	assert mmio::read64(device, 8) == 0x8123456789abcdef;
40	+	assert mmio::read32(env.argsPointer.clock, 0) as u64 >= before;
41	+	sys::exit(61);
42	+	}
43	+	if scenario == 1 or scenario == 2 {
44	+	mmio::write32(device, 0, 0x5555);
45	+	}
46	+	if scenario == 3 {
47	+	let value = mmio::read8(device, 0);
48	+	}
49	+	if scenario == 4 {
50	+	mmio::write32(device, 1, 1);
51	+	}
52	+	if scenario == 5 {
53	+	let value = mmio::read8(device, 4096);
54	+	}
55	+	if scenario == 6 {
56	+	try! sys::capabilityDrop(device);
57	+	mmio::write32(device, 0, 0x5555);
58	+	}
59	+	if scenario == 7 {
60	+	let value = mmio::read8(device, 0);
61	+	}
62	+	if scenario == 8 {
63	+	mmio::write64(device, 0xfffffffffffffff8, 1);
64	+	}
65	+	sys::exit(99);
66	+	}

test/modules/run +1 -1

    set -- "$@" -mod "$work/capacity/m$index.rad"
    index=$((index + 1))
done
printf 'use capacity::m126;\n/// Check a package root beyond module index 127.\n@default fn main() -> u64 { assert m126::value() == 126; return 0; }\n' > "$work/entry.rad"
set -- "$@" -pkg entry -mod "$work/entry.rad"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev "$@" -entry entry -o "$work/capacity.rv64"
"$emulator" -run "$work/capacity.rv64"
printf 'module scopes: references beyond index 127 passed\n'

test/packages/run +2 -2

emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/first" "$work/second"
for output in first second; do
    "$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    "$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
        -run bin/radiance.rv64.dev \
        -pkg support -mod test/packages/support.rad \
        -pkg app -mod test/packages/app.rad -entry app -ril "$work/$output"
done
cmp "$work/first/support.ril" "$work/second/support.ril"
cmp "$work/first/app.ril" "$work/second/app.ril"
"$emulator" -run bin/packages.check.rv64 -- "$work/first/support.ril" "$work/first/app.ril"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev \
    -pkg support -mod test/packages/support.rad \
    -pkg app -mod test/packages/app.rad -entry app -o "$work/hosted.rv64"
status=0
"$emulator" -run "$work/hosted.rv64" || status=$?

test/runtime/run +8 -4

work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp "test/$fixture/spin.rad" "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
cp test/loader/loaded.rad "$work/loaded.rad"
index=0
while [ "$index" -lt 128 ]; do
    printf '\n/// Compute a distinct exported value.\nexport fn value%s(input: u64) -> u64 { return input + %s; }\n' "$index" "$index" >> "$work/loaded.rad"
    index=$((index + 1))
done
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
    -pkg loaded -mod "$work/loaded.rad" -entry loaded -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"

printf '\n/// Package compiled by scheduled user calls.\nexport static PAYLOAD: [u8; %s] = [\n' "$length" >> "$work/kernel/dispatchinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
printf '];\n' >> "$work/kernel/dispatchinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
"$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
    sh test/acceptance/replay runtime "$emulator" "$work/dispatch.rv64" 1
else
    "$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
fi
printf 'runtime: scheduled cancellation, retry, and publication passed\n'

test/scheduling/kernel/dispatchcheck.rad added +76 -0

1	+	//! Observe user-space bootstrap and scheduling at the trap boundary.
2	+	use kernel::abi;
3	+	use std::lang::il;
4	+	use kernel::trap;
5	+	use kernel::dispatch;
6	+	use kernel::domains;
7	+	use kernel::budgets;
8	+	use kernel::events;
9	+	use kernel::sync;
10	+
11	+	/// Root has requested normal termination.
12	+	static ROOT_EXITED: bool = false;
13	+	/// Workers that returned after root released its authority.
14	+	static WORKERS: u32 = 0;
15	+	/// Write one byte to the test UART.
16	+	fn put(value: u8);
17	+	/// Most recent user operation for fault diagnosis.
18	+	static LAST_CALL: u64 = 0;
19	+
20	+	/// Print a complete hexadecimal diagnostic word.
21	+	fn number(value: u64) {
22	+	let digits = "0123456789abcdef";
23	+	for i in 0..16 {
24	+	put(digits[((value >> ((15 - i) as u64 * 4)) & 15) as u32]);
25	+	}
26	+	put(10);
27	+	}
28	+
29	+	/// Check terminal calls without changing user-domain resources or execution.
30	+	export unsafe fn observe(frame: &trap::Frame) {
31	+	if not trap::fromUser(frame) {
32	+	return;
33	+	}
34	+	let cause = trap::classify(frame.cause);
35	+	if cause == trap::Cause::Timer or cause == trap::Cause::Software or cause == trap::Cause::External {
36	+	return;
37	+	}
38	+	if cause <> trap::Cause::UserCall {
39	+	number(frame.cause); number(frame.pc); number(LAST_CALL); number(WORKERS as u64);
40	+	assert false;
41	+	}
42	+	set LAST_CALL = frame.registers[17];
43	+	assert frame.registers[17] <> abi::Operation::Abort as u64;
44	+	let shutdown = frame.registers[17] == il::DEVICE_ACCESS as u64 and frame.registers[12] == 0x104;
45	+	if frame.registers[17] <> abi::Operation::Exit as u64 and not shutdown {
46	+	return;
47	+	}
48	+	let current = dispatch::current();
49	+	let context = abi::reference(current.context);
50	+	let guard = sync::enter();
51	+	let owner = domains::STORE.contexts[context.index].owner;
52	+	if owner.index == 0 {
53	+	assert frame.registers[10] == 0;
54	+	set ROOT_EXITED = true;
55	+	} else if owner.index == 1 {
56	+	assert ROOT_EXITED and WORKERS == 4 and shutdown and frame.registers[13] == 0x5555;
57	+	let scheduler = try! domains::get(&domains::STORE, owner);
58	+	assert scheduler.parent == nil;
59	+	let ring = &scheduler.memory.ring;
60	+	assert ring.tail - ring.head == 5;
61	+	assert ring.data[ring.head & ring.mask].kind == events::CHILD_EXIT;
62	+	assert ring.data[ring.head & ring.mask].code == 0;
63	+	for i in 1..5 {
64	+	let event = ring.data[(ring.head + i) & ring.mask];
65	+	assert event.kind == events::CHILD_EXIT and event.code == 7;
66	+	}
67	+	let future = try! budgets::query(&budgets::STORE, &scheduler.memory.table, abi::Handle(current.budget));
68	+	assert future.end > dispatch::clock();
69	+	sync::leave(guard);
70	+	return;
71	+	} else {
72	+	assert ROOT_EXITED and frame.registers[10] == 7;
73	+	set WORKERS += 1;
74	+	}
75	+	sync::leave(guard);
76	+	}

test/scheduling/machine.ras added +10 -0

1	+	//! Run production boot and finish after user-space scheduling checks.
2	+	.text;
3	+	call @kernel::boot::initialize;
4	+	call @kernel::boot::launch;
5	+	ebreak;
6	+	.export @kernel::dispatchcheck::put;
7	+	@kernel::dispatchcheck::put
8	+	li %t0 0x10000000;
9	+	sb %a0 0(%t0);
10	+	ret;

test/scheduling/run added +25 -0

1	+	#!/bin/sh
2	+	# Execute root handoff and successive user-space yields with the production packages.
3	+	set -eu
4	+	emulator=${RAD_EMULATOR:-emulator}
5	+	harts=${1:-1 2 8}
6	+	work=$(mktemp -d)
7	+	trap 'rm -rf "$work"' EXIT HUP INT TERM
8	+	cp kernel/kernel.rad "$work/kernel.rad"
9	+	printf '\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
10	+	mkdir "$work/kernel"
11	+	cp kernel/kernel/*.rad "$work/kernel/"
12	+	cp test/scheduling/kernel/dispatchcheck.rad "$work/kernel/"
13	+	sed -i '/let cause = trap::classify(frame.cause);/a\ super::dispatchcheck::observe(frame);' "$work/kernel/boot.rad"
14	+	sh test/acceptance/compile "$emulator" "$work"
15	+	cat test/scheduling/machine.ras kernel/kernel/*.ras > "$work/scheduling.ras"
16	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
17	+	-- "$work/std.ril" "$work/kernel.ril" "$work/scheduling.ras" "$work/scheduling.rv64" bin/root.ril bin/scheduler.ril
18	+	for count in $harts; do
19	+	if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
20	+	sh test/acceptance/replay scheduling "$emulator" "$work/scheduling.rv64" "$count"
21	+	else
22	+	"$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 -run "$work/scheduling.rv64"
23	+	fi
24	+	printf 'scheduling: %s-hart root handoff, four user-space yields, and checked shutdown passed\n' "$count"
25	+	done

test/shared/run +1 -1

# Compile two packages once and execute them with two private state graphs.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
    -run bin/radiance.rv64.dev \
    -pkg support -mod test/shared/support.rad \
    -pkg app -mod test/shared/app.rad -entry app -ril "$work"
"$emulator" -run bin/shared.build.rv64 -- "$work/support.ril" "$work/app.ril" "$work/native.rv64"
"$emulator" -machine -run "$work/native.rv64"

test/slots/run +1 -1

for name in duplicate abandon; do
    case "$name" in
        duplicate) diagnostic='linear value used after consumption' ;;
        abandon) diagnostic='linear value is not consumed' ;;
    esac
    if "$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    if "$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
        -run bin/radiance.rv64.dev "$@" -test \
        -pkg "$name" -mod "test/slots/$name.rad" -entry "$name" \
        -o "$work/test.rv64" > "$work/log" 2>&1; then
        printf 'reservation misuse compiled: %s\n' "$name" >&2
        exit 1

test/termination/kernel/dispatchcheck.rad +29 -11

use kernel::dispatch;
use kernel::boot;
use kernel::events;
use kernel::sync;
use kernel::dispatchinput;
use kernel::devices;
use kernel::platform;

/// Bootstrap creation, allocation, and budget authority.
unsafe static TABLE: capability::Table = undefined;
/// Child domain identities retained by terminal events.
static CHILDREN: [abi::Ref; 5] = undefined;
unsafe static CHILDREN: [abi::Ref; 6] = undefined;
/// Surviving event receiver and native completion checker.
unsafe static PARENT: abi::Ref = undefined;
/// Map a validated physical page.
fn memory(address: u64) -> *mut u8;
/// Current kernel package-state base.

/// Native entry for the completion checker.
fn completion() -> u64;
/// Signal successful completion to the machine fixture.
fn finish();

/// Give four children disjoint windows, followed by the surviving parent.
/// Give each child a disjoint window, followed by the surviving parent.
export unsafe fn setup() {
    let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
    let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
    capability::initialize(&mut TABLE, owner);
    try! backing::registerDomain(&mut pages::STORE.backings, owner);

    set domains::STORE.records[PARENT.index].state = domains::Lifecycle::Active;
    set domains::STORE.contexts[parent.initial.index].frame.pc = completion();
    set domains::STORE.contexts[parent.initial.index].frame.status = 0x1880;
    set domains::STORE.contexts[parent.initial.index].frame.registers[2] = domains::STORE.contexts[parent.initial.index].kernelStack.end;
    set domains::STORE.contexts[parent.initial.index].frame.registers[3] = kernelGp();
    let mut handles: [abi::Handle; 5] = undefined;
    let mut contexts: [abi::Ref; 5] = undefined;
    for i in 0..5 {
    let mut handles: [abi::Handle; 6] = undefined;
    let mut contexts: [abi::Ref; 6] = undefined;
    for i in 0..6 {
        let handle = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
        let object = (try! capability::get(&TABLE, handle)).object;
        let mut child = try! domains::get(&domains::STORE, object);
        set CHILDREN[i] = object;
        set handles[i] = handle;
        set contexts[i] = child.initial;
        try! domains::reparent(&mut domains::STORE, &TABLE, handle, parentHandle);
        let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
        let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut child.memory.table, stack, (abi::READ | abi::WRITE) as u64);
        let storage = try! pages::get(&pages::STORE, (try! capability::get(&TABLE, stack)).object);
        try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, i as u64);
        let mut size = i as u64;
        if i == 5 {
            let mut installed = false;
            for index in 0..devices::STORE.count {
                if devices::STORE.regions[index].kind <> platform::Kind::Finish {
                    continue;
                }
                let device = try! devices::install(&devices::STORE, &mut child.memory.table, index);
                let args = memory(storage.base) as *mut u64;
                set *args = *device;
                set installed = true;
                break;
            }
            assert installed;
            set size = 8;
        }
        try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, handle, storage.base + 4096, storage.base, size);
    }
    let timer = try! dispatch::timer(&boot::PLATFORM, 0);
    let clock = dispatch::now(timer);
    let start = clock + 1000000;
    let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 12000000);
    for i in 0..5 {
    let mut remaining = try! budgets::seed(&mut budgets::STORE, &mut TABLE, 0, start, start + 14000000);
    for i in 0..6 {
        let next = try! budgets::split(&mut budgets::STORE, &mut TABLE, remaining, start + (i as u64 + 1) * 2000000, clock);
        let bound = try! budgets::bind(&mut budgets::STORE, &domains::STORE, &mut TABLE,
            budgets::Binding { budget: remaining, domain: handles[i], context: contexts[i] }, clock);
        set remaining = next;
    }


/// Check each terminal kind, status, and generation while the parent continues to run.
export unsafe fn verify() {
    let guard = sync::enter();
    let mut parent = try! domains::get(&domains::STORE, PARENT);
    let codes = [37 as u32, abi::FAULT_ABORT, 3, 0, abi::FAULT_ABORT];
    for i in 0..5 {
    let codes = [37 as u32, abi::FAULT_ABORT, 3, 0, abi::FAULT_ABORT, 51];
    for i in 0..6 {
        let event = try! events::pop(&mut parent.memory.ring) else panic "missing termination event";
        assert event.value == abi::id(CHILDREN[i]) and event.code == codes[i];
        if i == 0 or i == 3 {
        if i == 0 or i == 3 or i == 5 {
            assert event.kind == events::CHILD_EXIT;
        }
        else {
            assert event.kind == events::FAULT;
        }

test/termination/spin.rad +6 -1

@intrinsic fn ecall(operation: u32, a0: i64, a1: i64, a2: i64, a3: i64) -> i64;

/// Startup values used to select the termination operation.
record Env: Copy {
    /// Optional argument bytes supplied by the parent.
    argsPointer: *u8,
    argsPointer: *abi::Handle,
    /// Termination case selected by the parent.
    argsSize: u64,
    /// Installed event handle.
    eventsHandle: u64,
    /// Shared event-ring address.

    }
    if env.argsSize == 4 {
        let result = ecall(47, 0, 0, 0, 0);
        sys::exit(99);
    }
    if env.argsSize == 8 {
        let device = sys::queryDevice(*env.argsPointer);
        assert device.base == 0x10001000 and device.size >= 4;
        sys::exit(51);
    }
    assert false;
}

249	254		case 22, 43, 49 => {
250	255		let stopped = try termination(owner, operation, arguments);
251	256		return 0;
252	257		},
253	258		case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
	259	+	case il::DEVICE_ACCESS => {
	260	+	let flags = arguments[2];
	261	+	if (flags & ~0x1ff) <> 0 {
	262	+	throw abi::Error::InvalidArg;
	263	+	}
	264	+	return try devices::address(&devices::STORE, &domain.memory.table, handle,
	265	+	arguments[1], flags & 0xff, (flags & 0x100) <> 0);
	266	+	},
254	267		case 12 => {
255	268		try drop(&mut domain, handle);
256	269		return 0;
257	270		},
258	271		case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,

298	311		let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
299	312		let page = try pages::get(&pages::STORE, entry.object);
300	313		try reply(&domain, arguments, &[page.base, page.count as u64]);
301	314		return 0;
302	315		},
	316	+	case 45 => {
	317	+	let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Device, abi::Rights(0));
	318	+	let device = try devices::get(&devices::STORE, entry.object);
	319	+	try reply(&domain, arguments, &[device.info.base, device.info.size]);
	320	+	return 0;
	321	+	},
303	322		case 46 => {
304	323		let number = try interrupts::query(&interrupts::STORE, &domain.memory.table, handle);
305	324		if arguments[2] <> @sizeOf(abi::InterruptInfo) as u64 or arguments[3] <> 0
306	325		or not buffer(&domain, arguments[1], 4, abi::Rights(abi::WRITE), 4) {
307	326		throw abi::Error::InvalidArg;

665	684		assert trap::fromUser(frame) and frame.cause == 8;
666	685		assert trap::advanceCall(frame);
667	686		let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
668	687		let result = try execute(owner, frame.registers[17], &args[..], now) catch error {
669	688		let operation = frame.registers[17];
670		-	if operation >= 44 and operation <= 48 and error == abi::Error::BadHandle {
	689	+	if operation == il::DEVICE_ACCESS or (operation >= 44 and operation <= 48 and error == abi::Error::BadHandle) {
671	690		try execute(owner, 43, &[0, 0, 0, 0], now) catch stopped {
672	691		assert stopped == abi::Error::BadHandle;
673	692		};
674	693		}
675	694		set frame.registers[10] = (0 as u64) - error as u64;

725	725		match reservation { case Reservation::Held(creation) => try prepareCreation(packages, creation), }
726	726		}
727	727
728	728		/// Initialize retained domain storage outside the metadata lock.
729	729		export unsafe fn prepareCreation(packages: &registry::Store, creation: &Creation) throws (abi::Error) {
730		-	let mut storage = creation.storage;
	730	+	try prepareStorage(packages, creation.storage, creation.domain);
	731	+	}
	732	+
	733	+	/// Initialize one domain's private allocations before capability publication.
	734	+	unsafe fn prepareStorage(packages: &registry::Store, storage: Prepared, owner: abi::Ref) throws (abi::Error) {
	735	+	let mut storage = storage;
731	736		zero(storage.memory as u64, storage.allocation.count);
732	737		zero(storage.kernelStack.start, storage.kernelFrames.count);
733	738		try instances::fill(packages, &storage.graph.graph, storage.graph.base);
734		-	capability::initialize(&mut storage.memory.table, creation.domain);
	739	+	capability::initialize(&mut storage.memory.table, owner);
735	740		}
736	741
737	742		/// Check the unpublished capacity retained by one creation transaction.
738	743		fn require(store: &Store, table: &capability::Table, creation: &Creation) {
739	744		assert table.owner == creation.owner;

752	757		release: fn(&mut frames::Pool, frames::Run) throws (abi::Error))
753	758		{
754	759		match reservation {
755	760		case Reservation::Held(creation) => {
756	761		require(store, table, &creation);
	762	+	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
757	763		discard(&mut backings.pool, creation.storage, release);
758	764		try! slots::cancel(&mut store.contextSlots[..], slots::Reservation::Held(creation.context));
759	765		try! slots::cancel(&mut store.slots[..], slots::Reservation::Held(creation.domain));
760		-	try! slots::cancel(&mut table.slots[..], slots::Reservation::Held(creation.handle));
761	766		},
762	767		}
763	768		}
764	769
765	770		/// Publish an initialized domain and its Events under metadata serialization.

12	12		export mod trap;
13	13		export mod capability;
14	14		export mod transactions;
15	15		export mod events;
16	16		export mod interrupts;
	17	+	export mod devices;
17	18		export mod plic;
18	19		export mod registry;
19	20		export mod frames;
20	21		export mod backing;
21	22		export mod pages;

28	29		export mod timers;
29	30		export mod remote;
30	31		export mod calls;
31	32		export mod sys;
32	33		export mod boot;
	34	+	export mod startup;
33	35		@test export mod tests;

15	15		eventsHandle: u64,
16	16		/// Address of the domain's shared event ring.
17	17		eventsPointer: u64,
18	18		}
19	19
	20	+	/// Initial root resources copied into root-owned startup memory.
	21	+	export record BootInfo: Copy {
	22	+	/// Number of Device handles after the self and Events slots.
	23	+	devices: u64,
	24	+	/// Number of Interrupt handles following the Device handles.
	25	+	interrupts: u64,
	26	+	/// Bit mask of online harts, whose Budget handles follow in hart order.
	27	+	harts: u64,
	28	+	/// Machine timer ticks per second.
	29	+	timebase: u64,
	30	+	/// Image used to construct root and additional instances of its program.
	31	+	rootImage: Handle,
	32	+	/// Initial user-space scheduler image.
	33	+	schedulerImage: Handle,
	34	+	/// Read-only machine-time device, or zero when absent.
	35	+	clock: Handle,
	36	+	/// Platform shutdown device, or zero when absent.
	37	+	shutdown: Handle,
	38	+	/// Page that owns this header and the copied handle array.
	39	+	arguments: Handle,
	40	+	/// Copied capability values in their initial slot order.
	41	+	handles: *unsafe mut [u64],
	42	+	}
	43	+
20	44		/// Physical extent returned by QueryPage.
21	45		export record PageInfo: Copy {
22	46		/// Address of the first physical frame.
23	47		base: u64,
24	48		/// Number of contiguous physical frames.
25	49		count: u64,
26	50		}
27	51
	52	+	/// Physical MMIO extent returned by QueryDevice.
	53	+	export record DeviceInfo: Copy {
	54	+	/// Physical base address of the register region.
	55	+	base: u64,
	56	+	/// Size of the register region in bytes.
	57	+	size: u64,
	58	+	}
	59	+
28	60		/// Physical source returned by QueryInterrupt.
29	61		export record InterruptInfo: Copy {
30	62		/// PLIC source number; zero does not name an interrupt.
31	63		number: u32,
32	64		}

1	1		//! Machine entry boundary and published platform state.
2	2
3	3		use super::platform;
	4	+	use super::startup;
4	5		use super::fdt;
5	6		use super::range;
6	7		use super::limits;
7	8		use super::sync;
8	9		use super::trap;

13	14		use super::budgets;
14	15		use super::dispatch;
15	16		use super::timers;
16	17		use super::remote;
17	18		use super::interrupts;
	19	+	use super::devices;
18	20		use super::plic;
19	21		use std::arch::rv64::shared::catalog;
20	22
21	23		/// Platform data published by hart zero before secondary initialization.
22	24		export unsafe static PLATFORM: platform::Platform = undefined;
23	25		/// Release/acquire publication flag for PLATFORM.
24	26		static READY: u64 = 0;
25	27		/// Number of harts that validated their machine stack.
26	28		static ARRIVED: u64 = 0;
	29	+	/// Root resource publication before user dispatch begins.
	30	+	static STARTED: u64 = 0;
27	31		/// Trap anchors owned by their physical harts.
28	32		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
29	33
30	34		/// Dispatch interrupts, user calls, and user faults; halt on kernel faults.
31	35		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {

93	97		budgets::initialize(&mut budgets::STORE);
94	98		timers::initialize(&mut timers::STORE);
95	99		remote::initialize(&mut remote::STORE, PLATFORM.harts);
96	100		interrupts::initialize(&mut interrupts::STORE, PLATFORM.irqSources);
97	101		try! plic::initialize(&PLATFORM);
98		-	assert entryCount == 2;
	102	+	try! devices::initialize(&mut devices::STORE, &PLATFORM);
	103	+	assert entryCount >= 2 and entryCount <= limits::PACKAGES;
99	104		registry::initialize(&mut registry::STORE);
100	105		let bootCatalog = @sliceOf(entries, entryCount);
101	106		try! registry::boot(&mut registry::STORE, &bootCatalog[..]);
102	107		let library = registry::find(&registry::STORE, &"std"[..]) else panic "boot std";
103	108		let kernel = registry::find(&registry::STORE, &"kernel"[..]) else panic "boot kernel";

128	133		}
129	134		}
130	135		return sync::fetchAdd(&mut ARRIVED, 1) + 1 == count;
131	136		}
132	137
	138	+	/// Publish configured root resources once, then enter dispatch on every hart.
	139	+	export unsafe fn launch(last: bool) -> ! {
	140	+	if last {
	141	+	if let root = registry::find(&registry::STORE, &"root"[..]) {
	142	+	let scheduler = registry::find(&registry::STORE, &"scheduler"[..]) else panic "missing scheduler image";
	143	+	let domain = try! startup::create(&PLATFORM, root, scheduler);
	144	+	print("kernel: root ready\n");
	145	+	}
	146	+	sync::storeRelease(&mut STARTED, 1);
	147	+	} else {
	148	+	while sync::loadAcquire(&STARTED) == 0 {
	149	+	}
	150	+	}
	151	+	run();
	152	+	}
	153	+
133	154		/// Enter dispatch after every online hart publishes its initialized state.
134	155		export unsafe fn run() -> ! {
135	156		let mut count: u64 = 0;
136	157		for id in 0..limits::HARTS {
137	158		if (PLATFORM.harts & (1 << id)) <> 0 {

5	5		.export @kernel::boot::read8;
6	6		.export @kernel::boot::write8;
7	7
8	8		@kernel::boot::start
9	9		call @kernel::boot::initialize;
10		-	call @kernel::boot::run;
	10	+	call @kernel::boot::launch;
11	11		ebreak;
12	12
13	13		// Initialize machine CSRs and preserve the firmware stack across Radiance entry.
14	14		@kernel::boot::initialize
15	15		csrw mie %zero;

34	34		@kernel::boot::write8
35	35		fence iorw iorw;
36	36		sb %a1 0(%a0);
37	37		fence iorw iorw;
38	38		ret;
	39	+
	40	+	.export @kernel::startup::info;
	41	+	.export @kernel::startup::words;
	42	+	// Resolve root-owned startup memory after physical allocation.
	43	+	@kernel::startup::info
	44	+	ret;
	45	+	@kernel::startup::words
	46	+	ret;

1	1		//! Direct-call execution and checked copies into domain-owned memory.
2	2
3	3		use super::abi;
	4	+	use std::lang::il;
4	5		use super::slots;
5	6		use super::limits;
6	7		use super::capability;
7	8		use super::pages;
8	9		use super::domains;

61	63		let object = try domains::get(&domains::STORE, entry.object);
62	64		if object.state == domains::Lifecycle::Dead {
63	65		throw abi::Error::BadHandle;
64	66		}
65	67		},
	68	+	case abi::Kind::Device => {
	69	+	let object = try devices::get(&devices::STORE, entry.object);
	70	+	},
66	71		case abi::Kind::Image => {
67	72		let object = try registry::get(&registry::STORE, entry.object);
68	73		},
69	74		else => throw abi::Error::BadHandle,
70	75		}

767	772		-> abi::Handle throws (abi::Error)
768	773		{
769	774		match reservation {
770	775		case Reservation::Held(creation) => {
771	776		require(store, table, &creation);
772		-	if not backing::domainLive(backings, creation.owner) {
	777	+	let root = creation.owner == creation.domain;
	778	+	if not root and not backing::domainLive(backings, creation.owner) {
773	779		cancel(store, backings, table, Reservation::Held(creation));
774	780		throw abi::Error::BadHandle;
775	781		}
776	782		let object = creation.domain;
777	783		let initial = creation.context;

795	801		owner: object, state: ContextState::Ready, hart: nil, pending: nil, reservation: false, kernelFrames: prepared.kernelFrames, kernelStack: prepared.kernelStack,
796	802		userStack: range::Range { start: 0, end: 0 }, frame,
797	803		};
798	804		set store.reclaimed[object.index] = false;
799	805		set store.reaping[object.index] = false;
	806	+	let mut parent: ?abi::Ref = nil;
	807	+	if not root {
	808	+	set parent = creation.owner;
	809	+	}
800	810		set store.records[object.index] = Domain {
801		-	state: Lifecycle::Pending, creator: creation.owner, parent: creation.owner, image: creation.image, initial,
	811	+	state: Lifecycle::Pending, creator: creation.owner, parent, image: creation.image, initial,
802	812		allocation: prepared.allocation, memory: prepared.memory, graph, events: queue,
803	813		};
804		-	set store.ancestors[object.index] = store.ancestors[creation.owner.index];
805		-	set store.ancestors[object.index][creation.owner.index / 32] \|= 1 << (creation.owner.index % 32);
	814	+	if not root {
	815	+	set store.ancestors[object.index] = store.ancestors[creation.owner.index];
	816	+	set store.ancestors[object.index][creation.owner.index / 32] \|= 1 << (creation.owner.index % 32);
	817	+	}
806	818		let context = try! slots::commit(&mut store.contextSlots[..], slots::Reservation::Held(initial));
807	819		let domain = try! slots::commit(&mut store.slots[..], slots::Reservation::Held(object));
808	820		return capability::publish(table, slots::Reservation::Held(creation.handle), capability::Entry {
809	821		kind: abi::Kind::Domain, object: domain, rights: creation.rights,
810	822		});
811	823		},
812	824		}
813	825		}
814	826
	827	+	/// Create the first domain with its self capability in slot zero and Events in slot one.
	828	+	/// Boot calls this before any domain, context, or exposure membership is published.
	829	+	export unsafe fn bootstrap(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store, image: abi::Ref)
	830	+	-> abi::Ref throws (abi::Error)
	831	+	{
	832	+	for slot in store.slots {
	833	+	if slot.state <> slots::State::Free {
	834	+	throw abi::Error::Busy;
	835	+	}
	836	+	}
	837	+	let package = try registry::get(packages, image);
	838	+	let entry = package.entry else {
	839	+	throw abi::Error::VerifyFailed;
	840	+	};
	841	+	let domainSlot = try slots::reserve(&mut store.slots[..]);
	842	+	let contextSlot = try slots::reserve(&mut store.contextSlots[..]) catch error {
	843	+	try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
	844	+	};
	845	+	let object = slots::reference(&domainSlot);
	846	+	let mut storage = try allocate(&mut backings.pool, packages, image) catch error {
	847	+	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
	848	+	try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
	849	+	};
	850	+	try prepareStorage(packages, storage, object) catch error {
	851	+	discard(&mut backings.pool, storage, frames::release);
	852	+	try! slots::cancel(&mut store.contextSlots[..], contextSlot);
	853	+	try! slots::cancel(&mut store.slots[..], domainSlot); throw error;
	854	+	};
	855	+	let handleSlot = try! slots::reserve(&mut storage.memory.table.slots[..]);
	856	+	assert slots::reference(&handleSlot).index == 0;
	857	+	match domainSlot {
	858	+	case slots::Reservation::Held(domain) => match contextSlot {
	859	+	case slots::Reservation::Held(context) => match handleSlot {
	860	+	case slots::Reservation::Held(handle) => {
	861	+	let result = try publish(store, backings, &mut storage.memory.table, Reservation::Held(Creation {
	862	+	owner: object, handle, domain, context,
	863	+	rights: abi::Rights(MANAGEMENT_RIGHTS \| abi::CREATE \| abi::ALLOCATE), image, entry, storage,
	864	+	}));
	865	+	assert (try! abi::decode(result)).object.index == 0;
	866	+	return object;
	867	+	},
	868	+	},
	869	+	},
	870	+	}
	871	+	}
	872	+
815	873		/// Create a private domain while the caller has exclusive access to metadata.
816	874		export unsafe fn create(store: &mut Store, backings: &mut backing::Store, packages: &registry::Store,
817	875		table: &mut capability::Table, authority: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error)
818	876		{
819	877		let pending = try reserve(store, backings, packages, table, authority, image);

146	146		try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
147	147		return info;
148	148		}
149	149		}
150	150
	151	+	/// Read Device metadata. An invalid handle is a program error.
	152	+	export fn queryDevice(handle: abi::Handle) -> abi::DeviceInfo {
	153	+	unsafe {
	154	+	let mut info: abi::DeviceInfo = undefined;
	155	+	try! result(ecall(abi::Operation::QueryDevice as u32, *handle as i64,
	156	+	(&mut info) as i64, @sizeOf(abi::DeviceInfo) as i64, 0));
	157	+	return info;
	158	+	}
	159	+	}
	160	+
151	161		/// Read a physical interrupt source. An invalid handle is a program error.
152	162		export fn queryInterrupt(handle: abi::Handle) -> abi::InterruptInfo {
153	163		unsafe {
154	164		let mut info: abi::InterruptInfo = undefined;
155	165		try! result(ecall(abi::Operation::QueryInterrupt as u32, *handle as i64,

14	14		use kernel::trap;
15	15		use kernel::budgets;
16	16		use kernel::remote;
17	17		use kernel::interrupts;
18	18		use kernel::events;
	19	+	use kernel::devices;
	20	+	use kernel::platform;
19	21
20	22		/// Event storage for administrative destruction targets.
21	23		unsafe static TARGETS: [domains::Memory; 2] = undefined;
22	24
23	25		/// Retained dead-domain records cannot be queried or delegated through handles.

381	383		try testing::expect(budgets::STORE.slots[0].state == slots::State::Free);
382	384		let installed = try! capability::get(&MEMORY.table, queue);
383	385		try testing::expect(installed.kind == abi::Kind::Events);
384	386		}
385	387
	388	+	/// Device queries copy only metadata into checked caller-owned memory.
	389	+	@test unsafe fn deviceRecord() throws (testing::TestError) {
	390	+	let owner = initialize();
	391	+	let storage = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
	392	+	set devices::STORE.count = 1;
	393	+	set devices::STORE.regions[0] = devices::Region {
	394	+	kind: platform::Kind::Uart, info: { base: 0x10000000, size: 256 }, rights: abi::Rights(abi::READ),
	395	+	};
	396	+	let device = try! devices::install(&devices::STORE, &mut MEMORY.table, 0);
	397	+	let base = (&RAM[OFFSET]) as u64;
	398	+	set RAM[OFFSET + 2] = 123;
	399	+	assert try! calls::invoke(owner, 45, &[*device, base, 16, 0], 0) == 0;
	400	+	assert RAM[OFFSET] == 0x10000000 and RAM[OFFSET + 1] == 256 and RAM[OFFSET + 2] == 123;
	401	+	let mut rejected: u32 = 0;
	402	+	for args in &[[device, base + 1, 16, 0], [device, base, 8, 0], [*device, base, 16, 1]] {
	403	+	try calls::invoke(owner, 45, &args[..], 0) catch error {
	404	+	assert error == abi::Error::InvalidArg; set rejected += 1;
	405	+	};
	406	+	}
	407	+	assert rejected == 3 and RAM[OFFSET + 2] == 123;
	408	+	}
	409	+
386	410		/// A query copies exactly its record into writable memory owned by the caller.
387	411		@test unsafe fn records() throws (testing::TestError) {
388	412		let owner = initialize();
389	413		let handle = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
390	414		let base = (&RAM[OFFSET]) as u64;

431	455		try testing::expect(rejected == 6 and domains::STORE.contextSlots[0].state == slots::State::Free);
432	456		}
433	457
434	458		/// Invalid infallible queries terminate the caller at the syscall boundary.
435	459		@test unsafe fn invalidQueryAborts() throws (testing::TestError) {
436		-	for operation in [44 as u64, 46, 47, 48] {
	460	+	for operation in [44 as u64, 45, 46, 47, 48] {
437	461		let owner = initialize();
438	462		try! events::open(&mut domains::STORE.events, owner, &mut MEMORY.ring);
439	463		let mut frame = trap::Frame { registers: [0; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
440	464		set frame.registers[17] = operation;
441	465		calls::handle(owner, &mut frame, 0, calls::invoke);

71	71		template: &[7], memory: 8, alignment: 8, relocations: &[],
72	72		});
73	73		set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
74	74		}
75	75
	76	+	/// Failed root construction restores capacity and permits a fresh attempt.
	77	+	@test unsafe fn bootstrapRollback() throws (testing::TestError) {
	78	+	initialize(abi::CREATE);
	79	+	let image = (try! capability::get(&TABLE, IMAGE)).object;
	80	+	domains::initialize(&mut DOMAINS);
	81	+	set BACKINGS.domains[0] = 0;
	82	+	let capacity = BACKINGS.pool.count;
	83	+	for count in [0 as u32, 1, 8, 32] {
	84	+	set BACKINGS.pool.count = count;
	85	+	let mut failed = false;
	86	+	try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
	87	+	assert error == abi::Error::OutOfMemory; set failed = true;
	88	+	};
	89	+	assert failed and DOMAINS.slots[0].state == slots::State::Free;
	90	+	assert DOMAINS.contextSlots[0].state == slots::State::Free;
	91	+	assert BACKINGS.domains[0] == 0 and DOMAINS.events.queues[0].generation == 0;
	92	+	for i in 0..count {
	93	+	assert frames::available(&BACKINGS.pool, i);
	94	+	}
	95	+	}
	96	+	set BACKINGS.pool.count = capacity;
	97	+	let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
	98	+	assert root.index == 0 and root.generation > 1;
	99	+	assert (try! domains::get(&DOMAINS, root)).parent == nil;
	100	+	}
	101	+
	102	+	/// Root construction owns slot zero and installs the fixed bootstrap handles.
	103	+	@test unsafe fn bootstrap() throws (testing::TestError) {
	104	+	initialize(abi::CREATE);
	105	+	let image = (try! capability::get(&TABLE, IMAGE)).object;
	106	+	domains::initialize(&mut DOMAINS);
	107	+	set BACKINGS.domains[0] = 0;
	108	+	let root = try! domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image);
	109	+	assert root.index == 0 and root.generation == 1;
	110	+	let domain = try! domains::get(&DOMAINS, root);
	111	+	assert domain.creator == root and domain.parent == nil and domain.state == domains::Lifecycle::Pending;
	112	+	assert domain.memory.table.slots[0].state == slots::State::Live;
	113	+	let self = domain.memory.table.entries[0];
	114	+	assert self.kind == abi::Kind::Domain and self.object == root;
	115	+	assert *self.rights == (domains::MANAGEMENT_RIGHTS \| abi::CREATE \| abi::ALLOCATE);
	116	+	let queue = try! abi::decode(domain.events);
	117	+	assert queue.object.index == 1 and queue.kind == abi::Kind::Events;
	118	+	assert domain.memory.table.entries[1].object == root;
	119	+	assert DOMAINS.contexts[domain.initial.index].frame.pc == 0x80000000;
	120	+	assert DOMAINS.contexts[domain.initial.index].frame.registers[3] == domain.graph.table.ptr as u64;
	121	+	for word in DOMAINS.ancestors[0] {
	122	+	assert word == 0;
	123	+	}
	124	+	let mut busy = false;
	125	+	try domains::bootstrap(&mut DOMAINS, &mut BACKINGS, &PACKAGES, image) catch error {
	126	+	assert error == abi::Error::Busy; set busy = true;
	127	+	};
	128	+	assert busy and DOMAINS.slots[1].state == slots::State::Free;
	129	+	}
	130	+
76	131		/// Interrupted creation returns all private storage after its caller releases its hart.
77	132		@test unsafe fn cancelledCreation() throws (testing::TestError) {
78	133		initialize(abi::CREATE);
79	134		lifecycle::initialize(&mut CALLS);
80	135		let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);

17	17		use std::arch::rv64::shared;
18	18		use std::arch::rv64::shared::catalog;
19	19
20	20		/// Native entry page, above the platform firmware data.
21	21		constant CODE_ADDRESS: u64 = 0x81000000;
	22	+	/// Maximum shared packages in the boot image.
	23	+	constant PACKAGES: u32 = 4;
22	24		/// Package code arenas retained until image output completes.
23		-	unsafe static CODE: [[u8; 16777216]; 2] = undefined;
	25	+	unsafe static CODE: [[u8; 16777216]; PACKAGES] = undefined;
24	26		/// Reusable function workspace.
25	27		static SCRATCH: [u8; 16777216] = [0; 16777216];
26	28		/// Persistent decoded RIL storage.
27	29		static DECODE: [u8; 67108864] = [0; 67108864];
28	30		/// Exact package bytes retained in the native catalog.
29		-	unsafe static INPUT: [[u8; 8388608]; 2] = undefined;
	31	+	unsafe static INPUT: [[u8; 8388608]; PACKAGES] = undefined;
30	32		/// Combined startup and boundary assembly source.
31	33		static SOURCE: [u8; 65536] = [0; 65536];
32	34		/// Assembler workspace.
33	35		static ASSEMBLY: [u8; 4194304] = [0; 4194304];
34	36		/// Assembled startup words.
35	37		static TEXT: [u32; 16384] = [0; 16384];
36	38		/// Assembly identifiers retained through package linking.
37	39		unsafe static STRINGS: strings::Pool = strings::Pool { table: undefined, count: 0 };
38	40		/// Per-package data layout workspaces.
39		-	unsafe static DATA: [[data::DataSym; 4096]; 2] = undefined;
	41	+	unsafe static DATA: [[data::DataSym; 4096]; PACKAGES] = undefined;
40	42		/// Local function and data definitions.
41		-	unsafe static SYMBOLS: [[shared::Symbol; 4096]; 2] = undefined;
	43	+	unsafe static SYMBOLS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
42	44		/// Public package symbols.
43		-	unsafe static EXPORTS: [[shared::Symbol; 4096]; 2] = undefined;
	45	+	unsafe static EXPORTS: [[shared::Symbol; 4096]; PACKAGES] = undefined;
	46	+	/// Exports from dependencies compiled before the current package.
	47	+	unsafe static IMPORTS: [shared::Symbol; 16384] = undefined;
44	48		/// Private-state relocation records.
45		-	unsafe static RELOCS: [[shared::Relocation; 4096]; 2] = undefined;
	49	+	unsafe static RELOCS: [[shared::Relocation; 4096]; PACKAGES] = undefined;
46	50		/// Initialized package templates.
47		-	unsafe static TEMPLATES: [[u8; 1048576]; 2] = undefined;
	51	+	unsafe static TEMPLATES: [[u8; 1048576]; PACKAGES] = undefined;
48	52		/// Read-only catalog and retained package payloads.
49	53		static RO: [u8; 16777216] = [0; 16777216];
50	54		/// Kernel state table and initialized private package graph.
51	55		static RW: [u8; 33554432] = [0; 33554432];
52	56		/// Contiguous native code segment, including entry trampoline and alignment padding.

83	87		template: &mut TEMPLATES[slot][..], relocations: &mut RELOCS[slot][..],
84	88		}, &mut arena, &mut scratch);
85	89		}
86	90
87	91		/// Emit a trampoline that preserves firmware arguments and supplies the native catalog.
88		-	unsafe fn trampoline(kernel: u64, ro: u64, rw: u64) {
	92	+	unsafe fn trampoline(kernel: u64, ro: u64, rw: u64, count: u32) {
89	93		let mut arena = alloc::new(&mut SCRATCH[..]);
90	94		let mut e = try! emit::emitter(&mut arena, false);
91	95		emit::loadImm(&mut e, rv64::GP, rw as i64);
92	96		emit::loadImm(&mut e, rv64::A3, ro as i64);
93		-	emit::loadImm(&mut e, rv64::A4, 2);
	97	+	emit::loadImm(&mut e, rv64::A4, count as i64);
94	98		emit::loadImm(&mut e, rv64::T0, kernel as i64);
95	99		emit::emit(&mut e, encode::jalr(rv64::ZERO, rv64::T0, 0));
96	100		assert e.codeLen <= 1024;
97	101		for word, i in emit::getCode(&e) {
98	102		set NATIVE[i] = word;
99	103		}
100	104		}
101	105
102	106		/// Link shared boot packages, retain their catalog, and instantiate the kernel graph.
103	107		@default unsafe fn main(env: *sys::Env) -> i32 {
104		-	assert env.args.len == 5;
	108	+	assert env.args.len >= 5 and env.args.len <= PACKAGES + 3;
	109	+	let count = env.args.len - 3;
105	110		let mut decoder = alloc::new(&mut DECODE[..]);
106	111		let libraryBytesLength = unix::readFile(env.args[1], &mut INPUT[0][..]) else panic "std RIL";
107	112		let libraryBytes = &INPUT[0][..libraryBytesLength];
108	113		let libraryInput = try! program::decode(libraryBytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
109	114		let kernelBytesLength = unix::readFile(env.args[2], &mut INPUT[1][..]) else panic "kernel RIL";

113	118		assert libraryInput.dependencies.len == 0;
114	119		for dependency in kernelInput.dependencies {
115	120		assert mem::eq(dependency, "std");
116	121		}
117	122		let sourceLength = unix::readFile(env.args[3], &mut SOURCE[..]) else panic "kernel assembly";
	123	+	let source = &SOURCE[..sourceLength];
118	124		let mut assemblyArena = alloc::new(&mut ASSEMBLY[..]);
119	125		let empty: *mut [u8] = &mut [];
120		-	let assembly = try! asm::assemble(asm::scanner::SourceKind::String, &SOURCE[..sourceLength],
	126	+	let assembly = try! asm::assemble(asm::scanner::SourceKind::String, source,
121	127		&mut TEXT[..], &mut empty[..], &mut assemblyArena, &mut STRINGS, 0);
122	128		let library = try compile(&libraryInput, 0, CODE_ADDRESS + 4096, &[],
123	129		asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error {
124	130		report(error);
125	131		return 1;

127	133		let kernelAddress = aligned(library.codeAddress + library.code.len as u64 * 4, 4096);
128	134		let kernel = try compile(&kernelInput, 1, kernelAddress, library.exports, assembly) catch error {
129	135		report(error);
130	136		return 1;
131	137		};
132		-	let entries = [
133		-	catalog::Entry { source: libraryBytes, package: library },
134		-	catalog::Entry { source: kernelBytes, package: kernel },
135		-	];
136		-	let codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
	138	+	let mut catalogEntries: [catalog::Entry; PACKAGES] = undefined;
	139	+	set catalogEntries[0] = catalog::Entry { source: libraryBytes, package: library };
	140	+	set catalogEntries[1] = catalog::Entry { source: kernelBytes, package: kernel };
	141	+	let mut codeEnd = kernel.codeAddress + kernel.code.len as u64 * 4;
	142	+	for slot in 2..count {
	143	+	let bytesLength = unix::readFile(env.args[slot + 3], &mut INPUT[slot][..]) else panic "boot package RIL";
	144	+	let bytes = &INPUT[slot][..bytesLength];
	145	+	let input = try! program::decode(bytes, &mut decoder, binary::Limits { registers: 8192, blocks: 4096 });
	146	+	let mut imported: u32 = 0;
	147	+	for previous in 0..slot {
	148	+	for symbol in catalogEntries[previous].package.exports {
	149	+	assert imported < IMPORTS.len;
	150	+	set IMPORTS[imported] = symbol;
	151	+	set imported += 1;
	152	+	}
	153	+	}
	154	+	let package = try compile(&input, slot, aligned(codeEnd, 4096), &IMPORTS[..imported],
	155	+	asm::Program { text: &[], data: &[], symbols: &[], externalFixups: &[] }) catch error {
	156	+	report(error);
	157	+	return 1;
	158	+	};
	159	+	set catalogEntries[slot] = catalog::Entry { source: bytes, package };
	160	+	set codeEnd = package.codeAddress + package.code.len as u64 * 4;
	161	+	}
137	162		let codeSize = (codeEnd - CODE_ADDRESS) as u32;
138	163		assert codeEnd - CODE_ADDRESS <= @sizeOf([u32; 4194304]) as u64;
139	164		let roAddress = aligned(codeEnd, 4096);
140		-	let roSize = try catalog::pack(&entries[..], roAddress, &mut RO[..]) catch error {
	165	+	let roSize = try catalog::pack(&catalogEntries[..count], roAddress, &mut RO[..]) catch error {
141	166		report(error);
142	167		return 1;
143	168		};
144	169		let rwAddress = aligned(roAddress + roSize as u64, 4096);
145	170		let mut bases: [u64; shared::MAX_PACKAGES] = [0; shared::MAX_PACKAGES];
146	171		let mut rwSize: u32 = @sizeOf([u64; shared::MAX_PACKAGES]);
147		-	for entry in &entries[..] {
	172	+	for entry in &catalogEntries[..2] {
148	173		let at = aligned(rwAddress + rwSize as u64, entry.package.alignment);
149	174		assert at - rwAddress <= RW.len as u64 and entry.package.memory <= RW.len - (at - rwAddress) as u32;
150	175		set bases[entry.package.slot] = at;
151	176		set rwSize = (at - rwAddress) as u32 + entry.package.memory;
152	177		}
153	178		try! mem::copy(&mut RW[..@sizeOf([u64; shared::MAX_PACKAGES])],
154	179		@sliceOf(&bases[0] as *unsafe u8, @sizeOf([u64; shared::MAX_PACKAGES])));
155		-	for entry in &entries[..] {
	180	+	for entry in &catalogEntries[..2] {
156	181		let at = (bases[entry.package.slot] - rwAddress) as u32;
157	182		try shared::instantiate(&entry.package, &bases[..], &mut RW[at..at + entry.package.memory])
158	183		catch error {
159	184		report(error);
160	185		return 1;
161	186		};
162	187		}
163	188		for i in 0..codeSize / 4 {
164	189		set NATIVE[i] = encode::nop();
165	190		}
166		-	for entry in &entries[..] {
	191	+	for entry in &catalogEntries[..count] {
167	192		let at = ((entry.package.codeAddress - CODE_ADDRESS) / 4) as u32;
168	193		for word, i in entry.package.code {
169	194		set NATIVE[at + i] = word;
170	195		}
171	196		}
172		-	trampoline(kernelAddress, roAddress, rwAddress);
	197	+	trampoline(kernelAddress, roAddress, rwAddress, count);
173	198		let header = try! image::header(image::Layout {
174	199		entry: CODE_ADDRESS,
175	200		code: image::Segment { address: CODE_ADDRESS, initialized: codeSize, memory: codeSize },
176	201		roData: image::Segment { address: roAddress, initialized: roSize, memory: roSize },
177	202		rwData: image::Segment { address: rwAddress, initialized: rwSize, memory: rwSize },

21	21		cat "$work/log" >&2
22	22		exit 1
23	23		fi
24	24		printf 'kernel boot: %s harts passed\n' "$harts"
25	25		done
	26	+	cat kernel/kernel/*.ras > "$work/idle.ras"
	27	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/idle.ras" "$work/idle.rv64"
26	28		status=0
27		-	"$emulator" -machine -harts=8 -max-steps=80000000 -run bin/kernel.rv64 > "$work/log" 2>&1 \|\| status=$?
	29	+	"$emulator" -machine -harts=8 -max-steps=80000000 -run "$work/idle.rv64" > "$work/log" 2>&1 \|\| status=$?
28	30		if [ "$status" -ne 2 ] \|\| [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
29	31		cat "$work/log" >&2
30	32		exit 1
31	33		fi
32	34		printf 'kernel startup: all eight harts reached machine idle without traps\n'
33	35		cat test/boot/trap.ras kernel/kernel/*.ras > "$work/trap.ras"
34		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
	36	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
35	37		status=0
36	38		"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 \|\| status=$?
37	39		if [ "$status" -ne 2 ] \|\| [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
38	40		\|\| [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then
39	41		cat "$work/log" >&2

8	8		work=$(mktemp -d)
9	9		trap 'rm -rf "$work"' EXIT HUP INT TERM
10	10		mkdir "$work/spin"
11	11		cp "test/$fixture/spin.rad" "$work/spin.rad"
12	12		cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
13		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
14		-	-pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
	13	+	set -- -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad"
	14	+	if [ "$fixture" = mmio ]; then
	15	+	cp kernel/mmio.rad "$work/spin/"
	16	+	set -- "$@" -mod "$work/spin/mmio.rad"
	17	+	fi
	18	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/radiance.rv64.dev \
	19	+	"$@" -entry spin -ril "$work"
15	20		cp kernel/kernel.rad "$work/kernel.rad"
16	21		printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
17	22		mkdir "$work/kernel"
18	23		cp kernel/kernel/*.rad "$work/kernel/"
19		-	cp "test/$fixture/kernel/dispatchcheck.rad" "$work/kernel/"
	24	+	for source in "test/$fixture"/kernel/*.rad; do
	25	+	cp "$source" "$work/kernel/"
	26	+	module=${source##*/}
	27	+	module=${module%.rad}
	28	+	if [ "$module" != dispatchcheck ]; then
	29	+	printf '\nexport mod %s;\n' "$module" >> "$work/kernel.rad"
	30	+	fi
	31	+	done
20	32		length=$(wc -c < "$work/spin.ril")
21	33		printf '//! Binary user loop.\n/// Complete trusted input package.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/dispatchinput.rad"
22	34		od -An -v -tu1 "$work/spin.ril" \| awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
23	35		printf '];\n' >> "$work/kernel/dispatchinput.rad"
24	36		sh test/acceptance/compile "$emulator" "$work"
25	37		cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
26		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
	38	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
27	39		-- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
28	40		set --
29	41		if [ "$irq" -ne 0 ]; then set -- "-irq=$irq" "-irq-at=1"; fi
30	42		for count in $harts; do
	43	+	if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
	44	+	sh test/acceptance/replay "$fixture" "$emulator" "$work/dispatch.rv64" "$count" "$@"
	45	+	else
31	46		"$emulator" -machine -harts="$count" -memory-size=262144 -max-steps=2000000000 "$@" -run "$work/dispatch.rv64"
	47	+	fi
32	48		printf '%s: %s-hart execution passed\n' "$fixture" "$count"
33	49		done

2	2		# Build a kernel fixture and call code produced by its runtime loader.
3	3		set -eu
4	4		emulator=${RAD_EMULATOR:-emulator}
5	5		work=$(mktemp -d)
6	6		trap 'rm -rf "$work"' EXIT HUP INT TERM
7		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
	7	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
8	8		-run bin/radiance.rv64.dev -pkg loaded -mod test/loader/loaded.rad -entry loaded -ril "$work"
9		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
	9	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
10	10		-run bin/radiance.rv64.dev -pkg loaded -mod test/loader/loaded.rad -entry loaded -o "$work/hosted.rv64"
11	11		status=0
12	12		"$emulator" -run "$work/hosted.rv64" \|\| status=$?
13	13		test "$status" -eq 43
14	14		cat kernel/kernel.rad > "$work/kernel.rad"

20	20		printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
21	21		od -An -v -tu1 "$work/loaded.ril" \| awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
22	22		printf '];\n' >> "$work/kernel/loadinput.rad"
23	23		sh test/acceptance/compile "$emulator" "$work"
24	24		cat test/loader/machine.ras kernel/kernel/*.ras > "$work/loader.ras"
25		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
	25	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
26	26		-run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
27	27		"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
28	28		printf 'runtime loader: hosted and loaded entries agree after workspace release\n'

13	13		set -- "$@" -mod "$work/capacity/m$index.rad"
14	14		index=$((index + 1))
15	15		done
16	16		printf 'use capacity::m126;\n/// Check a package root beyond module index 127.\n@default fn main() -> u64 { assert m126::value() == 126; return 0; }\n' > "$work/entry.rad"
17	17		set -- "$@" -pkg entry -mod "$work/entry.rad"
18		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
	18	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
19	19		-run bin/radiance.rv64.dev "$@" -entry entry -o "$work/capacity.rv64"
20	20		"$emulator" -run "$work/capacity.rv64"
21	21		printf 'module scopes: references beyond index 127 passed\n'

31	31		printf '\n/// Package compiled by scheduled user calls.\nexport static PAYLOAD: [u8; %s] = [\n' "$length" >> "$work/kernel/dispatchinput.rad"
32	32		od -An -v -tu1 "$work/loaded.ril" \| awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/dispatchinput.rad"
33	33		printf '];\n' >> "$work/kernel/dispatchinput.rad"
34	34		sh test/acceptance/compile "$emulator" "$work"
35	35		cat "test/$fixture/machine.ras" kernel/kernel/*.ras > "$work/dispatch.ras"
36		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
	36	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 -run bin/kernel.build.rv64 \
37	37		-- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
38		-	"$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
	38	+	if [ "${KERNEL_REPLAY:-0}" = 1 ]; then
	39	+	sh test/acceptance/replay runtime "$emulator" "$work/dispatch.rv64" 1
	40	+	else
	41	+	"$emulator" -machine -harts=1 -memory-size=262144 -max-steps=2000000000 -run "$work/dispatch.rv64"
	42	+	fi
39	43		printf 'runtime: scheduled cancellation, retry, and publication passed\n'

2	2		# Compile two packages once and execute them with two private state graphs.
3	3		set -eu
4	4		emulator=${RAD_EMULATOR:-emulator}
5	5		work=$(mktemp -d)
6	6		trap 'rm -rf "$work"' EXIT HUP INT TERM
7		-	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
	7	+	"$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
8	8		-run bin/radiance.rv64.dev \
9	9		-pkg support -mod test/shared/support.rad \
10	10		-pkg app -mod test/shared/app.rad -entry app -ril "$work"
11	11		"$emulator" -run bin/shared.build.rv64 -- "$work/support.ril" "$work/app.ril" "$work/native.rv64"
12	12		"$emulator" -machine -run "$work/native.rv64"

8	8		for name in duplicate abandon; do
9	9		case "$name" in
10	10		duplicate) diagnostic='linear value used after consumption' ;;
11	11		abandon) diagnostic='linear value is not consumed' ;;
12	12		esac
13		-	if "$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
	13	+	if "$emulator" -memory-size=385024 -data-size=348160 -stack-size=1024 \
14	14		-run bin/radiance.rv64.dev "$@" -test \
15	15		-pkg "$name" -mod "test/slots/$name.rad" -entry "$name" \
16	16		-o "$work/test.rv64" > "$work/log" 2>&1; then
17	17		printf 'reservation misuse compiled: %s\n' "$name" >&2
18	18		exit 1