radiance · commit

kernel: Connect the syscall ABI

a63245246ef5cb1df068cbc0e73e8d6ef3eb3c15150cb2e340ea6aeb3a604a3e

Assisted-by: Codex:gpt-6

Alexis Sellier committed 13 days ago 1 parent 03e8c2d1

Makefile +1 -1

		lib/std.test.rv64.s \
		lib/std.test.rv64.o

# Kernel modules and tests use a separate package and test entry point.
KERNEL_SRC := kernel/kernel.rad $(shell find kernel/kernel -name '*.rad' ! -name 'tests.rad' ! -path '*/tests/*' 2>/dev/null)
KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras
KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras
KERNEL_TEST_SRC := kernel/kernel/tests.rad $(shell find kernel/kernel/tests -name '*.rad' 2>/dev/null)
KERNEL := -pkg kernel $(patsubst %,-mod %,$(sort $(KERNEL_SRC)))
KERNEL_TEST := $(BIN_DIR)/kernel.test.rv64

kernel-test: emulator $(KERNEL_TEST)

kernel/kernel.rad +2 -0

export mod loader;
export mod instances;
export mod domains;
export mod budgets;
export mod dispatch;
export mod calls;
export mod sys;
export mod boot;
@test export mod tests;

kernel/kernel/abi.rad +74 -0

/// Domain-relative capability value. Zero is the self sentinel where allowed.
export record Handle: Copy(u64);
/// Rights granted by a live capability-table entry.
export record Rights: Copy(u16);

/// Startup values retained for the lifetime of a protection domain.
export record Env: Copy {
    /// Address of the readable startup argument block.
    argsPointer: u64,
    /// Number of startup argument bytes.
    argsSize: u64,
    /// Installed domain-relative Events handle.
    eventsHandle: u64,
    /// Address of the domain's shared event ring.
    eventsPointer: u64,
}

/// Physical extent returned by QueryPage.
export record PageInfo: Copy {
    /// Address of the first physical frame.
    base: u64,
    /// Number of contiguous physical frames.
    count: u64,
}

/// Domain identity and lifetime returned by QueryDomain.
export record DomainInfo: Copy {
    /// Generation-bearing domain identity.
    id: u64,
    /// Historical creator identity.
    creator: u64,
    /// Current parent identity, or zero if absent.
    parent: u64,
    /// Pending, Active, or Dead encoded as zero, one, or two.
    state: u64,
}

/// Shared notification queue shape returned by QueryEvents.
export record EventsInfo: Copy {
    /// Number of entries in the installed ring.
    capacity: u64,
}

/// Shared code and private state extents returned by QueryImage.
export record ImageInfo: Copy {
    /// Generation-bearing resident package identity.
    id: u64,
    /// Default native entry, or zero if absent.
    entry: u64,
    /// Address of shared executable code.
    codeBase: u64,
    /// Number of shared executable bytes.
    codeBytes: u64,
    /// Number of private state bytes per instance.
    dataBytes: u64,
}

/// Identity and CPU authority of the calling execution context.
export record CurrentContextInfo: Copy {
    /// Generation-bearing context identity.
    context: u64,
    /// Physical hart executing this call.
    hart: u64,
    /// Domain-relative handle for the executing budget.
    budget: u64,
}

/// Exclusive CPU authority returned by QueryBudget.
export record BudgetInfo: Copy {
    /// Physical hart authorized by this window.
    hart: u64,
    /// Inclusive first mtime tick.
    start: u64,
    /// Exclusive last mtime tick.
    end: u64,
    /// Generation-bearing context identity, or zero if unbound.
    context: u64,
}

/// Read object contents or metadata.
export constant READ: u16 = 1;
/// Modify object contents.
export constant WRITE: u16 = 2;
/// Execute an image or entry.

kernel/kernel/boot.rad +5 -1

/// Number of harts that validated their machine stack.
static ARRIVED: u64 = 0;
/// Trap anchors owned by their physical harts.
unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;

/// Handle timer boundaries and stop on unsupported traps.
/// Handle timer boundaries and user calls, and stop on unsupported traps.
unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
    if trap::classify(frame.cause) == trap::Cause::Timer {
        dispatch::interrupt(frame, hart);
        return;
    }
    if trap::classify(frame.cause) == trap::Cause::UserCall and trap::fromUser(frame) {
        dispatch::call(frame, hart);
        return;
    }
    print("kernel: unexpected trap\n");
    trap::halt();
}

/// Read a byte from a device register with I/O ordering.

kernel/kernel/budgets.rad +7 -0

    let removed = try! capability::invalidate(table, second);
    try! slots::release(&mut store.slots[..], b);
    return first;
}

/// Relinquish a held interval and invalidate its sole capability.
export fn drop(store: &mut Store, table: &mut capability::Table, handle: abi::Handle) throws (abi::Error) {
    let object = try owned(store, table, handle, 0);
    let removed = try! capability::invalidate(table, handle);
    try! slots::release(&mut store.slots[..], object);
}

/// Retire an expired interval and its sole handle.
export fn expire(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, now: u64) -> bool throws (abi::Error) {
    let object = try owned(store, table, handle, 0);
    if now < store.windows[object.index].end { return false; }
    let removed = try! capability::invalidate(table, handle);

kernel/kernel/calls.rad added +180 -0

1	+	//! Direct-call execution and checked copies into domain-owned memory.
2	+
3	+	use super::abi;
4	+	use super::slots;
5	+	use super::limits;
6	+	use super::capability;
7	+	use super::pages;
8	+	use super::domains;
9	+	use super::registry;
10	+	use super::budgets;
11	+	use super::trap;
12	+	use super::events;
13	+	use super::transactions;
14	+	use super::loader;
15	+	use super::dispatch;
16	+
17	+	/// Validate resident object generations for capabilities without page ownership.
18	+	unsafe fn resident(entry: capability::Entry) throws (abi::Error) {
19	+	match entry.kind {
20	+	case abi::Kind::Domain, abi::Kind::Events => { let object = try domains::get(&domains::STORE, entry.object); },
21	+	case abi::Kind::Image => { let object = try registry::get(&registry::STORE, entry.object); },
22	+	else => throw abi::Error::BadHandle,
23	+	}
24	+	}
25	+
26	+	/// Grant or transfer authority while both domain tables remain serialized.
27	+	unsafe fn delegate(domain: &mut domains::Domain, handle: abi::Handle, target: abi::Handle, rights: u64, moving: bool, now: u64)
28	+	-> abi::Handle throws (abi::Error)
29	+	{
30	+	let owner = domain.memory.table.owner;
31	+	let mut receiver = owner;
32	+	if *target <> 0 {
33	+	let permission = try capability::lookup(&domain.memory.table, target, abi::Kind::Domain, abi::Rights(0));
34	+	set receiver = permission.object;
35	+	}
36	+	if moving and receiver == owner { throw abi::Error::InvalidArg; }
37	+	let mut destination = try domains::get(&domains::STORE, receiver);
38	+	if destination.state == domains::Lifecycle::Dead { throw abi::Error::BadHandle; }
39	+	let entry = try capability::get(&domain.memory.table, handle);
40	+	if entry.kind == abi::Kind::Page {
41	+	if moving { return try pages::transfer(&mut pages::STORE, &mut domain.memory.table, &mut destination.memory.table, handle, rights); }
42	+	if receiver == owner { return try pages::selfGrant(&mut pages::STORE, &mut domain.memory.table, handle, rights); }
43	+	return try pages::grant(&mut pages::STORE, &domain.memory.table, &mut destination.memory.table, handle, rights);
44	+	}
45	+	if entry.kind == abi::Kind::Budget {
46	+	if not moving { throw abi::Error::Denied; }
47	+	return try budgets::transfer(&mut budgets::STORE, &mut domain.memory.table, &mut destination.memory.table, handle, rights, now);
48	+	}
49	+	try resident(entry);
50	+	let pending = try transactions::reserve(&mut destination.memory.table, owner, handle, entry, rights, moving);
51	+	if moving { return transactions::transfer(&mut domain.memory.table, &mut destination.memory.table, pending); }
52	+	return transactions::grant(&mut destination.memory.table, pending);
53	+	}
54	+
55	+	/// Drop one capability and release the object-specific ownership it carries.
56	+	unsafe fn drop(domain: &mut domains::Domain, handle: abi::Handle) throws (abi::Error) {
57	+	let entry = try transactions::dropping(&domain.memory.table, handle, domain.events);
58	+	let table: &mut capability::Table = &mut domain.memory.table;
59	+	match entry.kind {
60	+	case abi::Kind::Page => try pages::drop(&mut pages::STORE, table, handle),
61	+	case abi::Kind::Budget => try budgets::drop(&mut budgets::STORE, table, handle),
62	+	else => {
63	+	try resident(entry);
64	+	let removed = try! capability::invalidate(table, handle);
65	+	},
66	+	}
67	+	}
68	+
69	+	/// Map a checked, aligned result address to writable words.
70	+	fn memory(address: u64) -> *mut u64;
71	+
72	+	/// Test whether one nonwrapping extent contains the complete result range.
73	+	fn contains(base: u64, size: u64, address: u64, length: u64) -> bool {
74	+	return address >= base and address - base <= size and length <= size - (address - base);
75	+	}
76	+
77	+	/// Validate writable Page or private package-data storage for a result record.
78	+	/// Package-state tables and kernel metadata are not writable result storage.
79	+	export unsafe fn resultBuffer(domain: &domains::Domain, address: u64, size: u64) -> bool {
80	+	if (address & 7) <> 0 or size > 0xffffffffffffffff - address { return false; }
81	+	if pages::accessible(&pages::STORE, &domain.memory.table, address, size, abi::Rights(abi::WRITE)) { return true; }
82	+	for i in 0..limits::PACKAGES {
83	+	if registry::STORE.slots[i].state <> slots::State::Live { continue; }
84	+	let base = domain.graph.table[i];
85	+	if base == 0 { continue; }
86	+	let package = try! registry::get(&registry::STORE, abi::Ref { index: i, generation: registry::STORE.slots[i].generation });
87	+	if contains(base, package.memory as u64, address, size) { return true; }
88	+	}
89	+	return false;
90	+	}
91	+
92	+	/// Copy a fixed query record only after validating its exact buffer size.
93	+	unsafe fn reply(domain: &domains::Domain, arguments: &[u64], words: &[u64]) throws (abi::Error) {
94	+	let size = words.len as u64 * 8;
95	+	if arguments[2] <> size or arguments[3] <> 0 or not resultBuffer(domain, arguments[1], size) {
96	+	throw abi::Error::InvalidArg;
97	+	}
98	+	let output = @sliceOf(memory(arguments[1]), words.len);
99	+	for i in 0..words.len { set output[i] = words[i]; }
100	+	}
101	+
102	+	/// Execute one operation with domain and resource metadata serialized.
103	+	export unsafe fn invoke(owner: abi::Ref, operation: u64, arguments: &[u64], now: u64) -> u64 throws (abi::Error) {
104	+	assert arguments.len == 4;
105	+	let mut domain = try domains::get(&domains::STORE, owner);
106	+	let handle = abi::Handle(arguments[0]);
107	+	match operation {
108	+	case 10, 11 => return *try delegate(&mut domain, handle, abi::Handle(arguments[1]), arguments[2], operation == 11, now),
109	+	case 12 => { try drop(&mut domain, handle); return 0; },
110	+	case 20 => return *try domains::create(&mut domains::STORE, &mut pages::STORE.backings,
111	+	&registry::STORE, &mut domain.memory.table, handle, abi::Handle(arguments[1])),
112	+	case 21 => {
113	+	try domains::activate(&mut domains::STORE, &pages::STORE, &domain.memory.table,
114	+	handle, arguments[1], arguments[2], arguments[3]);
115	+	return 0;
116	+	},
117	+	case 30 => return *try pages::allocate(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
118	+	case 31 => return *try pages::split(&mut pages::STORE, &mut domain.memory.table, handle, arguments[1]),
119	+	case 44 => {
120	+	let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Page, abi::Rights(0));
121	+	let page = try pages::get(&pages::STORE, entry.object);
122	+	try reply(&domain, arguments, &[page.base, page.count as u64]);
123	+	return 0;
124	+	},
125	+	case 47 => {
126	+	let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Domain, abi::Rights(0));
127	+	let target = try domains::get(&domains::STORE, entry.object);
128	+	let mut parent: u64 = 0;
129	+	if let object = target.parent { set parent = abi::id(object); }
130	+	try reply(&domain, arguments, &[abi::id(entry.object), abi::id(target.creator), parent, target.state as u64]);
131	+	return 0;
132	+	},
133	+	case 48 => {
134	+	let entry = try capability::lookup(&domain.memory.table, handle, abi::Kind::Events, abi::Rights(0));
135	+	let target = try domains::get(&domains::STORE, entry.object);
136	+	try reply(&domain, arguments, &[events::CAPACITY as u64]);
137	+	return 0;
138	+	},
139	+	case 51 => return *try loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut domain.memory.table,
140	+	loader::Request { authority: handle, source: abi::Handle(arguments[1]), offset: arguments[2], length: arguments[3] }),
141	+	case 52 => {
142	+	let object = try registry::image(&registry::STORE, &domain.memory.table, handle, abi::Rights(abi::READ));
143	+	let package = try! registry::get(&registry::STORE, object);
144	+	let mut entry: u64 = 0;
145	+	if let address = package.entry { set entry = address; }
146	+	try reply(&domain, arguments, &[abi::id(object), entry, package.codeAddress, package.code.len as u64 * 4, package.memory as u64]);
147	+	return 0;
148	+	},
149	+	case 63 => {
150	+	if arguments[0] <> 0 { throw abi::Error::InvalidArg; }
151	+	let current = dispatch::current();
152	+	try reply(&domain, arguments, &[current.context, current.hart, current.budget]);
153	+	return 0;
154	+	},
155	+	case 70 => return *try budgets::split(&mut budgets::STORE, &mut domain.memory.table, handle, arguments[1], now),
156	+	case 71 => return *try budgets::merge(&mut budgets::STORE, &mut domain.memory.table, handle, abi::Handle(arguments[1]), now),
157	+	case 72 => {
158	+	let window = try budgets::query(&budgets::STORE, &domain.memory.table, handle);
159	+	let mut context: u64 = 0;
160	+	if let object = window.context { set context = abi::id(object); }
161	+	try reply(&domain, arguments, &[window.hart as u64, window.start, window.end, context]);
162	+	return 0;
163	+	},
164	+	case 73 => return *try budgets::bind(&mut budgets::STORE, &domains::STORE, &mut domain.memory.table,
165	+	budgets::Binding { budget: handle, domain: abi::Handle(arguments[1]), context: abi::reference(arguments[2]) }, now),
166	+	else => throw abi::Error::InvalidArg,
167	+	}
168	+	}
169	+
170	+	/// Complete a user ecall and preserve all registers except its scalar result.
171	+	export unsafe fn handle(owner: abi::Ref, frame: &mut trap::Frame, now: u64) {
172	+	assert trap::fromUser(frame) and frame.cause == 8;
173	+	assert trap::advanceCall(frame);
174	+	let args = [frame.registers[10], frame.registers[11], frame.registers[12], frame.registers[13]];
175	+	let result = try invoke(owner, frame.registers[17], &args[..], now) catch error {
176	+	set frame.registers[10] = (0 as u64) - error as u64;
177	+	return;
178	+	};
179	+	set frame.registers[10] = result;
180	+	}

kernel/kernel/calls.ras added +7 -0

1	+	//! Typed access to a checked user result buffer.
2	+	.text;
3	+	.export @kernel::calls::memory;
4	+
5	+	// The caller checked alignment, extent, and writable domain ownership.
6	+	@kernel::calls::memory
7	+	ret;

kernel/kernel/dispatch.rad +21 -0

use super::budgets;
use super::domains;
use super::platform;
use super::trap;
use super::range;
use super::calls;

/// Dispatch ownership and idle state retained by one physical hart.
export record State: Copy {
    /// Physical hart whose windows this state selects.
    hart: u32,

    let clock = now(HARTS[id].timer);
    let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
    arm(HARTS[id].timer, choice.deadline);
}

/// Read the running context's identity and domain-relative CPU authority.
export unsafe fn current() -> abi::CurrentContextInfo {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else panic "no current context";
    let budget = HARTS[id].state.budget else panic "no current budget";
    let window = try! budgets::get(&budgets::STORE, budget);
    return abi::CurrentContextInfo { context: abi::id(context), hart: id as u64, budget: *window.handle };
}

/// Complete the current user's call and select authority at the return boundary.
export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
    let id = hart();
    assert id < limits::HARTS;
    let context = HARTS[id].state.current else { trap::halt(); return; };
    assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
    calls::handle(domains::STORE.contexts[context.index].owner, frame, now(HARTS[id].timer));
    interrupt(frame, anchor);
}

/// Enter the first authorized context or the hart's retained idle frame.
export unsafe fn start(anchor: &mut trap::Hart) -> ! {
    let id = hart();
    assert id < limits::HARTS;
    let mut frame = HARTS[id].state.idle;

kernel/kernel/domains.rad +28 -0

use super::registry;
use super::instances;
use super::events;
use super::trap;
use super::range;
use super::pages;

/// Retained kernel stack capacity for one suspended execution context.
export constant KERNEL_STACK_PAGES: u32 = 64;

/// Management rights returned to a domain's creator.

export record Memory: Copy {
    /// Authoritative capabilities; only the kernel accesses this table.
    table: capability::Table,
    /// Event payload and consumer counters exposed to this domain.
    ring: events::Ring,
    /// Startup values exposed read-only to this domain.
    env: abi::Env,
}

/// Saved integer state owned by one execution context.
export record Context: Copy {
    /// Domain that owns this context.

export fn get(store: &Store, object: abi::Ref) -> Domain throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) { throw abi::Error::BadHandle; }
    return store.records[object.index];
}

/// Activate a pending domain after checking its stack and argument authority.
/// The caller serializes metadata until the initial frame and state are published.
export unsafe fn activate(store: &mut Store, memory: &pages::Store, table: &capability::Table,
    handle: abi::Handle, stack: u64, args: u64, size: u64) throws (abi::Error)
{
    let entry = try capability::lookup(table, handle, abi::Kind::Domain, abi::Rights(abi::EXECUTE));
    let mut domain = try get(store, entry.object);
    if domain.state <> Lifecycle::Pending { throw abi::Error::NotPending; }
    if (stack & 15) <> 0 or stack < 16
        or not pages::accessible(memory, &domain.memory.table, stack - 16, 16, abi::Rights(abi::WRITE)) {
        throw abi::Error::InvalidArg;
    }
    if size <> 0 and not pages::accessible(memory, &domain.memory.table, args, size, abi::Rights(abi::READ)) {
        throw abi::Error::InvalidArg;
    }
    assert slots::matches(&store.contextSlots[..], domain.initial, slots::State::Live);
    set domain.memory.env = abi::Env {
        argsPointer: args, argsSize: size, eventsHandle: *domain.events,
        eventsPointer: (&domain.memory.ring) as u64,
    };
    set store.contexts[domain.initial.index].frame.registers[2] = stack;
    set store.contexts[domain.initial.index].frame.registers[10] = (&domain.memory.env) as u64;
    set store.records[entry.object.index].state = Lifecycle::Active;
}

/// Memory allocated before the domain and its context become observable.
record Prepared: Copy {
    /// Frames for capabilities and the event ring.
    allocation: frames::Run,
    /// Mapped metadata at the allocation's start.

kernel/kernel/pages.rad +25 -0

export fn get(store: &Store, object: abi::Ref) -> Page throws (abi::Error) {
    if not slots::matches(&store.slots[..], object, slots::State::Live) { throw abi::Error::BadHandle; }
    return store.records[object.index];
}

/// Check complete byte-range authority through live Page handles.
/// Adjacent authorized pages can cover one range. No memory is accessed here.
export fn accessible(store: &Store, table: &capability::Table, address: u64, size: u64, rights: abi::Rights) -> bool {
    if size > 0xffffffffffffffff - address { return false; }
    let end = address + size;
    let mut cursor = address;
    while cursor < end {
        let before = cursor;
        for i in 0..limits::HANDLES {
            if table.slots[i].state <> slots::State::Live { continue; }
            let entry = table.entries[i];
            if entry.kind <> abi::Kind::Page or not abi::permits(entry.rights, rights) { continue; }
            if not slots::matches(&store.slots[..], entry.object, slots::State::Live) { continue; }
            let page = store.records[entry.object.index];
            let limit = page.base + page.count as u64 * 4096;
            if page.base <= cursor and cursor < limit {
                set cursor = limit;
                if cursor >= end { return true; }
            }
        }
        if cursor == before { return false; }
    }
    return true;
}

/// Allocate and zero frames before publishing a page and its capability.
/// Allocation authority must name the calling domain. Failure cancels reservations.
export unsafe fn allocate(store: &mut Store, table: &mut capability::Table, authority: abi::Handle, count: u64)
    -> abi::Handle throws (abi::Error)
{

kernel/kernel/sys.rad added +132 -0

1	+	//! Typed user entry points for the RV64 direct-call ABI.
2	+
3	+	use super::abi;
4	+
5	+	/// Issue one machine call with four scalar arguments.
6	+	@intrinsic fn ecall(operation: u32, a0: i64, a1: i64, a2: i64, a3: i64) -> i64;
7	+
8	+	/// Translate scalar ABI errors into Radiance throws.
9	+	fn result(value: i64) -> u64 throws (abi::Error) {
10	+	if value >= 0 { return value as u64; }
11	+	match value {
12	+	case -1 => throw abi::Error::BadHandle,
13	+	case -2 => throw abi::Error::Denied,
14	+	case -3 => throw abi::Error::OutOfMemory,
15	+	case -4 => throw abi::Error::InvalidArg,
16	+	case -5 => throw abi::Error::Busy,
17	+	case -6 => throw abi::Error::VerifyFailed,
18	+	case -7 => throw abi::Error::NotPending,
19	+	case -8 => throw abi::Error::Exhausted,
20	+	else => panic "invalid kernel error",
21	+	}
22	+	}
23	+
24	+	/// Copy authority to a domain with optional rights narrowing.
25	+	export fn capabilityGrant(handle: abi::Handle, target: abi::Handle, rights: u64) -> abi::Handle throws (abi::Error) {
26	+	return abi::Handle(try result(ecall(abi::Operation::CapabilityGrant as u32, handle as i64, target as i64, rights as i64, 0)));
27	+	}
28	+
29	+	/// Load a trusted binary package from a readable Page range.
30	+	export fn imageLoad(authority: abi::Handle, source: abi::Handle, offset: u64, length: u64) -> abi::Handle throws (abi::Error) {
31	+	return abi::Handle(try result(ecall(abi::Operation::ImageLoad as u32, authority as i64, source as i64, offset as i64, length as i64)));
32	+	}
33	+
34	+	/// Read resident package metadata through a readable Image capability.
35	+	export fn queryImage(handle: abi::Handle) -> abi::ImageInfo throws (abi::Error) {
36	+	unsafe {
37	+	let mut info: abi::ImageInfo = undefined;
38	+	try result(ecall(abi::Operation::QueryImage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::ImageInfo) as i64, 0));
39	+	return info;
40	+	}
41	+	}
42	+
43	+	/// Read the calling context's identity, hart, and active CPU authority.
44	+	export fn currentContext() -> abi::CurrentContextInfo {
45	+	unsafe {
46	+	let mut info: abi::CurrentContextInfo = undefined;
47	+	try! result(ecall(abi::Operation::CurrentContext as u32, 0, (&mut info) as i64, @sizeOf(abi::CurrentContextInfo) as i64, 0));
48	+	return info;
49	+	}
50	+	}
51	+
52	+	/// Partition a held CPU window at an interior mtime tick.
53	+	export fn budgetSplit(handle: abi::Handle, at: u64) -> abi::Handle throws (abi::Error) {
54	+	return abi::Handle(try result(ecall(abi::Operation::BudgetSplit as u32, *handle as i64, at as i64, 0, 0)));
55	+	}
56	+
57	+	/// Combine adjacent CPU windows with the same context binding.
58	+	export fn budgetMerge(first: abi::Handle, second: abi::Handle) -> abi::Handle throws (abi::Error) {
59	+	return abi::Handle(try result(ecall(abi::Operation::BudgetMerge as u32, first as i64, second as i64, 0, 0)));
60	+	}
61	+
62	+	/// Bind a held CPU window to a context, transferring it to the target domain.
63	+	export fn budgetBind(handle: abi::Handle, domain: abi::Handle, context: u64) -> abi::Handle throws (abi::Error) {
64	+	return abi::Handle(try result(ecall(abi::Operation::BudgetBind as u32, handle as i64, domain as i64, context as i64, 0)));
65	+	}
66	+
67	+	/// Move authority to another domain with optional rights narrowing.
68	+	export fn capabilityTransfer(handle: abi::Handle, target: abi::Handle, rights: u64) -> abi::Handle throws (abi::Error) {
69	+	return abi::Handle(try result(ecall(abi::Operation::CapabilityTransfer as u32, handle as i64, target as i64, rights as i64, 0)));
70	+	}
71	+
72	+	/// Relinquish a handle and its object-specific authority.
73	+	export fn capabilityDrop(handle: abi::Handle) throws (abi::Error) {
74	+	try result(ecall(abi::Operation::CapabilityDrop as u32, *handle as i64, 0, 0, 0));
75	+	}
76	+
77	+	/// Allocate contiguous zeroed physical frames with self Allocate authority.
78	+	export fn pageAllocate(handle: abi::Handle, count: u64) -> abi::Handle throws (abi::Error) {
79	+	return abi::Handle(try result(ecall(abi::Operation::PageAllocate as u32, *handle as i64, count as i64, 0, 0)));
80	+	}
81	+
82	+	/// Partition an exclusively held Page, returning its right-hand segment.
83	+	export fn pageSplit(handle: abi::Handle, leftCount: u64) -> abi::Handle throws (abi::Error) {
84	+	return abi::Handle(try result(ecall(abi::Operation::PageSplit as u32, *handle as i64, leftCount as i64, 0, 0)));
85	+	}
86	+
87	+	/// Create a pending domain with a private instance of the resident image.
88	+	export fn domainCreate(handle: abi::Handle, image: abi::Handle) -> abi::Handle throws (abi::Error) {
89	+	return abi::Handle(try result(ecall(abi::Operation::DomainCreate as u32, handle as i64, image as i64, 0, 0)));
90	+	}
91	+
92	+	/// Activate a pending domain with its authorized stack and readable arguments.
93	+	export unsafe fn domainActivate(handle: abi::Handle, stackPointer: u64, argsPointer: *unsafe opaque, argsSize: u64) throws (abi::Error) {
94	+	try result(ecall(abi::Operation::DomainActivate as u32, *handle as i64,
95	+	stackPointer as i64, argsPointer as i64, argsSize as i64));
96	+	}
97	+
98	+	/// Read Page metadata. An invalid handle is a program error.
99	+	export fn queryPage(handle: abi::Handle) -> abi::PageInfo {
100	+	unsafe {
101	+	let mut info: abi::PageInfo = undefined;
102	+	try! result(ecall(abi::Operation::QueryPage as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::PageInfo) as i64, 0));
103	+	return info;
104	+	}
105	+	}
106	+
107	+	/// Read domain identity and lifetime. An invalid handle is a program error.
108	+	export fn queryDomain(handle: abi::Handle) -> abi::DomainInfo {
109	+	unsafe {
110	+	let mut info: abi::DomainInfo = undefined;
111	+	try! result(ecall(abi::Operation::QueryDomain as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::DomainInfo) as i64, 0));
112	+	return info;
113	+	}
114	+	}
115	+
116	+	/// Read event queue capacity. An invalid handle is a program error.
117	+	export fn queryEvents(handle: abi::Handle) -> abi::EventsInfo {
118	+	unsafe {
119	+	let mut info: abi::EventsInfo = undefined;
120	+	try! result(ecall(abi::Operation::QueryEvents as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::EventsInfo) as i64, 0));
121	+	return info;
122	+	}
123	+	}
124	+
125	+	/// Read the current extent and binding of a held CPU window.
126	+	export fn queryBudget(handle: abi::Handle) -> abi::BudgetInfo throws (abi::Error) {
127	+	unsafe {
128	+	let mut info: abi::BudgetInfo = undefined;
129	+	try result(ecall(abi::Operation::QueryBudget as u32, *handle as i64, (&mut info) as i64, @sizeOf(abi::BudgetInfo) as i64, 0));
130	+	return info;
131	+	}
132	+	}

kernel/kernel/tests.rad +1 -0

export mod loader;
export mod instances;
export mod domains;
export mod budgets;
export mod dispatch;
export mod calls;

kernel/kernel/tests/calls.rad added +156 -0

1	+	//! Direct-call scalar results, checked record copies, and architectural replies.
2	+
3	+	use std::testing;
4	+	use kernel::abi;
5	+	use kernel::calls;
6	+	use kernel::domains;
7	+	use kernel::pages;
8	+	use kernel::backing;
9	+	use kernel::capability;
10	+	use kernel::slots;
11	+	use kernel::registry;
12	+	use kernel::limits;
13	+	use kernel::trap;
14	+	use kernel::budgets;
15	+
16	+	/// Mapped physical storage for four pages plus alignment padding.
17	+	static RAM: [u64; 2560] = [0; 2560];
18	+	/// Kernel-only metadata for the calling domain.
19	+	unsafe static MEMORY: domains::Memory = undefined;
20	+	/// Metadata for a separate capability receiver.
21	+	unsafe static RECEIVER: domains::Memory = undefined;
22	+	/// Empty package-state graph for the calling domain.
23	+	static BASES: [u64; limits::PACKAGES] = [0; limits::PACKAGES];
24	+	/// Offset to the first aligned physical word.
25	+	static OFFSET: u32 = 0;
26	+
27	+	/// Initialize one active caller with physical allocation authority.
28	+	unsafe fn initialize() -> abi::Ref {
29	+	domains::initialize(&mut domains::STORE);
30	+	registry::initialize(&mut registry::STORE);
31	+	budgets::initialize(&mut budgets::STORE);
32	+	slots::initialize(&mut pages::STORE.slots[..]);
33	+	backing::initialize(&mut pages::STORE.backings);
34	+	let base = (&RAM[0]) as u64;
35	+	let aligned = (base + 4095) & ~4095;
36	+	set OFFSET = ((aligned - base) / 8) as u32;
37	+	set pages::STORE.backings.pool.count = 4;
38	+	for i in 0..4 {
39	+	set pages::STORE.backings.pool.addresses[i] = aligned + i as u64 * 4096;
40	+	set pages::STORE.backings.pool.free[i] = true;
41	+	}
42	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
43	+	let owner = try! slots::commit(&mut domains::STORE.slots[..], pending);
44	+	try! backing::registerDomain(&mut pages::STORE.backings, owner);
45	+	capability::initialize(&mut MEMORY.table, owner);
46	+	let authority = try! capability::install(&mut MEMORY.table, capability::Entry {
47	+	kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::ALLOCATE),
48	+	});
49	+	set domains::STORE.records[owner.index] = domains::Domain {
50	+	state: domains::Lifecycle::Active, creator: owner, parent: nil, image: owner, initial: owner,
51	+	allocation: { first: 0, count: 0 }, memory: &mut MEMORY,
52	+	graph: { frames: { first: 0, count: 0 }, table: &BASES[..] }, events: abi::Handle(0),
53	+	};
54	+	return owner;
55	+	}
56	+
57	+	/// Failed transfers retain the source; successful moves preserve page exposure.
58	+	@test unsafe fn capabilityCalls() throws (testing::TestError) {
59	+	let owner = initialize();
60	+	let pending = try! slots::reserve(&mut domains::STORE.slots[..]);
61	+	let receiver = try! slots::commit(&mut domains::STORE.slots[..], pending);
62	+	try! backing::registerDomain(&mut pages::STORE.backings, receiver);
63	+	capability::initialize(&mut RECEIVER.table, receiver);
64	+	set domains::STORE.records[receiver.index] = domains::STORE.records[owner.index];
65	+	set domains::STORE.records[receiver.index].memory = &mut RECEIVER;
66	+	let target = try! capability::install(&mut MEMORY.table, capability::Entry {
67	+	kind: abi::Kind::Domain, object: receiver, rights: abi::Rights(0),
68	+	});
69	+	let handle = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
70	+	let entry = try! capability::get(&MEMORY.table, abi::Handle(handle));
71	+	let narrowed = try! calls::invoke(owner, 10, &[handle, 0, abi::READ as u64, 0], 0);
72	+	let granted = try! capability::get(&MEMORY.table, abi::Handle(narrowed));
73	+	try testing::expect(*granted.rights == abi::READ and pages::STORE.records[entry.object.index].handles == 2);
74	+	let dropped = try! calls::invoke(owner, 12, &[narrowed, 0, 0, 0], 0);
75	+	try testing::expect(dropped == 0 and pages::STORE.records[entry.object.index].handles == 1);
76	+	for i in 0..limits::HANDLES { set RECEIVER.table.slots[i].state = slots::State::Retired; }
77	+	let mut rejected = false;
78	+	try calls::invoke(owner, 11, &[handle, *target, abi::READ as u64, 0], 0) catch error {
79	+	try testing::expect(error == abi::Error::InvalidArg); set rejected = true;
80	+	};
81	+	try testing::expect(rejected and try! capability::get(&MEMORY.table, abi::Handle(handle)) == entry);
82	+	capability::initialize(&mut RECEIVER.table, receiver);
83	+	let moved = try! calls::invoke(owner, 11, &[handle, *target, abi::READ as u64, 0], 0);
84	+	let received = try! capability::get(&RECEIVER.table, abi::Handle(moved));
85	+	try testing::expect(received.object == entry.object and *received.rights == abi::READ);
86	+	let page = try! pages::get(&pages::STORE, entry.object);
87	+	try testing::expect(page.handles == 1 and pages::STORE.backings.records[page.backing.index].exposed == 2);
88	+	let decoded = try! abi::decode(abi::Handle(handle));
89	+	try testing::expect(MEMORY.table.slots[decoded.object.index].state == slots::State::Free);
90	+	}
91	+
92	+	/// Events remain installed, and budgets cannot duplicate execution authority.
93	+	@test unsafe fn nondelegableCalls() throws (testing::TestError) {
94	+	let owner = initialize();
95	+	let queue = try! capability::install(&mut MEMORY.table, capability::Entry {
96	+	kind: abi::Kind::Events, object: owner, rights: abi::Rights(abi::READ \| abi::WRITE),
97	+	});
98	+	set domains::STORE.records[owner.index].events = queue;
99	+	let window = try! budgets::seed(&mut budgets::STORE, &mut MEMORY.table, 0, 10, 20);
100	+	let mut rejected: u32 = 0;
101	+	try calls::invoke(owner, 12, &[*queue, 0, 0, 0], 0) catch error {
102	+	try testing::expect(error == abi::Error::Denied); set rejected += 1;
103	+	};
104	+	try calls::invoke(owner, 10, &[*window, 0, abi::READ as u64, 0], 0) catch error {
105	+	try testing::expect(error == abi::Error::Denied); set rejected += 1;
106	+	};
107	+	let narrowed = try! calls::invoke(owner, 10, &[*queue, 0, abi::READ as u64, 0], 0);
108	+	let dropped = try! calls::invoke(owner, 12, &[narrowed, 0, 0, 0], 0);
109	+	let removed = try! calls::invoke(owner, 12, &[*window, 0, 0, 0], 0);
110	+	try testing::expect(rejected == 2 and dropped == 0 and removed == 0);
111	+	try testing::expect(budgets::STORE.slots[0].state == slots::State::Free);
112	+	let installed = try! capability::get(&MEMORY.table, queue);
113	+	try testing::expect(installed.kind == abi::Kind::Events);
114	+	}
115	+
116	+	/// A query copies exactly its record into writable memory owned by the caller.
117	+	@test unsafe fn records() throws (testing::TestError) {
118	+	let owner = initialize();
119	+	let handle = try! calls::invoke(owner, 30, &[0, 1, 0, 0], 0);
120	+	let base = (&RAM[OFFSET]) as u64;
121	+	set RAM[OFFSET + 2] = 0x12345678;
122	+	let result = try! calls::invoke(owner, 44, &[handle, base, 16, 0], 0);
123	+	try testing::expect(result == 0 and RAM[OFFSET] == base and RAM[OFFSET + 1] == 1);
124	+	try testing::expect(RAM[OFFSET + 2] == 0x12345678);
125	+	let mut rejected: u32 = 0;
126	+	for args in &[
127	+	[handle, base + 1, 16, 0], [handle, base, 8, 0],
128	+	[handle, base + 4096, 16, 0], [handle, base, 16, 1],
129	+	[handle, 0xfffffffffffffff8, 16, 0],
130	+	] {
131	+	try calls::invoke(owner, 44, &args[..], 0) catch error {
132	+	try testing::expect(error == abi::Error::InvalidArg); set rejected += 1;
133	+	};
134	+	}
135	+	try testing::expect(rejected == 5 and RAM[OFFSET] == base and RAM[OFFSET + 1] == 1);
136	+	let decoded = try! abi::decode(abi::Handle(handle));
137	+	set MEMORY.table.entries[decoded.object.index].rights = abi::Rights(abi::READ);
138	+	try calls::invoke(owner, 44, &[handle, base, 16, 0], 0) catch error {
139	+	try testing::expect(error == abi::Error::InvalidArg); set rejected += 1;
140	+	};
141	+	try testing::expect(rejected == 6 and RAM[OFFSET + 2] == 0x12345678);
142	+	}
143	+
144	+	/// An unknown user operation advances PC and returns only a negative a0 error.
145	+	@test unsafe fn trapReply() throws (testing::TestError) {
146	+	let owner = initialize();
147	+	let mut frame = trap::Frame { registers: [123; 32], pc: 0x80000000, status: 0x80, cause: 8, value: 0 };
148	+	set frame.registers[17] = 0xffffffffffffffff;
149	+	calls::handle(owner, &mut frame, 0);
150	+	try testing::expect(frame.pc == 0x80000004 and frame.registers[10] == 0xfffffffffffffffc);
151	+	for i in 0..32 {
152	+	if i == 10 or i == 17 { continue; }
153	+	try testing::expect(frame.registers[i] == 123);
154	+	}
155	+	try testing::expect(frame.registers[17] == 0xffffffffffffffff and frame.status == 0x80);
156	+	}

kernel/kernel/tests/domains.rad +50 -0

use kernel::backing;
use kernel::capability;
use kernel::registry;
use kernel::domains;
use kernel::transactions;
use kernel::pages;

/// Mapped physical memory for child metadata and package instances.
static RAM: [u64; 131072] = [0; 131072];
/// Domain and context metadata under test.
unsafe static DOMAINS: domains::Store = undefined;

unsafe static PACKAGES: registry::Store = undefined;
/// Calling bootstrap domain's capability table.
unsafe static TABLE: capability::Table = undefined;
/// Executable image capability in the calling table.
static IMAGE: abi::Handle = abi::Handle(0);
/// Page metadata used to check activation authority without reading user memory.
unsafe static PAGES: pages::Store = undefined;

/// Initialize bootstrap authority and one executable image over mapped frames.
unsafe fn initialize(rights: u16) {
    domains::initialize(&mut DOMAINS);
    backing::initialize(&mut BACKINGS);

        template: &[7], memory: 8, alignment: 8, relocations: &[],
    });
    set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
}

/// Activation validates child authority before publishing its initial state.
@test unsafe fn activation() throws (testing::TestError) {
    initialize(abi::CREATE);
    slots::initialize(&mut PAGES.slots[..]);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let entry = try! capability::get(&TABLE, handle);
    let mut child = try! domains::get(&DOMAINS, entry.object);
    let pending = try! slots::reserve(&mut PAGES.slots[..]);
    let object = slots::reference(&pending);
    set PAGES.records[object.index] = pages::Page {
        backing: object, base: 0x90000000, count: 1, origin: entry.object, handles: 1,
    };
    let published = try! slots::commit(&mut PAGES.slots[..], pending);
    let memory = try! capability::install(&mut child.memory.table, capability::Entry {
        kind: abi::Kind::Page, object: published, rights: abi::Rights(abi::READ),
    });
    let mut failures: u32 = 0;
    try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0, 0) catch err {
        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
    };
    let slot = try! abi::decode(memory);
    set child.memory.table.entries[slot.object.index].rights = abi::Rights(abi::READ | abi::WRITE);
    try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90000fff, 0, 0) catch err {
        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
    };
    try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90000000, 0, 0) catch err {
        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
    };
    try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0x90000fff, 2) catch err {
        try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
    };
    try testing::expect(failures == 4 and DOMAINS.records[entry.object.index].state == domains::Lifecycle::Pending);
    try testing::expect(DOMAINS.contexts[child.initial.index].frame.registers[2] == 0);
    try! domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0x90000000, 4096);
    try testing::expect(DOMAINS.records[entry.object.index].state == domains::Lifecycle::Active);
    let frame = DOMAINS.contexts[child.initial.index].frame;
    try testing::expect(frame.registers[2] == 0x90001000);
    try testing::expect(frame.registers[10] == (&child.memory.env) as u64);
    try testing::expect(child.memory.env.argsPointer == 0x90000000 and child.memory.env.argsSize == 4096);
    try testing::expect(child.memory.env.eventsHandle == *child.events);
    try testing::expect(child.memory.env.eventsPointer == (&child.memory.ring) as u64);
    try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0, 0) catch err {
        try testing::expect(err == abi::Error::NotPending); set failures += 1;
    };
    try testing::expect(failures == 5);
}

/// Creation installs only Events in the child and preserves selected authorizing rights.
@test unsafe fn pending() throws (testing::TestError) {
    initialize(abi::CREATE | abi::ALLOCATE);
    let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
    let authority = try! capability::get(&TABLE, handle);

kernel/kernel/tests/pages.rad +21 -0

    try pages::allocate(&mut STORE, &mut TABLE, other, 1) catch err {
        try testing::expect(err == abi::Error::Denied); set denied = true;
    };
    try testing::expect(denied and STORE.backings.pool.free[0] and TABLE.slots[2].state == slots::State::Free);
}

/// Buffer authority covers complete ranges and rejects overflow and stale objects.
@test unsafe fn bufferAuthority() throws (testing::TestError) {
    initialize();
    let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
    let entry = try! capability::get(&TABLE, handle);
    let page = try! pages::get(&STORE, entry.object);
    try testing::expect(pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::READ | abi::WRITE)));
    try testing::expect(pages::accessible(&STORE, &TABLE, page.base + 8191, 1, abi::Rights(abi::WRITE)));
    try testing::expect(not pages::accessible(&STORE, &TABLE, page.base + 8192, 1, abi::Rights(abi::READ)));
    try testing::expect(not pages::accessible(&STORE, &TABLE, page.base - 1, 2, abi::Rights(abi::READ)));
    try testing::expect(not pages::accessible(&STORE, &TABLE, page.base, 0xffffffffffffffff, abi::Rights(abi::READ)));
    let right = try! pages::split(&mut STORE, &mut TABLE, handle, 1);
    try testing::expect(pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::WRITE)));
    let rightSlot = try! abi::decode(right);
    set TABLE.entries[rightSlot.object.index].rights = abi::Rights(abi::READ);
    try testing::expect(not pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::WRITE)));
    try testing::expect(pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::READ)));
    set TABLE.entries[rightSlot.object.index].object.generation += 1;
    try testing::expect(not pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::READ)));
}

test/boot/run +2 -2

# Boot the kernel through its production initialization on each supported hart count.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/boot.ras"
cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/boot.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
for harts in 1 2 8; do
    # The deterministic emulator shares its tick budget across all harts.
    steps=$((10000000 * harts))
    if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then

if [ "$status" -ne 2 ] || [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
    cat "$work/log" >&2
    exit 1
fi
printf 'kernel startup: all eight harts reached machine idle without traps\n'
cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/trap.ras"
cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/trap.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
status=0
"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 || status=$?
if [ "$status" -ne 2 ] || [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
    || [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then

test/dispatch/kernel/dispatchcheck.rad +14 -4

    capability::initialize(&mut TABLE, owner);
    try! backing::registerDomain(&mut pages::STORE.backings, owner);
    let authority = try! capability::install(&mut TABLE, capability::Entry {
        kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE | abi::ALLOCATE),
    });
    let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
    let sourceEntry = try! capability::get(&TABLE, source);
    let page = try! pages::get(&pages::STORE, sourceEntry.object);
    let bytes = @sliceOf(memory(page.base), 4096);
    let bytes = @sliceOf(memory(page.base), page.count * 4096);
    let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
    let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
        loader::Request { authority, source, offset: 0, length: length as u64 });
    for i in 0..3 {
        set HANDLES[i] = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
        let cap = try! capability::get(&TABLE, HANDLES[i]);
        let child = try! domains::get(&domains::STORE, cap.object);
        set CONTEXTS[i] = child.initial;
        set domains::STORE.records[cap.object.index].state = domains::Lifecycle::Active;
        if i > 0 {
            set domains::STORE.records[cap.object.index].state = domains::Lifecycle::Active;
            set domains::STORE.contexts[child.initial.index].frame.status = 0x1880;
            set domains::STORE.contexts[child.initial.index].frame.registers[2] = domains::STORE.contexts[child.initial.index].kernelStack.end;
            set domains::STORE.contexts[child.initial.index].frame.registers[3] = kernelGp();
        }
    }
    let cap = try! capability::get(&TABLE, HANDLES[0]);
    let mut user = try! domains::get(&domains::STORE, cap.object);
    let selfAuthority = try! capability::install(&mut user.memory.table, capability::Entry {
        kind: abi::Kind::Domain, object: cap.object, rights: abi::Rights(abi::CREATE | abi::ALLOCATE),
    });
    let root = user.image;
    let executable = try! registry::install(&registry::STORE, &mut user.memory.table, root);
    let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
    let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ | abi::WRITE) as u64);
    let stackCap = try! capability::get(&TABLE, stack);
    let stackPage = try! pages::get(&pages::STORE, stackCap.object);
    set domains::STORE.contexts[CONTEXTS[0].index].frame.registers[2] = stackPage.base + 4096;
    let args = @sliceOf(memory(stackPage.base), 24);
    let words = [*granted, *executable, *selfAuthority];
    for word in 0..3 {
        for byte in 0..8 { set args[word * 8 + byte] = (words[word] >> (byte as u64 * 8)) as u8; }
    }
    try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, HANDLES[0], stackPage.base + 4096, stackPage.base, 24);
    let imageRef = try! registry::image(&registry::STORE, &TABLE, image, abi::Rights(abi::EXECUTE));
    let target = try! registry::exported(&registry::STORE, imageRef, &"spin::count"[..]);
    let case shared::Target::Data(data) = target else panic "counter";
    let timer = try! dispatch::timer(&boot::PLATFORM, 0);
    let clock = dispatch::now(timer);

test/dispatch/run +5 -2

# Preempt and resume U-mode and M-mode work through production dispatch.
set -eu
emulator=${RAD_EMULATOR:-emulator}
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT HUP INT TERM
mkdir "$work/spin"
cp test/dispatch/spin.rad "$work/spin.rad"
cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
    -pkg spin -mod test/dispatch/spin.rad -entry spin -ril "$work"
    -pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
cp kernel/kernel.rad "$work/kernel.rad"
printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
mkdir "$work/kernel"
cp kernel/kernel/*.rad "$work/kernel/"
cp test/dispatch/kernel/dispatchcheck.rad "$work/kernel/"

sh test/acceptance/compile "$emulator" "$work"
cat test/dispatch/machine.ras kernel/kernel/*.ras > "$work/dispatch.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
    -- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=80000000 -run "$work/dispatch.rv64"
printf 'dispatch: user preemption, idle gap, and retained machine continuation passed\n'
printf 'dispatch: user calls, preemption, idle gap, and retained machine continuation passed\n'

test/dispatch/spin.rad +80 -2

1	1		//! Non-yielding user code preempted by hardware timer boundaries.
	2	+	export mod abi;
	3	+	export mod sys;
	4	+	/// RV64 direct-call intrinsic used to check the machine ABI.
	5	+	@intrinsic fn ecall(number: u32, a0: i64, a1: i64, a2: i64, a3: i64) -> i64;
	6	+	/// Local capabilities supplied to the user fixture.
	7	+	record Arguments: Copy {
	8	+	/// Local capability for the stack allocation.
	9	+	stack: u64,
	10	+	/// Local capability for the resident user package.
	11	+	image: u64,
	12	+	/// Local self-Domain capability with allocation authority.
	13	+	domain: u64,
	14	+	}
	15	+
	16	+	/// Startup environment supplied by the kernel.
	17	+	record Env: Copy {
	18	+	/// Readable startup capabilities.
	19	+	argsPointer: *Arguments,
	20	+	/// Size of the startup word.
	21	+	argsSize: u64,
	22	+	/// Installed event handle.
	23	+	eventsHandle: u64,
	24	+	/// Shared event ring address.
	25	+	eventsPointer: u64,
	26	+	}
	27	+	/// Private query output used to check writable package-state authority.
	28	+	static INFO: [u64; 2] = [0; 2];
2	29		/// Private progress counter observed by the machine fixture.
3	30		export static count: u64 = 0;
4		-	/// Consume each assigned window without making a kernel call.
5		-	@default fn main() { while true { set count += 1; } }
	31	+	/// Check direct calls, then consume each window without yielding.
	32	+	@default unsafe fn main(env: *Env) {
	33	+	assert env.argsSize == 24;
	34	+	let handle = env.argsPointer.stack as i64;
	35	+	let info = sys::queryPage(abi::Handle(handle as u64));
	36	+	assert info.base <> 0 and info.count == 1;
	37	+	let mut rejected = false;
	38	+	try sys::pageAllocate(abi::Handle(0), 0) catch error {
	39	+	assert error == abi::Error::InvalidArg;
	40	+	set rejected = true;
	41	+	};
	42	+	assert rejected;
	43	+	let allocated = try! sys::pageAllocate(abi::Handle(0), 1);
	44	+	assert sys::queryPage(allocated).count == 1;
	45	+	let narrowed = try! sys::capabilityGrant(allocated, abi::Handle(0), abi::READ as u64);
	46	+	try! sys::capabilityDrop(narrowed);
	47	+	try! sys::capabilityDrop(allocated);
	48	+	assert sys::queryEvents(abi::Handle(env.eventsHandle)).capacity == 256;
	49	+	let domain = sys::queryDomain(abi::Handle(env.argsPointer.domain));
	50	+	assert domain.id <> 0 and domain.state == 1;
	51	+	let image = try! sys::queryImage(abi::Handle(env.argsPointer.image));
	52	+	assert image.entry <> 0 and image.codeBase <> 0 and image.codeBytes > 0 and image.dataBytes > 0;
	53	+	let child = try! sys::domainCreate(abi::Handle(0), abi::Handle(env.argsPointer.image));
	54	+	let stack = try! sys::pageAllocate(abi::Handle(0), 1);
	55	+	let granted = try! sys::capabilityGrant(stack, child, (abi::READ \| abi::WRITE) as u64);
	56	+	let childStack = sys::queryPage(stack);
	57	+	try! sys::domainActivate(child, childStack.base + 4096, (&env) as unsafe opaque, 0);
	58	+	assert sys::queryDomain(child).state == 1;
	59	+	set rejected = false;
	60	+	try sys::domainActivate(child, childStack.base + 4096, (&env) as unsafe opaque, 0) catch error {
	61	+	assert error == abi::Error::NotPending;
	62	+	set rejected = true;
	63	+	};
	64	+	assert rejected;
	65	+	set rejected = false;
	66	+	try sys::imageLoad(abi::Handle(0), stack, 0, 0) catch error {
	67	+	assert error == abi::Error::InvalidArg;
	68	+	set rejected = true;
	69	+	};
	70	+	assert rejected;
	71	+	let current = sys::currentContext();
	72	+	assert current.context <> 0 and current.hart == 0 and current.budget <> 0;
	73	+	let window = try! sys::queryBudget(abi::Handle(current.budget));
	74	+	assert window.context == current.context and window.hart == current.hart and window.start < window.end;
	75	+	let mut local: [u64; 2] = [0; 2];
	76	+	assert ecall(44, handle, (&mut local[0]) as i64, 16, 0) == 0;
	77	+	assert local[0] <> 0 and local[1] == 1;
	78	+	assert ecall(44, handle, (&mut INFO[0]) as i64, 16, 0) == 0;
	79	+	assert INFO[0] == local[0] and INFO[1] == local[1];
	80	+	assert ecall(44, handle, 0, 16, 0) == -4;
	81	+	assert ecall(0xffffffff, 0, 0, 0, 0) == -4;
	82	+	while true { set count += 1; }
	83	+	}

test/loader/run +1 -1

printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
od -An -v -tu1 "$work/loaded.ril" | awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
printf '];\n' >> "$work/kernel/loadinput.rad"
sh test/acceptance/compile "$emulator" "$work"
cat test/loader/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras \
    kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/loader.ras"
    kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/loader.ras"
"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
    -run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
printf 'runtime loader: hosted and loaded entries agree after workspace release\n'

265	265		try pages::allocate(&mut STORE, &mut TABLE, other, 1) catch err {
266	266		try testing::expect(err == abi::Error::Denied); set denied = true;
267	267		};
268	268		try testing::expect(denied and STORE.backings.pool.free[0] and TABLE.slots[2].state == slots::State::Free);
269	269		}
	270	+
	271	+	/// Buffer authority covers complete ranges and rejects overflow and stale objects.
	272	+	@test unsafe fn bufferAuthority() throws (testing::TestError) {
	273	+	initialize();
	274	+	let handle = try! pages::allocate(&mut STORE, &mut TABLE, abi::Handle(0), 2);
	275	+	let entry = try! capability::get(&TABLE, handle);
	276	+	let page = try! pages::get(&STORE, entry.object);
	277	+	try testing::expect(pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::READ \| abi::WRITE)));
	278	+	try testing::expect(pages::accessible(&STORE, &TABLE, page.base + 8191, 1, abi::Rights(abi::WRITE)));
	279	+	try testing::expect(not pages::accessible(&STORE, &TABLE, page.base + 8192, 1, abi::Rights(abi::READ)));
	280	+	try testing::expect(not pages::accessible(&STORE, &TABLE, page.base - 1, 2, abi::Rights(abi::READ)));
	281	+	try testing::expect(not pages::accessible(&STORE, &TABLE, page.base, 0xffffffffffffffff, abi::Rights(abi::READ)));
	282	+	let right = try! pages::split(&mut STORE, &mut TABLE, handle, 1);
	283	+	try testing::expect(pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::WRITE)));
	284	+	let rightSlot = try! abi::decode(right);
	285	+	set TABLE.entries[rightSlot.object.index].rights = abi::Rights(abi::READ);
	286	+	try testing::expect(not pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::WRITE)));
	287	+	try testing::expect(pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::READ)));
	288	+	set TABLE.entries[rightSlot.object.index].object.generation += 1;
	289	+	try testing::expect(not pages::accessible(&STORE, &TABLE, page.base, 8192, abi::Rights(abi::READ)));
	290	+	}

66	66		lib/std.test.rv64.s \
67	67		lib/std.test.rv64.o
68	68
69	69		# Kernel modules and tests use a separate package and test entry point.
70	70		KERNEL_SRC := kernel/kernel.rad $(shell find kernel/kernel -name '.rad' ! -name 'tests.rad' ! -path '/tests/*' 2>/dev/null)
71		-	KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras
	71	+	KERNEL_ASM := kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras
72	72		KERNEL_TEST_SRC := kernel/kernel/tests.rad $(shell find kernel/kernel/tests -name '*.rad' 2>/dev/null)
73	73		KERNEL := -pkg kernel $(patsubst %,-mod %,$(sort $(KERNEL_SRC)))
74	74		KERNEL_TEST := $(BIN_DIR)/kernel.test.rv64
75	75
76	76		kernel-test: emulator $(KERNEL_TEST)

20	20		export mod loader;
21	21		export mod instances;
22	22		export mod domains;
23	23		export mod budgets;
24	24		export mod dispatch;
	25	+	export mod calls;
	26	+	export mod sys;
25	27		export mod boot;
26	28		@test export mod tests;

3	3		/// Domain-relative capability value. Zero is the self sentinel where allowed.
4	4		export record Handle: Copy(u64);
5	5		/// Rights granted by a live capability-table entry.
6	6		export record Rights: Copy(u16);
7	7
	8	+	/// Startup values retained for the lifetime of a protection domain.
	9	+	export record Env: Copy {
	10	+	/// Address of the readable startup argument block.
	11	+	argsPointer: u64,
	12	+	/// Number of startup argument bytes.
	13	+	argsSize: u64,
	14	+	/// Installed domain-relative Events handle.
	15	+	eventsHandle: u64,
	16	+	/// Address of the domain's shared event ring.
	17	+	eventsPointer: u64,
	18	+	}
	19	+
	20	+	/// Physical extent returned by QueryPage.
	21	+	export record PageInfo: Copy {
	22	+	/// Address of the first physical frame.
	23	+	base: u64,
	24	+	/// Number of contiguous physical frames.
	25	+	count: u64,
	26	+	}
	27	+
	28	+	/// Domain identity and lifetime returned by QueryDomain.
	29	+	export record DomainInfo: Copy {
	30	+	/// Generation-bearing domain identity.
	31	+	id: u64,
	32	+	/// Historical creator identity.
	33	+	creator: u64,
	34	+	/// Current parent identity, or zero if absent.
	35	+	parent: u64,
	36	+	/// Pending, Active, or Dead encoded as zero, one, or two.
	37	+	state: u64,
	38	+	}
	39	+
	40	+	/// Shared notification queue shape returned by QueryEvents.
	41	+	export record EventsInfo: Copy {
	42	+	/// Number of entries in the installed ring.
	43	+	capacity: u64,
	44	+	}
	45	+
	46	+	/// Shared code and private state extents returned by QueryImage.
	47	+	export record ImageInfo: Copy {
	48	+	/// Generation-bearing resident package identity.
	49	+	id: u64,
	50	+	/// Default native entry, or zero if absent.
	51	+	entry: u64,
	52	+	/// Address of shared executable code.
	53	+	codeBase: u64,
	54	+	/// Number of shared executable bytes.
	55	+	codeBytes: u64,
	56	+	/// Number of private state bytes per instance.
	57	+	dataBytes: u64,
	58	+	}
	59	+
	60	+	/// Identity and CPU authority of the calling execution context.
	61	+	export record CurrentContextInfo: Copy {
	62	+	/// Generation-bearing context identity.
	63	+	context: u64,
	64	+	/// Physical hart executing this call.
	65	+	hart: u64,
	66	+	/// Domain-relative handle for the executing budget.
	67	+	budget: u64,
	68	+	}
	69	+
	70	+	/// Exclusive CPU authority returned by QueryBudget.
	71	+	export record BudgetInfo: Copy {
	72	+	/// Physical hart authorized by this window.
	73	+	hart: u64,
	74	+	/// Inclusive first mtime tick.
	75	+	start: u64,
	76	+	/// Exclusive last mtime tick.
	77	+	end: u64,
	78	+	/// Generation-bearing context identity, or zero if unbound.
	79	+	context: u64,
	80	+	}
	81	+
8	82		/// Read object contents or metadata.
9	83		export constant READ: u16 = 1;
10	84		/// Modify object contents.
11	85		export constant WRITE: u16 = 2;
12	86		/// Execute an image or entry.

20	20		/// Number of harts that validated their machine stack.
21	21		static ARRIVED: u64 = 0;
22	22		/// Trap anchors owned by their physical harts.
23	23		unsafe static HARTS: [trap::Hart; limits::HARTS] = undefined;
24	24
25		-	/// Handle timer boundaries and stop on unsupported traps.
	25	+	/// Handle timer boundaries and user calls, and stop on unsupported traps.
26	26		unsafe fn unexpected(frame: &mut trap::Frame, hart: &mut trap::Hart) {
27	27		if trap::classify(frame.cause) == trap::Cause::Timer {
28	28		dispatch::interrupt(frame, hart);
29	29		return;
30	30		}
	31	+	if trap::classify(frame.cause) == trap::Cause::UserCall and trap::fromUser(frame) {
	32	+	dispatch::call(frame, hart);
	33	+	return;
	34	+	}
31	35		print("kernel: unexpected trap\n");
32	36		trap::halt();
33	37		}
34	38
35	39		/// Read a byte from a device register with I/O ordering.

145	145		let removed = try! capability::invalidate(table, second);
146	146		try! slots::release(&mut store.slots[..], b);
147	147		return first;
148	148		}
149	149
	150	+	/// Relinquish a held interval and invalidate its sole capability.
	151	+	export fn drop(store: &mut Store, table: &mut capability::Table, handle: abi::Handle) throws (abi::Error) {
	152	+	let object = try owned(store, table, handle, 0);
	153	+	let removed = try! capability::invalidate(table, handle);
	154	+	try! slots::release(&mut store.slots[..], object);
	155	+	}
	156	+
150	157		/// Retire an expired interval and its sole handle.
151	158		export fn expire(store: &mut Store, table: &mut capability::Table, handle: abi::Handle, now: u64) -> bool throws (abi::Error) {
152	159		let object = try owned(store, table, handle, 0);
153	160		if now < store.windows[object.index].end { return false; }
154	161		let removed = try! capability::invalidate(table, handle);

6	6		use super::budgets;
7	7		use super::domains;
8	8		use super::platform;
9	9		use super::trap;
10	10		use super::range;
	11	+	use super::calls;
11	12
12	13		/// Dispatch ownership and idle state retained by one physical hart.
13	14		export record State: Copy {
14	15		/// Physical hart whose windows this state selects.
15	16		hart: u32,

69	70		let clock = now(HARTS[id].timer);
70	71		let choice = try! exchange(&mut HARTS[id].state, &budgets::STORE, &mut domains::STORE, frame, anchor, clock);
71	72		arm(HARTS[id].timer, choice.deadline);
72	73		}
73	74
	75	+	/// Read the running context's identity and domain-relative CPU authority.
	76	+	export unsafe fn current() -> abi::CurrentContextInfo {
	77	+	let id = hart();
	78	+	assert id < limits::HARTS;
	79	+	let context = HARTS[id].state.current else panic "no current context";
	80	+	let budget = HARTS[id].state.budget else panic "no current budget";
	81	+	let window = try! budgets::get(&budgets::STORE, budget);
	82	+	return abi::CurrentContextInfo { context: abi::id(context), hart: id as u64, budget: *window.handle };
	83	+	}
	84	+
	85	+	/// Complete the current user's call and select authority at the return boundary.
	86	+	export unsafe fn call(frame: &mut trap::Frame, anchor: &mut trap::Hart) {
	87	+	let id = hart();
	88	+	assert id < limits::HARTS;
	89	+	let context = HARTS[id].state.current else { trap::halt(); return; };
	90	+	assert slots::matches(&domains::STORE.contextSlots[..], context, slots::State::Live);
	91	+	calls::handle(domains::STORE.contexts[context.index].owner, frame, now(HARTS[id].timer));
	92	+	interrupt(frame, anchor);
	93	+	}
	94	+
74	95		/// Enter the first authorized context or the hart's retained idle frame.
75	96		export unsafe fn start(anchor: &mut trap::Hart) -> ! {
76	97		let id = hart();
77	98		assert id < limits::HARTS;
78	99		let mut frame = HARTS[id].state.idle;

9	9		use super::registry;
10	10		use super::instances;
11	11		use super::events;
12	12		use super::trap;
13	13		use super::range;
	14	+	use super::pages;
14	15
15	16		/// Retained kernel stack capacity for one suspended execution context.
16	17		export constant KERNEL_STACK_PAGES: u32 = 64;
17	18
18	19		/// Management rights returned to a domain's creator.

32	33		export record Memory: Copy {
33	34		/// Authoritative capabilities; only the kernel accesses this table.
34	35		table: capability::Table,
35	36		/// Event payload and consumer counters exposed to this domain.
36	37		ring: events::Ring,
	38	+	/// Startup values exposed read-only to this domain.
	39	+	env: abi::Env,
37	40		}
38	41
39	42		/// Saved integer state owned by one execution context.
40	43		export record Context: Copy {
41	44		/// Domain that owns this context.

103	106		export fn get(store: &Store, object: abi::Ref) -> Domain throws (abi::Error) {
104	107		if not slots::matches(&store.slots[..], object, slots::State::Live) { throw abi::Error::BadHandle; }
105	108		return store.records[object.index];
106	109		}
107	110
	111	+	/// Activate a pending domain after checking its stack and argument authority.
	112	+	/// The caller serializes metadata until the initial frame and state are published.
	113	+	export unsafe fn activate(store: &mut Store, memory: &pages::Store, table: &capability::Table,
	114	+	handle: abi::Handle, stack: u64, args: u64, size: u64) throws (abi::Error)
	115	+	{
	116	+	let entry = try capability::lookup(table, handle, abi::Kind::Domain, abi::Rights(abi::EXECUTE));
	117	+	let mut domain = try get(store, entry.object);
	118	+	if domain.state <> Lifecycle::Pending { throw abi::Error::NotPending; }
	119	+	if (stack & 15) <> 0 or stack < 16
	120	+	or not pages::accessible(memory, &domain.memory.table, stack - 16, 16, abi::Rights(abi::WRITE)) {
	121	+	throw abi::Error::InvalidArg;
	122	+	}
	123	+	if size <> 0 and not pages::accessible(memory, &domain.memory.table, args, size, abi::Rights(abi::READ)) {
	124	+	throw abi::Error::InvalidArg;
	125	+	}
	126	+	assert slots::matches(&store.contextSlots[..], domain.initial, slots::State::Live);
	127	+	set domain.memory.env = abi::Env {
	128	+	argsPointer: args, argsSize: size, eventsHandle: *domain.events,
	129	+	eventsPointer: (&domain.memory.ring) as u64,
	130	+	};
	131	+	set store.contexts[domain.initial.index].frame.registers[2] = stack;
	132	+	set store.contexts[domain.initial.index].frame.registers[10] = (&domain.memory.env) as u64;
	133	+	set store.records[entry.object.index].state = Lifecycle::Active;
	134	+	}
	135	+
108	136		/// Memory allocated before the domain and its context become observable.
109	137		record Prepared: Copy {
110	138		/// Frames for capabilities and the event ring.
111	139		allocation: frames::Run,
112	140		/// Mapped metadata at the allocation's start.

20	21		unsafe static PACKAGES: registry::Store = undefined;
21	22		/// Calling bootstrap domain's capability table.
22	23		unsafe static TABLE: capability::Table = undefined;
23	24		/// Executable image capability in the calling table.
24	25		static IMAGE: abi::Handle = abi::Handle(0);
	26	+	/// Page metadata used to check activation authority without reading user memory.
	27	+	unsafe static PAGES: pages::Store = undefined;
25	28
26	29		/// Initialize bootstrap authority and one executable image over mapped frames.
27	30		unsafe fn initialize(rights: u16) {
28	31		domains::initialize(&mut DOMAINS);
29	32		backing::initialize(&mut BACKINGS);

51	54		template: &[7], memory: 8, alignment: 8, relocations: &[],
52	55		});
53	56		set IMAGE = try! registry::install(&PACKAGES, &mut TABLE, object);
54	57		}
55	58
	59	+	/// Activation validates child authority before publishing its initial state.
	60	+	@test unsafe fn activation() throws (testing::TestError) {
	61	+	initialize(abi::CREATE);
	62	+	slots::initialize(&mut PAGES.slots[..]);
	63	+	let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
	64	+	let entry = try! capability::get(&TABLE, handle);
	65	+	let mut child = try! domains::get(&DOMAINS, entry.object);
	66	+	let pending = try! slots::reserve(&mut PAGES.slots[..]);
	67	+	let object = slots::reference(&pending);
	68	+	set PAGES.records[object.index] = pages::Page {
	69	+	backing: object, base: 0x90000000, count: 1, origin: entry.object, handles: 1,
	70	+	};
	71	+	let published = try! slots::commit(&mut PAGES.slots[..], pending);
	72	+	let memory = try! capability::install(&mut child.memory.table, capability::Entry {
	73	+	kind: abi::Kind::Page, object: published, rights: abi::Rights(abi::READ),
	74	+	});
	75	+	let mut failures: u32 = 0;
	76	+	try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0, 0) catch err {
	77	+	try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
	78	+	};
	79	+	let slot = try! abi::decode(memory);
	80	+	set child.memory.table.entries[slot.object.index].rights = abi::Rights(abi::READ \| abi::WRITE);
	81	+	try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90000fff, 0, 0) catch err {
	82	+	try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
	83	+	};
	84	+	try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90000000, 0, 0) catch err {
	85	+	try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
	86	+	};
	87	+	try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0x90000fff, 2) catch err {
	88	+	try testing::expect(err == abi::Error::InvalidArg); set failures += 1;
	89	+	};
	90	+	try testing::expect(failures == 4 and DOMAINS.records[entry.object.index].state == domains::Lifecycle::Pending);
	91	+	try testing::expect(DOMAINS.contexts[child.initial.index].frame.registers[2] == 0);
	92	+	try! domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0x90000000, 4096);
	93	+	try testing::expect(DOMAINS.records[entry.object.index].state == domains::Lifecycle::Active);
	94	+	let frame = DOMAINS.contexts[child.initial.index].frame;
	95	+	try testing::expect(frame.registers[2] == 0x90001000);
	96	+	try testing::expect(frame.registers[10] == (&child.memory.env) as u64);
	97	+	try testing::expect(child.memory.env.argsPointer == 0x90000000 and child.memory.env.argsSize == 4096);
	98	+	try testing::expect(child.memory.env.eventsHandle == *child.events);
	99	+	try testing::expect(child.memory.env.eventsPointer == (&child.memory.ring) as u64);
	100	+	try domains::activate(&mut DOMAINS, &PAGES, &TABLE, handle, 0x90001000, 0, 0) catch err {
	101	+	try testing::expect(err == abi::Error::NotPending); set failures += 1;
	102	+	};
	103	+	try testing::expect(failures == 5);
	104	+	}
	105	+
56	106		/// Creation installs only Events in the child and preserves selected authorizing rights.
57	107		@test unsafe fn pending() throws (testing::TestError) {
58	108		initialize(abi::CREATE \| abi::ALLOCATE);
59	109		let handle = try! domains::create(&mut DOMAINS, &mut BACKINGS, &PACKAGES, &mut TABLE, abi::Handle(0), IMAGE);
60	110		let authority = try! capability::get(&TABLE, handle);

2	2		# Boot the kernel through its production initialization on each supported hart count.
3	3		set -eu
4	4		emulator=${RAD_EMULATOR:-emulator}
5	5		work=$(mktemp -d)
6	6		trap 'rm -rf "$work"' EXIT HUP INT TERM
7		-	cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/boot.ras"
	7	+	cat test/boot/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/boot.ras"
8	8		"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/boot.ras" "$work/boot.rv64"
9	9		for harts in 1 2 8; do
10	10		# The deterministic emulator shares its tick budget across all harts.
11	11		steps=$((10000000 * harts))
12	12		if ! "$emulator" -machine -max-steps="$steps" -harts="$harts" -run "$work/boot.rv64" > "$work/log" 2>&1; then

28	28		if [ "$status" -ne 2 ] \|\| [ "$(grep -c 'wfi=1 mcause=0x0' "$work/log")" -ne 8 ]; then
29	29		cat "$work/log" >&2
30	30		exit 1
31	31		fi
32	32		printf 'kernel startup: all eight harts reached machine idle without traps\n'
33		-	cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/trap.ras"
	33	+	cat test/boot/trap.ras kernel/kernel/boot.ras kernel/kernel/sync.ras kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/trap.ras"
34	34		"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 -- bin/std.ril bin/kernel.ril "$work/trap.ras" "$work/trap.rv64"
35	35		status=0
36	36		"$emulator" -machine -harts=2 -max-steps=20000000 -run "$work/trap.rv64" > "$work/log" 2>&1 \|\| status=$?
37	37		if [ "$status" -ne 2 ] \|\| [ "$(grep -c '^kernel: unexpected trap$' "$work/log")" -ne 1 ] \
38	38		\|\| [ "$(grep -c 'wfi=1' "$work/log")" -ne 2 ]; then

38	38		capability::initialize(&mut TABLE, owner);
39	39		try! backing::registerDomain(&mut pages::STORE.backings, owner);
40	40		let authority = try! capability::install(&mut TABLE, capability::Entry {
41	41		kind: abi::Kind::Domain, object: owner, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
42	42		});
43		-	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
	43	+	let source = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, (dispatchinput::INPUT.len as u64 + 4095) / 4096);
44	44		let sourceEntry = try! capability::get(&TABLE, source);
45	45		let page = try! pages::get(&pages::STORE, sourceEntry.object);
46		-	let bytes = @sliceOf(memory(page.base), 4096);
	46	+	let bytes = @sliceOf(memory(page.base), page.count * 4096);
47	47		let length = try! mem::copy(&mut bytes[..], &dispatchinput::INPUT[..]);
48	48		let image = try! loader::load(&mut loader::STATE, &mut pages::STORE, &mut registry::STORE, &mut TABLE,
49	49		loader::Request { authority, source, offset: 0, length: length as u64 });
50	50		for i in 0..3 {
51	51		set HANDLES[i] = try! domains::create(&mut domains::STORE, &mut pages::STORE.backings, &registry::STORE, &mut TABLE, authority, image);
52	52		let cap = try! capability::get(&TABLE, HANDLES[i]);
53	53		let child = try! domains::get(&domains::STORE, cap.object);
54	54		set CONTEXTS[i] = child.initial;
55		-	set domains::STORE.records[cap.object.index].state = domains::Lifecycle::Active;
56	55		if i > 0 {
	56	+	set domains::STORE.records[cap.object.index].state = domains::Lifecycle::Active;
57	57		set domains::STORE.contexts[child.initial.index].frame.status = 0x1880;
58	58		set domains::STORE.contexts[child.initial.index].frame.registers[2] = domains::STORE.contexts[child.initial.index].kernelStack.end;
59	59		set domains::STORE.contexts[child.initial.index].frame.registers[3] = kernelGp();
60	60		}
61	61		}
62	62		let cap = try! capability::get(&TABLE, HANDLES[0]);
63	63		let mut user = try! domains::get(&domains::STORE, cap.object);
	64	+	let selfAuthority = try! capability::install(&mut user.memory.table, capability::Entry {
	65	+	kind: abi::Kind::Domain, object: cap.object, rights: abi::Rights(abi::CREATE \| abi::ALLOCATE),
	66	+	});
	67	+	let root = user.image;
	68	+	let executable = try! registry::install(&registry::STORE, &mut user.memory.table, root);
64	69		let stack = try! pages::allocate(&mut pages::STORE, &mut TABLE, authority, 1);
65	70		let granted = try! pages::grant(&mut pages::STORE, &TABLE, &mut user.memory.table, stack, (abi::READ \| abi::WRITE) as u64);
66	71		let stackCap = try! capability::get(&TABLE, stack);
67	72		let stackPage = try! pages::get(&pages::STORE, stackCap.object);
68		-	set domains::STORE.contexts[CONTEXTS[0].index].frame.registers[2] = stackPage.base + 4096;
	73	+	let args = @sliceOf(memory(stackPage.base), 24);
	74	+	let words = [granted, executable, *selfAuthority];
	75	+	for word in 0..3 {
	76	+	for byte in 0..8 { set args[word * 8 + byte] = (words[word] >> (byte as u64 * 8)) as u8; }
	77	+	}
	78	+	try! domains::activate(&mut domains::STORE, &pages::STORE, &TABLE, HANDLES[0], stackPage.base + 4096, stackPage.base, 24);
69	79		let imageRef = try! registry::image(&registry::STORE, &TABLE, image, abi::Rights(abi::EXECUTE));
70	80		let target = try! registry::exported(&registry::STORE, imageRef, &"spin::count"[..]);
71	81		let case shared::Target::Data(data) = target else panic "counter";
72	82		let timer = try! dispatch::timer(&boot::PLATFORM, 0);
73	83		let clock = dispatch::now(timer);

2	2		# Preempt and resume U-mode and M-mode work through production dispatch.
3	3		set -eu
4	4		emulator=${RAD_EMULATOR:-emulator}
5	5		work=$(mktemp -d)
6	6		trap 'rm -rf "$work"' EXIT HUP INT TERM
	7	+	mkdir "$work/spin"
	8	+	cp test/dispatch/spin.rad "$work/spin.rad"
	9	+	cp kernel/kernel/abi.rad kernel/kernel/sys.rad "$work/spin/"
7	10		"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/radiance.rv64.dev \
8		-	-pkg spin -mod test/dispatch/spin.rad -entry spin -ril "$work"
	11	+	-pkg spin -mod "$work/spin.rad" -mod "$work/spin/abi.rad" -mod "$work/spin/sys.rad" -entry spin -ril "$work"
9	12		cp kernel/kernel.rad "$work/kernel.rad"
10	13		printf '\nexport mod dispatchinput;\nexport mod dispatchcheck;\n' >> "$work/kernel.rad"
11	14		mkdir "$work/kernel"
12	15		cp kernel/kernel/*.rad "$work/kernel/"
13	16		cp test/dispatch/kernel/dispatchcheck.rad "$work/kernel/"

18	21		sh test/acceptance/compile "$emulator" "$work"
19	22		cat test/dispatch/machine.ras kernel/kernel/*.ras > "$work/dispatch.ras"
20	23		"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 -run bin/kernel.build.rv64 \
21	24		-- "$work/std.ril" "$work/kernel.ril" "$work/dispatch.ras" "$work/dispatch.rv64"
22	25		"$emulator" -machine -memory-size=262144 -max-steps=80000000 -run "$work/dispatch.rv64"
23		-	printf 'dispatch: user preemption, idle gap, and retained machine continuation passed\n'
	26	+	printf 'dispatch: user calls, preemption, idle gap, and retained machine continuation passed\n'

20	20		printf '//! Binary package used by both execution paths.\n/// Complete trusted binary RIL input.\nexport static INPUT: [u8; %s] = [\n' "$length" > "$work/kernel/loadinput.rad"
21	21		od -An -v -tu1 "$work/loaded.ril" \| awk '{ for (i = 1; i <= NF; i++) printf "%s,", $i; print "" }' >> "$work/kernel/loadinput.rad"
22	22		printf '];\n' >> "$work/kernel/loadinput.rad"
23	23		sh test/acceptance/compile "$emulator" "$work"
24	24		cat test/loader/machine.ras kernel/kernel/boot.ras kernel/kernel/sync.ras \
25		-	kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras > "$work/loader.ras"
	25	+	kernel/kernel/trap.ras kernel/kernel/pages.ras kernel/kernel/loader.ras kernel/kernel/instances.ras kernel/kernel/domains.ras kernel/kernel/dispatch.ras kernel/kernel/calls.ras > "$work/loader.ras"
26	26		"$emulator" -memory-size=385024 -data-size=348160 -stack-size=512 \
27	27		-run bin/kernel.build.rv64 -- "$work/std.ril" "$work/kernel.ril" "$work/loader.ras" "$work/loader.rv64"
28	28		"$emulator" -machine -memory-size=262144 -max-steps=200000000 -run "$work/loader.rv64"
29	29		printf 'runtime loader: hosted and loaded entries agree after workspace release\n'